Merge remote-tracking branch 'origin/litellm_internal_staging' into litellm_deflake_20260902

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

.github/actions/setup-uv-with-retries/action.yml

@@ -1,11 +1,7 @@
 name: "Set up uv with retries"
 description: >-
-  Install uv via astral-sh/setup-uv, retrying on transient failures. Even with
-  an exact pinned version, the action resolves the artifact URL by fetching
-  https://raw.githubusercontent.com/astral-sh/versions/main/v1/uv.ndjson in a
-  single request with no retry, timeout, or fallback, so one connection-level
-  network error ("fetch failed") fails the whole job before any test runs.
-  Retrying the full step covers the manifest fetch and the binary download.
+  Install uv via astral-sh/setup-uv, retrying the full setup step so manifest
+  resolution and binary downloads get fresh attempts after transient failures.
 
 inputs:
   version:
@@ -18,7 +14,7 @@ runs:
     - name: Set up uv (attempt 1)
       id: attempt-1
       continue-on-error: true
-      uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7.6.0
+      uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1
       with:
         version: ${{ inputs.version }}
 
@@ -31,7 +27,7 @@ runs:
       id: attempt-2
       if: steps.attempt-1.outcome == 'failure'
       continue-on-error: true
-      uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7.6.0
+      uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1
       with:
         version: ${{ inputs.version }}
 
@@ -42,6 +38,6 @@ runs:
 
     - name: Set up uv (attempt 3)
       if: steps.attempt-2.outcome == 'failure'
-      uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7.6.0
+      uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1
       with:
         version: ${{ inputs.version }}

.github/ci-coverage-allowlist.yml

@@ -4,6 +4,11 @@ description: >-
   by a job nor listed here, so every entry below is a decision on the record.
 
 test_paths:
+  - reason: >-
+      The Rust/Python parity harness is run manually through its local CLI. Recorded replay,
+      fixture generation, and harness checks are intentionally outside pull request CI
+    paths:
+      - tests/rust-python-harness
   - reason: >-
       What is left of the caching suite in tests/local_testing that runs nowhere. Every job that
       globs that directory either deselects it (local_testing_part1 and part2 carry `-k "... and
@@ -74,6 +79,11 @@ test_paths:
       - tests/load_tests/test_otel_load_test.py
       - tests/load_tests/test_vertex_embeddings_load_test.py
       - tests/load_tests/test_vertex_load_tests.py
+  - reason: >-
+      Env-gated saturation benchmark requires a live proxy and provider credentials, so it is run
+      locally rather than in pull-request jobs
+    paths:
+      - tests/load_tests/test_granian_admission_saturation.py
   - reason: >-
       A local-only agent rig: test_a2a_completion_bridge.py needs a LangGraph server on
       localhost:2024 and test_a2a.py drives a live A2A endpoint, so neither can run in a

.github/workflows/test-litellm-ui-build.yml

@@ -19,9 +19,6 @@ jobs:
   build-ui:
     runs-on: ubuntu-latest
     timeout-minutes: 10
-    defaults:
-      run:
-        working-directory: ui/litellm-dashboard
 
     steps:
       - name: Checkout repository
@@ -35,18 +32,11 @@ jobs:
         with:
           category: ui
 
-      - name: Setup Node.js
+      # Built through the image stage rather than the checkout, because the
+      # stage copies ui/litellm-dashboard/ alone: an import reaching above the
+      # dashboard root resolves in a checkout and fails in every image we ship.
+      # Dockerfile, docker/Dockerfile.non_root and ui/Dockerfile share this
+      # stage verbatim, so building one covers all three.
+      - name: Build the dashboard as the shipped images build it
         if: steps.changes.outputs.decision != 'skip'
-        uses: actions/setup-node@a0853c24544627f65ddf259abe73b1d18a591444 # v5.0.0
-        with:
-          node-version-file: ui/litellm-dashboard/.nvmrc
-          cache: "npm"
-          cache-dependency-path: ui/litellm-dashboard/package-lock.json
-
-      - name: Install dependencies
-        if: steps.changes.outputs.decision != 'skip'
-        run: npm ci
-
-      - name: Build
-        if: steps.changes.outputs.decision != 'skip'
-        run: npm run build
+        run: docker build --target ui-builder -f Dockerfile .

.github/workflows/test-rust.yml

@@ -74,12 +74,19 @@ jobs:
       - name: Run Clippy with Bedrock auth
         run: cargo clippy -p litellm-core --all-targets --features bedrock-auth --locked -- -D warnings
 
+      - name: Run Clippy with all gateway features
+        run: cargo clippy -p litellm-ai-gateway --all-targets --all-features --locked -- -D warnings
+
       - name: Run Rust tests
         run: cargo test --workspace --locked
 
       - name: Run core tests with Bedrock auth
         run: cargo test -p litellm-core --features bedrock-auth --locked
 
+      # Not --all-features: python-config links libpython, which this job does not install.
+      - name: Run gateway tests with the server feature
+        run: cargo test -p litellm-ai-gateway --features server --locked
+
   release-wheel:
     name: release wheel
     runs-on: ubuntu-latest

.github/workflows/test-unit.yml

@@ -96,7 +96,6 @@ jobs:
           - shard: misc
             artifact-name: misc
             test-path: >-
-              tests/sdk_function_trace
               tests/test_litellm/batches
               tests/test_litellm/secret_managers
               tests/test_litellm/a2a_protocol
@@ -151,6 +150,7 @@ jobs:
               tests/test_litellm/proxy/google_endpoints
               tests/test_litellm/proxy/openai_files_endpoint
               tests/test_litellm/proxy/batches_endpoints
+              tests/test_litellm/proxy/container_endpoints
               tests/test_litellm/proxy/fine_tuning_endpoints
               tests/test_litellm/proxy/vector_store_files_endpoints
               tests/test_litellm/proxy/video_endpoints

basedpyright-code-budget.json

@@ -3,7 +3,7 @@
     "limit": 14074
   },
   "reportArgumentType": {
-    "limit": 2215
+    "limit": 2206
   },
   "reportAssignmentType": {
     "limit": 319
@@ -42,7 +42,7 @@
     "limit": 12
   },
   "reportIndexIssue": {
-    "limit": 25
+    "limit": 24
   },
   "reportInvalidTypeForm": {
     "limit": 34
@@ -57,7 +57,7 @@
     "limit": 5601
   },
   "reportMissingTypeArgument": {
-    "limit": 15288
+    "limit": 15285
   },
   "reportMissingTypeStubs": {
     "limit": 40
@@ -99,19 +99,19 @@
     "limit": 0
   },
   "reportUnknownArgumentType": {
-    "limit": 44362
+    "limit": 44360
   },
   "reportUnknownLambdaType": {
     "limit": 109
   },
   "reportUnknownMemberType": {
-    "limit": 38324
+    "limit": 38311
   },
   "reportUnknownParameterType": {
-    "limit": 19625
+    "limit": 19624
   },
   "reportUnknownVariableType": {
-    "limit": 29861
+    "limit": 29847
   },
   "reportUnnecessaryCast": {
     "limit": 111

docker/build_from_pip/Dockerfile.build_from_pip

@@ -55,7 +55,10 @@ RUN HOME=/opt/prisma XDG_CACHE_HOME=/opt/prisma/.cache PRISMA_BINARY_CACHE_DIR=/
 
 ENV PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries
 
+COPY docker/prod_entrypoint.sh /app/docker/prod_entrypoint.sh
+RUN sed -i 's/\r$//' /app/docker/prod_entrypoint.sh && chmod +x /app/docker/prod_entrypoint.sh
+
 EXPOSE 4000/tcp
 
-ENTRYPOINT ["litellm"]
+ENTRYPOINT ["/app/docker/prod_entrypoint.sh"]
 CMD ["--port", "4000"]

docker/component_entrypoint.sh

@@ -1,8 +1,10 @@
 #!/bin/sh
 
-if [ "$USE_DDTRACE" = "true" ]; then
-    export DD_TRACE_OPENAI_ENABLED="False"
-    exec ddtrace-run "$@"
-fi
+case "$USE_DDTRACE" in
+    [Tt][Rr][Uu][Ee])
+        export DD_TRACE_OPENAI_ENABLED="False"
+        exec ddtrace-run "$@"
+        ;;
+esac
 
 exec "$@"

docker/prod_entrypoint.sh

@@ -1,8 +1,10 @@
 #!/bin/sh
 
-if [ "$USE_DDTRACE" = "true" ]; then
-    export DD_TRACE_OPENAI_ENABLED="False"
-    exec ddtrace-run litellm "$@"
-else
-    exec litellm "$@"
-fi
+case "$USE_DDTRACE" in
+    [Tt][Rr][Uu][Ee])
+        export DD_TRACE_OPENAI_ENABLED="False"
+        exec ddtrace-run litellm "$@"
+        ;;
+esac
+
+exec litellm "$@"

enterprise/litellm_enterprise/proxy/audit_logging_endpoints.py

@@ -48,6 +48,18 @@ def _build_json_field_or_condition(json_key: str, value: str) -> dict[str, objec
     }
 
 
+def _build_search_condition(search: str) -> dict[str, object]:
+    """Match a row whose id, changed_by, object_id, or changed_by_api_key equals the search value."""
+    return {
+        "OR": (
+            {"id": search},
+            {"changed_by": search},
+            {"object_id": search},
+            {"changed_by_api_key": search},
+        )
+    }
+
+
 @router.get(
     "/audit",
     tags=["Audit Logging"],
@@ -83,6 +95,10 @@ async def get_audit_logs(
         None,
         description="Filter by token (key hash) present in before_value or updated_values JSON (PostgreSQL only)",
     ),
+    search: str | None = Query(
+        None,
+        description="Match a row whose id, object_id, changed_by, or changed_by_api_key equals this value",
+    ),
     # Sorting parameters
     sort_by: str | None = Query(
         None,
@@ -118,6 +134,11 @@ async def get_audit_logs(
         *([_build_json_field_or_condition("token", object_key_hash)] if object_key_hash else []),
     ]
 
+    and_conditions: Final[tuple[dict[str, object], ...]] = (
+        *json_field_conditions,
+        *((_build_search_condition(search),) if search else ()),
+    )
+
     # Build filter conditions
     where_conditions: Final[dict[str, object]] = {
         **({"changed_by": changed_by} if changed_by else {}),
@@ -126,14 +147,14 @@ async def get_audit_logs(
         **({"table_name": table_name} if table_name else {}),
         **({"object_id": object_id} if object_id else {}),
         **({"updated_at": date_filter} if start_date or end_date else {}),
-        **({"AND": json_field_conditions} if json_field_conditions else {}),
+        **({"AND": and_conditions} if and_conditions else {}),
     }
 
     order_by: Final[dict[str, str]] = (
         {sort_by: sort_order} if sort_by and isinstance(sort_by, str) else {"updated_at": sort_order}
     )
 
-    audit_log_table: Final[TableActions["prisma_models.LiteLLM_AuditLog"]] = AuditLogRepository(prisma_client).table
+    audit_log_table: Final[TableActions[prisma_models.LiteLLM_AuditLog]] = AuditLogRepository(prisma_client).table
 
     # Get paginated results
     audit_logs: Final = await audit_log_table.find_many(
@@ -195,7 +216,7 @@ async def get_audit_log_by_id(
             detail={"message": CommonProxyErrors.db_not_connected_error.value},
         )
 
-    audit_log_table: Final[TableActions["prisma_models.LiteLLM_AuditLog"]] = AuditLogRepository(prisma_client).table
+    audit_log_table: Final[TableActions[prisma_models.LiteLLM_AuditLog]] = AuditLogRepository(prisma_client).table
 
     # Get the audit log by ID
     audit_log: Final = await audit_log_table.find_unique(where={"id": id})

enterprise/litellm_enterprise/proxy/hooks/managed_files.py

@@ -50,10 +50,12 @@ from litellm.proxy.openai_files_endpoints.common_utils import (
     MAX_FILE_LIST_LIMIT,
     _is_base64_encoded_unified_file_id,
     apply_unified_file_ids,
+    decode_model_from_file_id,
     ensure_batch_response_managed_file_ids,
     get_batch_id_from_unified_batch_id,
     get_content_type_from_file_object,
     get_model_id_from_unified_batch_id,
+    get_original_file_id,
     map_raw_file_ids_to_unified,
     normalize_mime_type_for_provider,
     resolve_managed_output_file_model_name,
@@ -427,6 +429,103 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
             detail=f"Object not found: {unified_object_id}",
         )
 
+    async def enforce_batch_object_access(
+        self, object_id: str, user_api_key_dict: UserAPIKeyAuth
+    ) -> None:
+        """Deny access to a provider-format batch id owned by another caller.
+
+        Ids with no ownership row (batches created before ownership tracking,
+        or directly on the provider account) stay accessible so pass-through
+        reads keep working.
+        """
+        if self.prisma_client is None:
+            return
+        managed_object = (
+            await self.prisma_client.db.litellm_managedobjecttable.find_first(
+                where={"OR": [{"unified_object_id": object_id}, {"model_object_id": object_id}]}
+            )
+        )
+        if managed_object is None:
+            return
+        if not can_access_resource(
+            user_api_key_dict=user_api_key_dict,
+            created_by=managed_object.created_by,
+            resource_team_id=managed_object.team_id,
+        ):
+            raise HTTPException(
+                status_code=403,
+                detail=f"User {user_api_key_dict.user_id} does not have access to the object {object_id}",
+            )
+
+    async def enforce_provider_file_access(
+        self, file_id: str, user_api_key_dict: UserAPIKeyAuth
+    ) -> None:
+        """Deny access to a provider-format file id owned by another caller.
+
+        Ownership rows for provider-format ids are written when a managed
+        batch's output/error files are first synced; ids with no row stay
+        accessible so pass-through reads keep working.
+        """
+        if self.prisma_client is None:
+            return
+        managed_file = (
+            await self.prisma_client.db.litellm_managedfiletable.find_first(
+                where={"OR": [{"unified_file_id": file_id}, {"flat_model_file_ids": {"has": file_id}}]}
+            )
+        )
+        if managed_file is None:
+            return
+        if not can_access_resource(
+            user_api_key_dict=user_api_key_dict,
+            created_by=managed_file.created_by,
+            resource_team_id=managed_file.team_id,
+        ):
+            raise HTTPException(
+                status_code=403,
+                detail=f"User {user_api_key_dict.user_id} does not have access to the file {file_id}",
+            )
+
+    async def store_batch_output_file_ownership(
+        self, response: LiteLLMBatch, litellm_parent_otel_span: Optional[Span]
+    ) -> None:
+        """Record ownership rows for a batch's provider-format output/error
+        file ids, inherited from the owning batch row (never the caller), so
+        file reads can be isolation-checked."""
+        provider_file_ids = tuple(
+            file_id
+            for file_id in (
+                getattr(response, "output_file_id", None),
+                getattr(response, "error_file_id", None),
+            )
+            if file_id and not _is_base64_encoded_unified_file_id(file_id)
+        )
+        if not provider_file_ids:
+            return
+        if self.prisma_client is None:
+            return
+        batch_row = (
+            await self.prisma_client.db.litellm_managedobjecttable.find_first(
+                where={"unified_object_id": response.id}
+            )
+        )
+        if batch_row is None or (
+            batch_row.created_by is None and batch_row.team_id is None
+        ):
+            return
+        owner_identity = UserAPIKeyAuth(
+            user_id=batch_row.created_by, team_id=batch_row.team_id
+        )
+        for file_id in provider_file_ids:
+            model_name = decode_model_from_file_id(file_id)
+            raw_file_id = get_original_file_id(file_id)
+            await self.store_unified_file_id(
+                file_id=file_id,
+                file_object=None,
+                litellm_parent_otel_span=litellm_parent_otel_span,
+                model_mappings={model_name: raw_file_id} if model_name else {},
+                user_api_key_dict=owner_identity,
+            )
+
     async def list_user_batches(
         self,
         user_api_key_dict: UserAPIKeyAuth,
@@ -613,6 +712,10 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
                     status_code=403,
                     detail=f"User {user_api_key_dict.user_id} does not have access to the file {retrieve_file_id}",
                 )
+        if retrieve_file_id:
+            await self.enforce_provider_file_access(
+                retrieve_file_id, user_api_key_dict
+            )
         return False
 
     async def check_file_ids_access(self, file_ids: List[str], user_api_key_dict: UserAPIKeyAuth) -> None:
@@ -765,6 +868,8 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
                     )
                 data["model"] = potential_model_id
                 data[accessor_key] = get_batch_id_from_unified_batch_id(potential_llm_object_id)
+            elif retrieve_object_id and accessor_key == "batch_id":
+                await self.enforce_batch_object_access(retrieve_object_id, user_api_key_dict)
         elif call_type == CallTypes.acreate_fine_tuning_job.value:
             input_file_id = cast(Optional[str], data.get("training_file"))
             if input_file_id:
@@ -1297,7 +1402,13 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
                 user_api_key_dict=user_api_key_dict,
                 request_tags=request_tags_from_metadata(request_metadata if isinstance(request_metadata, dict) else {}),
                 persist_attribution=is_batch_create,
+                create_if_missing=is_batch_create,
             )
+            if not is_batch_create:
+                await self.store_batch_output_file_ownership(
+                    response=response,
+                    litellm_parent_otel_span=user_api_key_dict.parent_otel_span,
+                )
 
             # Only record batch creation metric on actual create (not retrieve/cancel).
             # unified_file_id in _hidden_params is only set by the create_batch endpoint.

enterprise/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "litellm-enterprise"
-version = "0.1.63"
+version = "0.1.64"
 description = "Package for LiteLLM Enterprise features"
 readme = "README.md"
 requires-python = ">=3.9"
@@ -26,7 +26,7 @@ required-version = ">=0.10.9"
 module-root = ""
 
 [tool.commitizen]
-version = "0.1.63"
+version = "0.1.64"
 version_files = [
     "pyproject.toml:^version",
     "../pyproject.toml:litellm-enterprise==",

helm/litellm/templates/_helpers.tpl

@@ -428,3 +428,16 @@ envFrom:
 {{- end }}
 {{- end }}
 {{- end -}}
+
+{{/*
+ingress-nginx's admission webhook rejects a dot in an Exact or Prefix path
+(strict-validate-path-type) and serves ImplementationSpecific as a plain
+prefix location, so a dotted path takes that type there.
+*/}}
+{{- define "litellm.ingress.pathType" -}}
+{{- if and (eq .controller "nginx") (contains "." .path) -}}
+ImplementationSpecific
+{{- else -}}
+{{- .pathType -}}
+{{- end -}}
+{{- end -}}

helm/litellm/templates/ingress.yaml

@@ -5,6 +5,10 @@
 {{- $gatewayPort := .Values.gateway.service.port -}}
 {{- $backendPort := .Values.backend.service.port -}}
 {{- $uiPort := .Values.ui.service.port -}}
+{{- $controller := .Values.ingress.controller | default "alb" -}}
+{{- if not (has $controller (list "alb" "nginx")) }}
+{{- fail (printf "ingress.controller: unknown controller %q, expected one of alb, nginx" $controller) }}
+{{- end }}
 {{/*
   Backends addressable from ingress.extraPaths, keyed by the `service` field.
 */}}
@@ -27,10 +31,11 @@
   /litellm-asset-prefix, so without /*.txt they fall to the backend catch-all
   → 404 → client-side navigation never settles and the login flow spins in an
   infinite redirect loop (/ ⇄ /ui/login). ui/nginx.conf already serves *.txt
-  from the export; the rule only routes the request to it. Needs an ingress
-  controller whose ImplementationSpecific path is a wildcard pattern
-  (AWS ALB: `*` = 0+ chars); this chart targets the AWS Load Balancer
-  Controller.
+  from the export; the rule only routes the request to it. It needs an
+  ingress controller whose ImplementationSpecific path is a wildcard pattern
+  (AWS ALB: `*` = 0+ chars), so it is rendered for ingress.controller=alb
+  only: ingress-nginx serves ImplementationSpecific as a literal prefix
+  location, where /*.txt can never match.
 */}}
 {{- $uiPaths := list
     (dict "path" "/" "pathType" "Exact")
@@ -38,8 +43,10 @@
     (dict "path" "/litellm-asset-prefix" "pathType" "Prefix")
     (dict "path" "/_next" "pathType" "Prefix")
     (dict "path" "/ui" "pathType" "Prefix")
-    (dict "path" "/*.txt" "pathType" "ImplementationSpecific")
 -}}
+{{- if eq $controller "alb" }}
+{{- $uiPaths = append $uiPaths (dict "path" "/*.txt" "pathType" "ImplementationSpecific") }}
+{{- end }}
 {{/*
   Gateway data-plane prefixes — must mirror gateway/routes/allowlist.py.
   Versioned paths are listed explicitly to avoid routing management routes
@@ -83,12 +90,6 @@
   adding to it.
 */}}
 {{- $builtinPathKeys := list "/test|Exact" "/|Prefix" -}}
-{{- range $uiPaths }}
-{{- $builtinPathKeys = append $builtinPathKeys (printf "%s|%s" .path .pathType) }}
-{{- end }}
-{{- range $gatewayPrefixes }}
-{{- $builtinPathKeys = append $builtinPathKeys (printf "%s|Prefix" .) }}
-{{- end }}
 apiVersion: networking.k8s.io/v1
 kind: Ingress
 metadata:
@@ -115,8 +116,10 @@ spec:
         paths:
           # --- UI (Next.js static export) ---
           {{- range $uiPaths }}
+          {{- $pathType := include "litellm.ingress.pathType" (dict "controller" $controller "path" .path "pathType" .pathType) }}
+          {{- $builtinPathKeys = append $builtinPathKeys (printf "%s|%s" .path $pathType) }}
           - path: {{ .path }}
-            pathType: {{ .pathType }}
+            pathType: {{ $pathType }}
             backend:
               service:
                 name: {{ $uiName }}
@@ -134,8 +137,10 @@ spec:
                 port:
                   number: {{ $gatewayPort }}
           {{- range $gatewayPrefixes }}
+          {{- $pathType := include "litellm.ingress.pathType" (dict "controller" $controller "path" . "pathType" "Prefix") }}
+          {{- $builtinPathKeys = append $builtinPathKeys (printf "%s|%s" . $pathType) }}
           - path: {{ . }}
-            pathType: Prefix
+            pathType: {{ $pathType }}
             backend:
               service:
                 name: {{ $gatewayName }}
@@ -147,10 +152,11 @@ spec:
             Rendered after every built-in path so an entry can never take
             precedence over a default, and before the backend catch-all.
             Position only decides the match on controllers that honour manifest
-            order: the AWS Load Balancer Controller this chart targets sorts
-            Exact paths first and Prefix paths longest-first, but keeps
+            order: the AWS Load Balancer Controller (ingress.controller=alb)
+            sorts Exact paths first and Prefix paths longest-first, but keeps
             ImplementationSpecific paths in manifest order, which is what the
-            /*.txt rule above already depends on.
+            /*.txt rule above already depends on. ingress-nginx ignores order
+            and serves the longest matching location.
           */}}
           {{- range $idx, $extra := .Values.ingress.extraPaths }}
           {{- if not (kindIs "map" $extra) }}
@@ -164,10 +170,11 @@ spec:
           {{- if not $target }}
           {{- fail (printf "ingress.extraPaths[%d] (path %s): unknown service %q, expected one of backend, gateway, ui" $idx $extra.path $service) }}
           {{- end }}
-          {{- $pathType := $extra.pathType | default "Prefix" }}
-          {{- if not (has $pathType (list "Prefix" "Exact" "ImplementationSpecific")) }}
-          {{- fail (printf "ingress.extraPaths[%d] (path %s): unknown pathType %q, expected one of Exact, ImplementationSpecific, Prefix" $idx $extra.path $pathType) }}
+          {{- $requestedPathType := $extra.pathType | default "Prefix" }}
+          {{- if not (has $requestedPathType (list "Prefix" "Exact" "ImplementationSpecific")) }}
+          {{- fail (printf "ingress.extraPaths[%d] (path %s): unknown pathType %q, expected one of Exact, ImplementationSpecific, Prefix" $idx $extra.path $requestedPathType) }}
           {{- end }}
+          {{- $pathType := include "litellm.ingress.pathType" (dict "controller" $controller "path" $extra.path "pathType" $requestedPathType) }}
           {{- if eq $extra.path "/" }}
           {{- fail (printf "ingress.extraPaths[%d]: path / is already routed in both directions, Exact to ui and Prefix to backend, so no pathType leaves a request for an entry here to capture" $idx) }}
           {{- end }}

helm/litellm/tests/ingress_controller_tests.yaml

@@ -0,0 +1,205 @@
+suite: test ingress.controller
+templates:
+  - ingress.yaml
+values:
+  - ./values/required.yaml
+tests:
+  - it: keeps the AWS Load Balancer Controller path types by default
+    set:
+      ingress.enabled: true
+    asserts:
+      - contains:
+          path: spec.rules[0].http.paths
+          content:
+            path: /favicon.ico
+            pathType: Exact
+            backend:
+              service:
+                name: RELEASE-NAME-litellm-ui
+                port:
+                  number: 3000
+      - contains:
+          path: spec.rules[0].http.paths
+          content:
+            path: /eu.assemblyai
+            pathType: Prefix
+            backend:
+              service:
+                name: RELEASE-NAME-litellm-gateway
+                port:
+                  number: 4000
+      - contains:
+          path: spec.rules[0].http.paths
+          content:
+            path: /*.txt
+            pathType: ImplementationSpecific
+            backend:
+              service:
+                name: RELEASE-NAME-litellm-ui
+                port:
+                  number: 3000
+
+  - it: renders no dotted Exact or Prefix path for ingress-nginx, whose admission webhook rejects them
+    set:
+      ingress.enabled: true
+      ingress.controller: nginx
+    asserts:
+      - notMatchRegexRaw:
+          pattern: 'path: /\S*\.\S*\n\s+pathType: (Exact|Prefix)\n'
+      - contains:
+          path: spec.rules[0].http.paths
+          content:
+            path: /favicon.ico
+            pathType: ImplementationSpecific
+            backend:
+              service:
+                name: RELEASE-NAME-litellm-ui
+                port:
+                  number: 3000
+      - contains:
+          path: spec.rules[0].http.paths
+          content:
+            path: /eu.assemblyai
+            pathType: ImplementationSpecific
+            backend:
+              service:
+                name: RELEASE-NAME-litellm-gateway
+                port:
+                  number: 4000
+
+  - it: drops the /*.txt wildcard for ingress-nginx and keeps every other route as is
+    set:
+      ingress.enabled: true
+      ingress.controller: nginx
+    asserts:
+      - notContains:
+          path: spec.rules[0].http.paths
+          content:
+            path: /*.txt
+          any: true
+      - contains:
+          path: spec.rules[0].http.paths
+          content:
+            path: /ui
+            pathType: Prefix
+            backend:
+              service:
+                name: RELEASE-NAME-litellm-ui
+                port:
+                  number: 3000
+      - contains:
+          path: spec.rules[0].http.paths
+          content:
+            path: /test
+            pathType: Exact
+            backend:
+              service:
+                name: RELEASE-NAME-litellm-gateway
+                port:
+                  number: 4000
+      - equal:
+          path: spec.rules[0].http.paths[-1]
+          value:
+            path: /
+            pathType: Prefix
+            backend:
+              service:
+                name: RELEASE-NAME-litellm-backend
+                port:
+                  number: 4001
+
+  - it: rejects an extraPaths entry that repeats a built-in path at the pathType ingress-nginx renders it with
+    set:
+      ingress.enabled: true
+      ingress.controller: nginx
+      ingress.extraPaths:
+        - path: /favicon.ico
+          service: ui
+          pathType: ImplementationSpecific
+    asserts:
+      - failedTemplate:
+          errorMessage: "ingress.extraPaths[0]: path /favicon.ico with pathType ImplementationSpecific is already routed by this chart, and a duplicate would take it over rather than add to it"
+
+  - it: rejects a controller it has no path types for
+    set:
+      ingress.enabled: true
+      ingress.controller: traefik
+    asserts:
+      - failedTemplate:
+          errorMessage: 'ingress.controller: unknown controller "traefik", expected one of alb, nginx'
+
+  - it: rejects an extraPaths entry that repeats a built-in path once ingress-nginx normalizes its pathType
+    set:
+      ingress.enabled: true
+      ingress.controller: nginx
+      ingress.extraPaths:
+        - path: /favicon.ico
+          service: ui
+    asserts:
+      - failedTemplate:
+          errorMessage: "ingress.extraPaths[0]: path /favicon.ico with pathType ImplementationSpecific is already routed by this chart, and a duplicate would take it over rather than add to it"
+
+  - it: renders a dotted extraPaths entry as ImplementationSpecific for ingress-nginx
+    set:
+      ingress.enabled: true
+      ingress.controller: nginx
+      ingress.extraPaths:
+        - path: /eu.assemblyai.custom
+          service: gateway
+        - path: /robots.txt
+          service: ui
+          pathType: Exact
+    asserts:
+      - notMatchRegexRaw:
+          pattern: 'path: "?/\S*\.\S*"?\n\s+pathType: (Exact|Prefix)\n'
+      - contains:
+          path: spec.rules[0].http.paths
+          content:
+            path: /eu.assemblyai.custom
+            pathType: ImplementationSpecific
+            backend:
+              service:
+                name: RELEASE-NAME-litellm-gateway
+                port:
+                  number: 4000
+      - contains:
+          path: spec.rules[0].http.paths
+          content:
+            path: /robots.txt
+            pathType: ImplementationSpecific
+            backend:
+              service:
+                name: RELEASE-NAME-litellm-ui
+                port:
+                  number: 3000
+
+  - it: keeps the requested pathType of a dotted extraPaths entry for the AWS Load Balancer Controller
+    set:
+      ingress.enabled: true
+      ingress.extraPaths:
+        - path: /eu.assemblyai.custom
+          service: gateway
+        - path: /robots.txt
+          service: ui
+          pathType: Exact
+    asserts:
+      - contains:
+          path: spec.rules[0].http.paths
+          content:
+            path: /eu.assemblyai.custom
+            pathType: Prefix
+            backend:
+              service:
+                name: RELEASE-NAME-litellm-gateway
+                port:
+                  number: 4000
+      - contains:
+          path: spec.rules[0].http.paths
+          content:
+            path: /robots.txt
+            pathType: Exact
+            backend:
+              service:
+                name: RELEASE-NAME-litellm-ui
+                port:
+                  number: 3000

helm/litellm/values.yaml

@@ -10,6 +10,18 @@ imagePullSecrets: []
 ingress:
   enabled: false
   className: ""
+  # Which ingress controller serves this Ingress. Controllers disagree on the
+  # pathTypes they accept, so this picks the pathType of the dotted paths, the
+  # built-in ones and any dotted extraPaths entry alike:
+  #   alb    AWS Load Balancer Controller (default): Exact and Prefix paths plus
+  #          the /*.txt wildcard that routes the UI's RSC payloads.
+  #   nginx  ingress-nginx: its admission webhook rejects a dot in an Exact or
+  #          Prefix path (strict-validate-path-type, on by default from v1.12.0
+  #          until v1.12.6 / v1.13.2 allowed dots again), so /favicon.ico and
+  #          /eu.assemblyai render as ImplementationSpecific, which nginx serves
+  #          as a plain prefix location. /*.txt is dropped: nginx has no
+  #          wildcard pathType, so that rule could never match there.
+  controller: alb
   annotations: {}
   host: ""        # optional; if set, becomes the rule's host
   tls: []
@@ -26,7 +38,8 @@ ingress:
   #
   #   path      required; the HTTP path to route
   #   service   which component serves it: gateway (default), backend, or ui
-  #   pathType  Prefix (default), Exact, or ImplementationSpecific
+  #   pathType  Prefix (default), Exact, or ImplementationSpecific; a dotted
+  #             path renders as ImplementationSpecific when controller is nginx
   #
   # The target component only answers paths its own route allowlist keeps, so
   # a path here still has to be one that component serves.

litellm-proxy-extras/litellm_proxy_extras/prisma_toolchain.py

@@ -18,10 +18,15 @@ recoverable one.
 constant: it grows with the number of pending migrations, so a fresh database
 that has to replay every migration this package ships overruns a per-command
 budget sized for the short bookkeeping commands, on a laptop as much as on a
-slow CI runner. The Python ``prisma`` wrapper spawns Node and the schema engine
-as separate children, so killing the wrapper on timeout leaves them running:
-the retry then contends with that orphan for Prisma's advisory lock and cannot
-finish any sooner. Migrate deploy therefore runs under its own budget.
+slow CI runner. Migrate deploy therefore runs under its own budget.
+
+The Python ``prisma`` wrapper spawns Node, which spawns the Rust schema
+engine, so killing only the wrapper on timeout leaves the engine running with
+no parent: it keeps mutating the database after the proxy has given up, holds
+Prisma's advisory lock so every retry and every later boot queues behind it,
+and dies mid-migration once its pipes close, leaving a half-applied ledger row.
+Every Prisma command therefore runs in a process group of its own, and a
+timeout kills the whole group.
 
 All three budgets are overridable so an operator can widen them without a
 release: ``LITELLM_PRISMA_BOOTSTRAP_TIMEOUT`` for the toolchain install,
@@ -35,10 +40,12 @@ the deploy override says otherwise.
 import math
 import os
 import shutil
+import signal
 import subprocess
+from collections.abc import Mapping, Sequence
 from dataclasses import dataclass
 from pathlib import Path
-from typing import Optional
+from typing import IO, Optional, Union
 
 from litellm_proxy_extras._logging import logger
 
@@ -167,6 +174,49 @@ def heal_incomplete_nodeenv_cache() -> bool:
     return True
 
 
+def _kill_process_group(process: "subprocess.Popen[str]") -> None:
+    if os.name == "nt":
+        process.kill()
+        return
+    try:
+        os.killpg(process.pid, signal.SIGKILL)
+    except ProcessLookupError:
+        return
+
+
+def run_prisma(
+    argv: Sequence[str],
+    *,
+    timeout: float,
+    env: Mapping[str, str],
+    stdout: Union[IO[str], int, None] = subprocess.PIPE,
+    stderr: Optional[int] = subprocess.PIPE,
+) -> "subprocess.CompletedProcess[str]":
+    """Run one Prisma CLI command in its own process group, bounded by ``timeout``.
+
+    Raises ``subprocess.TimeoutExpired`` once the budget is spent, after killing
+    the command together with every process it spawned, and
+    ``subprocess.CalledProcessError`` on a non-zero exit. Output is captured as
+    text unless ``stdout``/``stderr`` say otherwise.
+    """
+    with subprocess.Popen(
+        argv,
+        env=env,
+        stdout=stdout,
+        stderr=stderr,
+        text=True,
+        start_new_session=True,
+    ) as process:
+        try:
+            out, err = process.communicate(timeout=timeout)
+        except BaseException:
+            _kill_process_group(process)
+            raise
+    if process.returncode:
+        raise subprocess.CalledProcessError(process.returncode, process.args, out, err)
+    return subprocess.CompletedProcess(process.args, process.returncode, out, err)
+
+
 def ensure_prisma_toolchain(
     prisma_command: str, prisma_env: dict[str, str]
 ) -> ToolchainBootstrap:
@@ -179,14 +229,7 @@ def ensure_prisma_toolchain(
     timeout = prisma_bootstrap_timeout()
     logger.info("Preparing the Prisma CLI toolchain (timeout %ss)", timeout)
     try:
-        subprocess.run(
-            [prisma_command, BOOTSTRAP_ARG],
-            timeout=timeout,
-            check=True,
-            capture_output=True,
-            text=True,
-            env=prisma_env,
-        )
+        run_prisma([prisma_command, BOOTSTRAP_ARG], timeout=timeout, env=prisma_env)
     except subprocess.TimeoutExpired:
         logger.warning(
             "Preparing the Prisma CLI toolchain timed out after %ss. Raise %s "

litellm-proxy-extras/litellm_proxy_extras/replica_identity.py

@@ -16,7 +16,7 @@ import tempfile
 from pathlib import Path
 
 from litellm_proxy_extras._logging import logger
-from litellm_proxy_extras.prisma_toolchain import prisma_command_timeout
+from litellm_proxy_extras.prisma_toolchain import prisma_command_timeout, run_prisma
 
 REPLICA_IDENTITY_FULL_ENV_VAR = "LITELLM_SET_REPLICA_IDENTITY_FULL"
 
@@ -66,7 +66,7 @@ def apply_replica_identity_full(
         with tempfile.TemporaryDirectory(prefix="litellm_replica_identity_") as tmp_dir:
             sql_path = Path(tmp_dir) / "replica_identity_full.sql"
             sql_path.write_text(REPLICA_IDENTITY_FULL_SQL)
-            subprocess.run(
+            run_prisma(
                 [
                     prisma_command,
                     "db",
@@ -77,9 +77,6 @@ def apply_replica_identity_full(
                     schema_path,
                 ],
                 timeout=prisma_command_timeout(),
-                check=True,
-                capture_output=True,
-                text=True,
                 env=prisma_env,
             )
     except subprocess.CalledProcessError as e:

litellm-proxy-extras/litellm_proxy_extras/utils.py

@@ -6,9 +6,11 @@ import shutil
 import subprocess
 import tempfile
 import time
+from dataclasses import dataclass, replace
 from pathlib import Path
 from typing import Optional
 
+from litellm_proxy_extras import prisma_toolchain
 from litellm_proxy_extras._logging import logger
 from litellm_proxy_extras.replica_identity import (
     REPLICA_IDENTITY_FULL_ENV_VAR,
@@ -45,6 +47,38 @@ _MIGRATION_TS_RE = re.compile(r"^(\d{14})_")
 
 _MIGRATION_DEADLOCK_MARKER = "deadlock detected"
 
+MAX_MIGRATE_DEPLOY_ATTEMPTS = 4
+
+
+@dataclass(frozen=True)
+class _MigrateAttemptBudget:
+    """Retries left, and the recoveries already run.
+
+    A recovery that lands something new costs nothing, so a database full of
+    objects `prisma db push` created works through them one per pass. Anything
+    that made no progress spends an attempt, so a stuck run still gives up.
+    """
+
+    attempts_left: int
+    recoveries: frozenset[str] = frozenset()
+
+    @property
+    def exhausted(self) -> bool:
+        return self.attempts_left <= 0
+
+    @property
+    def attempt_number(self) -> int:
+        return MAX_MIGRATE_DEPLOY_ATTEMPTS - self.attempts_left + 1
+
+    def spend(self) -> "_MigrateAttemptBudget":
+        return replace(self, attempts_left=self.attempts_left - 1)
+
+    def after_recovery(self, recovery: str) -> "_MigrateAttemptBudget":
+        if recovery in self.recoveries:
+            return self.spend()
+        return replace(self, recoveries=self.recoveries | {recovery})
+
+
 _SPEND_LOGS_ALTER_RE = re.compile(r'^ALTER\s+TABLE\s+"LiteLLM_SpendLogs"\s', re.IGNORECASE)
 _SPEND_LOGS_ARTIFACT_DROP_RE = re.compile(
     r'^DROP\s+TABLE\s+"LiteLLM_SpendLogs_[^"]*"', re.IGNORECASE
@@ -198,7 +232,7 @@ class ProxyExtrasDBManager:
             # 1. Generate migration SQL file by comparing empty state to current db state
             logger.info("Generating baseline migration...")
             migration_file = init_dir / "migration.sql"
-            subprocess.run(
+            prisma_toolchain.run_prisma(
                 [
                     _get_prisma_command(),
                     "migrate",
@@ -209,14 +243,13 @@ class ProxyExtrasDBManager:
                     "--script",
                 ],
                 stdout=open(migration_file, "w"),
-                check=True,
                 timeout=prisma_command_timeout(),
                 env=prisma_env,
             )
 
             # 3. Mark the migration as applied since it represents current state
             logger.info("Marking baseline migration as applied...")
-            subprocess.run(
+            prisma_toolchain.run_prisma(
                 [
                     _get_prisma_command(),
                     "migrate",
@@ -224,7 +257,6 @@ class ProxyExtrasDBManager:
                     "--applied",
                     "0_init",
                 ],
-                check=True,
                 timeout=prisma_command_timeout(),
                 env=prisma_env,
             )
@@ -253,7 +285,7 @@ class ProxyExtrasDBManager:
         """Mark a specific migration as rolled back"""
         # Set up environment for offline mode if configured
         prisma_env = _get_prisma_env()
-        subprocess.run(
+        prisma_toolchain.run_prisma(
             [
                 _get_prisma_command(),
                 "migrate",
@@ -262,8 +294,6 @@ class ProxyExtrasDBManager:
                 migration_name,
             ],
             timeout=prisma_command_timeout(),
-            check=True,
-            capture_output=True,
             env=prisma_env,
         )
 
@@ -315,11 +345,9 @@ class ProxyExtrasDBManager:
     def _resolve_specific_migration(migration_name: str):
         """Mark a specific migration as applied"""
         prisma_env = _get_prisma_env()
-        subprocess.run(
+        prisma_toolchain.run_prisma(
             [_get_prisma_command(), "migrate", "resolve", "--applied", migration_name],
             timeout=prisma_command_timeout(),
-            check=True,
-            capture_output=True,
             env=prisma_env,
         )
 
@@ -403,7 +431,7 @@ class ProxyExtrasDBManager:
         try:
             logger.info("Generating migration diff between DB and schema.prisma...")
             with open(diff_sql_path, "w") as f:
-                subprocess.run(
+                prisma_toolchain.run_prisma(
                     [
                         _get_prisma_command(),
                         "migrate",
@@ -414,7 +442,6 @@ class ProxyExtrasDBManager:
                         schema_path,
                         "--script",
                     ],
-                    check=True,
                     timeout=prisma_command_timeout(),
                     stdout=f,
                     env=_get_prisma_env(),
@@ -437,7 +464,7 @@ class ProxyExtrasDBManager:
             migration_files = sorted(Path(migrations_dir).glob("*/migration.sql"))
             for mig_file in migration_files:
                 try:
-                    subprocess.run(
+                    prisma_toolchain.run_prisma(
                         [
                             _get_prisma_command(),
                             "db",
@@ -448,9 +475,6 @@ class ProxyExtrasDBManager:
                             schema_path,
                         ],
                         timeout=prisma_command_timeout(),
-                        check=True,
-                        capture_output=True,
-                        text=True,
                         env=_get_prisma_env(),
                     )
                     logger.info(f"Applied migration: {mig_file.parent.name}")
@@ -483,7 +507,7 @@ class ProxyExtrasDBManager:
         applied_ok = False
         try:
             logger.info("Running prisma db execute to apply the migration diff...")
-            result = subprocess.run(
+            result = prisma_toolchain.run_prisma(
                 [
                     _get_prisma_command(),
                     "db",
@@ -494,9 +518,6 @@ class ProxyExtrasDBManager:
                     schema_path,
                 ],
                 timeout=prisma_command_timeout(),
-                check=True,
-                capture_output=True,
-                text=True,
                 env=_get_prisma_env(),
             )
             logger.info(f"prisma db execute stdout: {result.stdout}")
@@ -525,7 +546,7 @@ class ProxyExtrasDBManager:
         for migration_name in migration_names:
             try:
                 logger.info(f"Resolving migration: {migration_name}")
-                subprocess.run(
+                prisma_toolchain.run_prisma(
                     [
                         _get_prisma_command(),
                         "migrate",
@@ -534,9 +555,6 @@ class ProxyExtrasDBManager:
                         migration_name,
                     ],
                     timeout=prisma_command_timeout(),
-                    check=True,
-                    capture_output=True,
-                    text=True,
                     env=_get_prisma_env(),
                 )
                 logger.debug(f"Resolved migration: {migration_name}")
@@ -716,6 +734,9 @@ class ProxyExtrasDBManager:
         Ahead-of-HEAD state (DB has migrations newer than this build ships)
         is logged as a warning, not a fatal error — users whose DBs got into
         weird shapes from the old thrashing should still be able to start.
+
+        The retry budget only counts attempts that made no progress: see
+        _MigrateAttemptBudget.
         """
         schema_path = ProxyExtrasDBManager._get_prisma_dir() + "/schema.prisma"
         migrations_dir = ProxyExtrasDBManager._get_prisma_dir()
@@ -726,11 +747,12 @@ class ProxyExtrasDBManager:
             original_dir = os.getcwd()
             os.chdir(migrations_dir)
             try:
-                subprocess.run(
+                prisma_toolchain.run_prisma(
                     [_get_prisma_command(), "db", "push", "--accept-data-loss"],
                     timeout=prisma_command_timeout(),
-                    check=True,
                     env=_get_prisma_env(),
+                    stdout=None,
+                    stderr=None,
                 )
                 return True
             except (
@@ -749,15 +771,13 @@ class ProxyExtrasDBManager:
         original_dir = os.getcwd()
         os.chdir(migrations_dir)
         deploy_timeout = prisma_migrate_deploy_timeout()
+        budget = _MigrateAttemptBudget(attempts_left=MAX_MIGRATE_DEPLOY_ATTEMPTS)
         try:
-            for attempt in range(4):
+            while not budget.exhausted:
                 try:
-                    result = subprocess.run(
+                    result = prisma_toolchain.run_prisma(
                         [_get_prisma_command(), "migrate", "deploy"],
                         timeout=deploy_timeout,
-                        check=True,
-                        capture_output=True,
-                        text=True,
                         env=_get_prisma_env(),
                     )
                     logger.info(f"prisma migrate deploy stdout: {result.stdout}")
@@ -767,168 +787,155 @@ class ProxyExtrasDBManager:
                     logger.warning(
                         "prisma migrate deploy attempt %s timed out after %ss, retrying. "
                         "Raise %s if this database needs longer to apply its pending migrations.",
-                        attempt + 1,
+                        budget.attempt_number,
                         deploy_timeout,
                         PRISMA_MIGRATE_DEPLOY_TIMEOUT_ENV_VAR,
                     )
-                    time.sleep(random.randrange(5, 15))
-                    continue
+                    next_budget = budget.spend()
 
                 except subprocess.CalledProcessError as e:
-                    stderr = e.stderr or ""
+                    next_budget = ProxyExtrasDBManager._budget_after_deploy_failure(
+                        e, budget, schema_path
+                    )
 
-                    if "P3005" in stderr and "database schema is not empty" in stderr:
-                        logger.info(
-                            "Schema exists but no migrations ledger — creating baseline"
-                        )
-                        ProxyExtrasDBManager._create_baseline_migration(schema_path)
-                        continue
-
-                    if "P3009" in stderr:
-                        migration_match = re.search(r"`(\d+_\S+?)`", stderr)
-                        if (
-                            migration_match
-                            and ProxyExtrasDBManager._is_idempotent_error(stderr)
-                        ):
-                            name = migration_match.group(1)
-                            logger.info(
-                                f"Migration {name} failed idempotently — marking applied and retrying"
-                            )
-                            try:
-                                ProxyExtrasDBManager._roll_back_migration(name)
-                            except (
-                                subprocess.CalledProcessError,
-                                subprocess.TimeoutExpired,
-                            ):
-                                pass  # may already be rolled-back
-                            try:
-                                ProxyExtrasDBManager._resolve_specific_migration(name)
-                            except (
-                                subprocess.CalledProcessError,
-                                subprocess.TimeoutExpired,
-                            ) as resolve_err:
-                                # We're already inside the outer
-                                # `except CalledProcessError` handler —
-                                # re-raising CalledProcessError from here
-                                # would escape as itself, bypassing
-                                # proxy_cli.py's `except RuntimeError`.
-                                raise RuntimeError(
-                                    f"Failed to mark migration {name} as applied "
-                                    f"after idempotent recovery. Manual "
-                                    f"intervention may be required.\n\n"
-                                    f"Detail: {resolve_err}"
-                                ) from resolve_err
-                            continue
-                        if migration_match:
-                            migration_name = migration_match.group(1)
-                            ledger_logs = ProxyExtrasDBManager._failed_migration_logs(migration_name)
-                            if ledger_logs is not None and (
-                                ledger_logs == "" or _MIGRATION_DEADLOCK_MARKER in ledger_logs
-                            ):
-                                logger.info(
-                                    "Migration %s failed in a concurrent migrate deploy "
-                                    "deadlock race, rolling its ledger row back and retrying",
-                                    migration_name,
-                                )
-                                ProxyExtrasDBManager._roll_back_migration_best_effort(migration_name)
-                                time.sleep(random.randrange(5, 15))
-                                continue
-                        raise RuntimeError(
-                            "Database migration failed and cannot be auto-recovered. "
-                            f"Manual intervention required.\n\nPrisma error:\n{stderr}"
-                        ) from e
-
-                    if "P3018" in stderr:
-                        if ProxyExtrasDBManager._is_permission_error(stderr):
-                            raise RuntimeError(
-                                "Database migration failed due to insufficient "
-                                "permissions. Please grant the required privileges "
-                                f"and retry.\n\nPrisma error:\n{stderr}"
-                            ) from e
-
-                        migration_match = re.search(
-                            r"Migration name: (\d+_\S+)", stderr
-                        )
-                        if (
-                            migration_match
-                            and ProxyExtrasDBManager._is_idempotent_error(stderr)
-                        ):
-                            name = migration_match.group(1)
-                            logger.info(
-                                f"Migration {name} SQL hit idempotent error — marking applied and retrying"
-                            )
-                            try:
-                                ProxyExtrasDBManager._roll_back_migration(name)
-                            except (
-                                subprocess.CalledProcessError,
-                                subprocess.TimeoutExpired,
-                            ):
-                                pass  # may already be rolled-back
-                            try:
-                                ProxyExtrasDBManager._resolve_specific_migration(name)
-                            except (
-                                subprocess.CalledProcessError,
-                                subprocess.TimeoutExpired,
-                            ) as resolve_err:
-                                raise RuntimeError(
-                                    f"Failed to mark migration {name} as applied "
-                                    f"after idempotent recovery. Manual "
-                                    f"intervention may be required.\n\n"
-                                    f"Detail: {resolve_err}"
-                                ) from resolve_err
-                            continue
-
-                        if migration_match and _MIGRATION_DEADLOCK_MARKER in stderr:
-                            logger.info(
-                                "Migration %s deadlocked against a concurrent "
-                                "migrate deploy, rolling its ledger row back "
-                                "and retrying",
-                                migration_match.group(1),
-                            )
-                            ProxyExtrasDBManager._roll_back_migration_best_effort(
-                                migration_match.group(1)
-                            )
-                            time.sleep(random.randrange(5, 15))
-                            continue
-
-                        raise RuntimeError(
-                            "Database migration failed and cannot be auto-recovered. "
-                            f"Manual intervention required.\n\nPrisma error:\n{stderr}"
-                        ) from e
-
-                    if _MIGRATION_DEADLOCK_MARKER in stderr:
-                        logger.info(
-                            "prisma migrate deploy attempt %s deadlocked against "
-                            "a concurrent migrate deploy, retrying",
-                            attempt + 1,
-                        )
-                        time.sleep(random.randrange(5, 15))
-                        continue
-
-                    if "P1002" in stderr and "advisory lock" in stderr:
-                        logger.info(
-                            "prisma migrate deploy attempt %s timed out waiting for "
-                            "the advisory lock a concurrent migrate deploy holds, retrying",
-                            attempt + 1,
-                        )
-                        time.sleep(random.randrange(5, 15))
-                        continue
-
-                    raise RuntimeError(
-                        "Database migration failed and cannot be auto-recovered. "
-                        f"Manual intervention required.\n\nPrisma error:\n{stderr}"
-                    ) from e
+                if next_budget.attempts_left < budget.attempts_left:
+                    time.sleep(random.randrange(5, 15))
+                budget = next_budget  # rebind-ok: the loop carries the budget from one migrate deploy pass to the next
 
             raise RuntimeError(
-                "Database migration failed after 4 attempts (retry loop "
-                "exhausted by timeouts, deadlock retries, or repeated "
-                "idempotent-recovery continues). Check database connectivity, "
+                f"Database migration failed after {MAX_MIGRATE_DEPLOY_ATTEMPTS} "
+                "attempts that made no progress (timeouts, deadlock retries, or a "
+                "recovery that had already run once). Check database connectivity, "
                 "load, and _prisma_migrations ledger state, and raise "
                 f"{PRISMA_MIGRATE_DEPLOY_TIMEOUT_ENV_VAR} if the attempts timed out."
             )
         finally:
             os.chdir(original_dir)
 
+    @staticmethod
+    def _budget_after_deploy_failure(
+        error: subprocess.CalledProcessError,
+        budget: "_MigrateAttemptBudget",
+        schema_path: str,
+    ) -> "_MigrateAttemptBudget":
+        """Recover from one failed `prisma migrate deploy`, and price the pass.
+
+        Returns the budget the next pass runs under, or raises when the failure
+        is not one this resolver knows how to recover from.
+        """
+        stderr = error.stderr or ""
+
+        if "P3005" in stderr and "database schema is not empty" in stderr:
+            logger.info("Schema exists but no migrations ledger — creating baseline")
+            if ProxyExtrasDBManager._create_baseline_migration(schema_path):
+                return budget.after_recovery("baseline")
+            return budget.spend()
+
+        if "P3009" in stderr:
+            migration_match = re.search(r"`(\d+_\S+?)`", stderr)
+            if migration_match and ProxyExtrasDBManager._is_idempotent_error(stderr):
+                name = migration_match.group(1)
+                logger.info(
+                    f"Migration {name} failed idempotently — marking applied and retrying"
+                )
+                ProxyExtrasDBManager._mark_migration_applied(name)
+                return budget.after_recovery(f"resolved:{name}")
+            if migration_match:
+                migration_name = migration_match.group(1)
+                ledger_logs = ProxyExtrasDBManager._failed_migration_logs(migration_name)
+                if ledger_logs is not None and (
+                    ledger_logs == "" or _MIGRATION_DEADLOCK_MARKER in ledger_logs
+                ):
+                    logger.info(
+                        "Migration %s failed in a concurrent migrate deploy "
+                        "deadlock race, rolling its ledger row back and retrying",
+                        migration_name,
+                    )
+                    ProxyExtrasDBManager._roll_back_migration_best_effort(migration_name)
+                    return budget.spend()
+            raise RuntimeError(
+                "Database migration failed and cannot be auto-recovered. "
+                f"Manual intervention required.\n\nPrisma error:\n{stderr}"
+            ) from error
+
+        if "P3018" in stderr:
+            if ProxyExtrasDBManager._is_permission_error(stderr):
+                raise RuntimeError(
+                    "Database migration failed due to insufficient "
+                    "permissions. Please grant the required privileges "
+                    f"and retry.\n\nPrisma error:\n{stderr}"
+                ) from error
+
+            migration_match = re.search(r"Migration name: (\d+_\S+)", stderr)
+            if migration_match and ProxyExtrasDBManager._is_idempotent_error(stderr):
+                name = migration_match.group(1)
+                logger.info(
+                    f"Migration {name} SQL hit idempotent error — marking applied and retrying"
+                )
+                ProxyExtrasDBManager._mark_migration_applied(name)
+                return budget.after_recovery(f"resolved:{name}")
+
+            if migration_match and _MIGRATION_DEADLOCK_MARKER in stderr:
+                logger.info(
+                    "Migration %s deadlocked against a concurrent "
+                    "migrate deploy, rolling its ledger row back "
+                    "and retrying",
+                    migration_match.group(1),
+                )
+                ProxyExtrasDBManager._roll_back_migration_best_effort(
+                    migration_match.group(1)
+                )
+                return budget.spend()
+
+            raise RuntimeError(
+                "Database migration failed and cannot be auto-recovered. "
+                f"Manual intervention required.\n\nPrisma error:\n{stderr}"
+            ) from error
+
+        if _MIGRATION_DEADLOCK_MARKER in stderr:
+            logger.info(
+                "prisma migrate deploy attempt %s deadlocked against "
+                "a concurrent migrate deploy, retrying",
+                budget.attempt_number,
+            )
+            return budget.spend()
+
+        if "P1002" in stderr and "advisory lock" in stderr:
+            logger.info(
+                "prisma migrate deploy attempt %s timed out waiting for "
+                "the advisory lock a concurrent migrate deploy holds, retrying",
+                budget.attempt_number,
+            )
+            return budget.spend()
+
+        raise RuntimeError(
+            "Database migration failed and cannot be auto-recovered. "
+            f"Manual intervention required.\n\nPrisma error:\n{stderr}"
+        ) from error
+
+    @staticmethod
+    def _mark_migration_applied(name: str) -> None:
+        """Roll a failed ledger row back if it is still there, then mark it applied."""
+        try:
+            ProxyExtrasDBManager._roll_back_migration(name)
+        except (subprocess.CalledProcessError, subprocess.TimeoutExpired):
+            pass  # may already be rolled-back
+        try:
+            ProxyExtrasDBManager._resolve_specific_migration(name)
+        except (
+            subprocess.CalledProcessError,
+            subprocess.TimeoutExpired,
+        ) as resolve_err:
+            # We're called from inside an `except CalledProcessError` handler —
+            # re-raising CalledProcessError from here would escape as itself,
+            # bypassing proxy_cli.py's `except RuntimeError`.
+            raise RuntimeError(
+                f"Failed to mark migration {name} as applied "
+                f"after idempotent recovery. Manual "
+                f"intervention may be required.\n\n"
+                f"Detail: {resolve_err}"
+            ) from resolve_err
+
     @staticmethod
     def apply_replica_identity_full_if_requested() -> bool:
         """
@@ -1007,12 +1014,9 @@ class ProxyExtrasDBManager:
                     logger.info("Running prisma migrate deploy")
                     try:
                         # Set migrations directory for Prisma
-                        result = subprocess.run(
+                        result = prisma_toolchain.run_prisma(
                             [_get_prisma_command(), "migrate", "deploy"],
                             timeout=prisma_migrate_deploy_timeout(),
-                            check=True,
-                            capture_output=True,
-                            text=True,
                             env=_get_prisma_env(),
                         )
                         logger.info(f"prisma migrate deploy stdout: {result.stdout}")
@@ -1084,7 +1088,7 @@ class ProxyExtrasDBManager:
                                         f"Found failed migration: {failed_migration}, marking as rolled back"
                                     )
                                     # Mark the failed migration as rolled back
-                                    subprocess.run(
+                                    prisma_toolchain.run_prisma(
                                         [
                                             _get_prisma_command(),
                                             "migrate",
@@ -1093,9 +1097,6 @@ class ProxyExtrasDBManager:
                                             failed_migration,
                                         ],
                                         timeout=prisma_command_timeout(),
-                                        check=True,
-                                        capture_output=True,
-                                        text=True,
                                         env=_get_prisma_env(),
                                     )
                                     logger.info(
@@ -1220,10 +1221,12 @@ class ProxyExtrasDBManager:
                     if ProxyExtrasDBManager.spend_logs_is_partitioned():
                         raise RuntimeError(PARTITIONED_SPEND_LOGS_PUSH_ERROR)
                     # Use prisma db push with increased timeout
-                    subprocess.run(
+                    prisma_toolchain.run_prisma(
                         [_get_prisma_command(), "db", "push", "--accept-data-loss"],
                         timeout=prisma_command_timeout(),
-                        check=True,
+                        stdout=None,
+                        stderr=None,
+                        env=_get_prisma_env(),
                     )
                     return True
             except subprocess.TimeoutExpired:

litellm-proxy-extras/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "litellm-proxy-extras"
-version = "0.4.92"
+version = "0.4.93"
 description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package."
 readme = "README.md"
 requires-python = ">=3.9"
@@ -26,7 +26,7 @@ required-version = ">=0.10.9"
 module-root = ""
 
 [tool.commitizen]
-version = "0.4.92"
+version = "0.4.93"
 version_files = [
     "pyproject.toml:^version",
     "../pyproject.toml:litellm-proxy-extras==",

litellm-proxy-extras/tests/test_setup_database_fail_fast.py

@@ -42,7 +42,7 @@ def test_v2_p3018_permission_error_raises_runtime_error(monkeypatch, tmp_path):
         "Error: P3018\nMigration name: 20250326162113_baseline\n"
         "Database error code: 42501\npermission denied for schema public"
     )
-    with patch("subprocess.run", side_effect=_fake_migrate_deploy_failure(1, stderr)):
+    with patch("litellm_proxy_extras.prisma_toolchain.run_prisma", side_effect=_fake_migrate_deploy_failure(1, stderr)):
         with pytest.raises(RuntimeError, match="permission"):
             ProxyExtrasDBManager.setup_database(use_migrate=True, use_v2_resolver=True)
 
@@ -60,7 +60,7 @@ def test_v2_non_idempotent_p3009_raises_runtime_error(monkeypatch, tmp_path):
         "Error: P3009\nMigration `20260101000000_genuinely_broken` failed\n"
         'Reason: syntax error at or near "BRKN" LINE 42'
     )
-    with patch("subprocess.run", side_effect=_fake_migrate_deploy_failure(1, stderr)):
+    with patch("litellm_proxy_extras.prisma_toolchain.run_prisma", side_effect=_fake_migrate_deploy_failure(1, stderr)):
         with pytest.raises(RuntimeError, match="cannot be auto-recovered"):
             ProxyExtrasDBManager.setup_database(use_migrate=True, use_v2_resolver=True)
 
@@ -124,7 +124,7 @@ def test_v1_default_still_calls_resolve_all_migrations(monkeypatch, tmp_path):
     def fake_resolve(*args, **kwargs):
         resolve_called["n"] += 1
 
-    monkeypatch.setattr("subprocess.run", fake_run)
+    monkeypatch.setattr("litellm_proxy_extras.prisma_toolchain.run_prisma", fake_run)
     monkeypatch.setattr(ProxyExtrasDBManager, "_resolve_all_migrations", fake_resolve)
 
     ok = ProxyExtrasDBManager.setup_database(use_migrate=True)  # v2 flag NOT set
@@ -139,7 +139,7 @@ def test_v2_db_push_wraps_subprocess_error_as_runtime_error(monkeypatch, tmp_pat
     (tmp_path / "schema.prisma").write_text("// stub")
 
     stderr = "db push error"
-    with patch("subprocess.run", side_effect=_fake_migrate_deploy_failure(1, stderr)):
+    with patch("litellm_proxy_extras.prisma_toolchain.run_prisma", side_effect=_fake_migrate_deploy_failure(1, stderr)):
         with pytest.raises(RuntimeError, match="prisma db push failed"):
             ProxyExtrasDBManager.setup_database(use_migrate=False, use_v2_resolver=True)
 
@@ -209,7 +209,7 @@ def test_v2_resolve_specific_migration_failure_raises_runtime_error(
         "Error: P3009\nMigration `20260101000000_some_migration` failed\n"
         "relation already exists"
     )
-    with patch("subprocess.run", side_effect=_fake_migrate_deploy_failure(1, stderr)):
+    with patch("litellm_proxy_extras.prisma_toolchain.run_prisma", side_effect=_fake_migrate_deploy_failure(1, stderr)):
         with pytest.raises(
             RuntimeError, match="Failed to mark migration .* as applied"
         ):
@@ -228,7 +228,7 @@ def test_v2_does_not_call_resolve_all_migrations(monkeypatch, tmp_path):
         stdout = "Applied migration.\n"
         stderr = ""
 
-    monkeypatch.setattr("subprocess.run", lambda *a, **kw: FakeResult())
+    monkeypatch.setattr("litellm_proxy_extras.prisma_toolchain.run_prisma", lambda *a, **kw: FakeResult())
 
     resolve_called = {"n": 0}
     monkeypatch.setattr(
@@ -296,7 +296,7 @@ def test_v2_p3018_deadlock_rolls_back_and_retries(monkeypatch, tmp_path):
         "_resolve_specific_migration",
         lambda name: pytest.fail("a deadlocked migration must never be marked applied"),
     )
-    monkeypatch.setattr("subprocess.run", _succeed_after(1, _DEADLOCK_P3018_STDERR))
+    monkeypatch.setattr("litellm_proxy_extras.prisma_toolchain.run_prisma", _succeed_after(1, _DEADLOCK_P3018_STDERR))
 
     ok = ProxyExtrasDBManager.setup_database(use_migrate=True, use_v2_resolver=True)
     assert ok is True
@@ -309,7 +309,7 @@ def test_v2_p3018_persistent_deadlock_exhausts_attempts(monkeypatch, tmp_path):
     monkeypatch.setattr(ProxyExtrasDBManager, "_roll_back_migration", lambda name: None)
 
     with patch(
-        "subprocess.run",
+        "litellm_proxy_extras.prisma_toolchain.run_prisma",
         side_effect=_fake_migrate_deploy_failure(1, _DEADLOCK_P3018_STDERR),
     ):
         with pytest.raises(RuntimeError, match="after 4 attempts"):
@@ -343,7 +343,7 @@ def test_v2_p3009_deadlocked_ledger_row_rolls_back_and_retries(monkeypatch, tmp_
         "_resolve_specific_migration",
         lambda name: pytest.fail("a deadlocked migration must never be marked applied"),
     )
-    monkeypatch.setattr("subprocess.run", _succeed_after(1, stderr))
+    monkeypatch.setattr("litellm_proxy_extras.prisma_toolchain.run_prisma", _succeed_after(1, stderr))
 
     ok = ProxyExtrasDBManager.setup_database(use_migrate=True, use_v2_resolver=True)
     assert ok is True
@@ -372,7 +372,7 @@ def test_v2_p3009_empty_ledger_logs_rolls_back_and_retries(monkeypatch, tmp_path
         "_resolve_specific_migration",
         lambda name: pytest.fail("a deadlocked migration must never be marked applied"),
     )
-    monkeypatch.setattr("subprocess.run", _succeed_after(1, stderr))
+    monkeypatch.setattr("litellm_proxy_extras.prisma_toolchain.run_prisma", _succeed_after(1, stderr))
 
     ok = ProxyExtrasDBManager.setup_database(use_migrate=True, use_v2_resolver=True)
     assert ok is True
@@ -395,7 +395,7 @@ def test_v2_p3009_unreadable_ledger_still_raises(monkeypatch, tmp_path):
         "_roll_back_migration",
         lambda name: pytest.fail("an unreadable ledger must not trigger a retry"),
     )
-    monkeypatch.setattr("subprocess.run", _succeed_after(1, stderr))
+    monkeypatch.setattr("litellm_proxy_extras.prisma_toolchain.run_prisma", _succeed_after(1, stderr))
 
     with pytest.raises(RuntimeError, match="cannot be auto-recovered"):
         ProxyExtrasDBManager.setup_database(use_migrate=True, use_v2_resolver=True)
@@ -417,7 +417,7 @@ def test_v2_p3009_non_deadlock_ledger_row_still_raises(monkeypatch, tmp_path):
         lambda name: 'ERROR: syntax error at or near "BRKN"',
     )
 
-    with patch("subprocess.run", side_effect=_fake_migrate_deploy_failure(1, stderr)):
+    with patch("litellm_proxy_extras.prisma_toolchain.run_prisma", side_effect=_fake_migrate_deploy_failure(1, stderr)):
         with pytest.raises(RuntimeError, match="cannot be auto-recovered"):
             ProxyExtrasDBManager.setup_database(use_migrate=True, use_v2_resolver=True)
 
@@ -427,7 +427,7 @@ def test_v2_bare_deadlock_stderr_retries(monkeypatch, tmp_path):
     waiter as victim) is retried, not fatal."""
     _stub_v2_env(monkeypatch, tmp_path)
     monkeypatch.setattr(
-        "subprocess.run", _succeed_after(1, "Database error: deadlock detected")
+        "litellm_proxy_extras.prisma_toolchain.run_prisma", _succeed_after(1, "Database error: deadlock detected")
     )
 
     ok = ProxyExtrasDBManager.setup_database(use_migrate=True, use_v2_resolver=True)
@@ -446,7 +446,10 @@ def test_v2_advisory_lock_timeout_retries(monkeypatch, tmp_path):
     """v2: the advisory-lock waiter that times out while a peer's retry holds
     the lock retries instead of dying."""
     _stub_v2_env(monkeypatch, tmp_path)
-    monkeypatch.setattr("subprocess.run", _succeed_after(2, _P1002_ADVISORY_LOCK_STDERR))
+    monkeypatch.setattr(
+        "litellm_proxy_extras.prisma_toolchain.run_prisma",
+        _succeed_after(2, _P1002_ADVISORY_LOCK_STDERR),
+    )
 
     ok = ProxyExtrasDBManager.setup_database(use_migrate=True, use_v2_resolver=True)
     assert ok is True
@@ -456,7 +459,7 @@ def test_v2_p1002_without_advisory_lock_context_still_raises(monkeypatch, tmp_pa
     """v2: a plain P1002 (database unreachable) stays fatal."""
     _stub_v2_env(monkeypatch, tmp_path)
     stderr = "Error: P1002\n\nThe database server at `db`:`5432` was reached but timed out."
-    monkeypatch.setattr("subprocess.run", _succeed_after(1, stderr))
+    monkeypatch.setattr("litellm_proxy_extras.prisma_toolchain.run_prisma", _succeed_after(1, stderr))
 
     with pytest.raises(RuntimeError, match="cannot be auto-recovered"):
         ProxyExtrasDBManager.setup_database(use_migrate=True, use_v2_resolver=True)

litellm-rust/ADDING_A_PROVIDER.md

@@ -26,4 +26,4 @@ variants of it. The test for a good abstraction is that adding the next provider
 is a few declarative lines, not a new file of duplicated flow. Only diverge from
 the base when behavior is genuinely different, and say so explicitly in the PR.
 
-**Calling:** hosts invoke the core entrypoint — the Python bridge and the `ai-gateway` route service both call `litellm_core::messages::messages`. Never add a provider handler to `ai-gateway`. Register new modules in `lib.rs` / `mod.rs`, then run `cargo fmt && cargo clippy --workspace -- -D warnings && cargo test --workspace`.
+**Calling:** hosts invoke the core entrypoint — the Python bridge and the `ai-gateway` route service both call `litellm_core::messages::messages`. Never add a provider handler to `ai-gateway`. Register new modules in `lib.rs` / `mod.rs`, then run the commands under "Checks" in [CLAUDE.md](CLAUDE.md).

litellm-rust/CLAUDE.md

@@ -174,10 +174,14 @@ for changes under `litellm-rust/`.
 ```bash
 cd litellm-rust
 cargo fmt --check
+cargo clippy --workspace --all-targets -- -D warnings
+cargo clippy -p litellm-core --all-targets --features bedrock-auth -- -D warnings
 # the ai-gateway binary + server code is behind the `server` feature
-cargo clippy -p litellm-ai-gateway --all-targets --features server -- -D warnings
-cargo clippy -p litellm-core -p litellm-python-interop -p litellm-python-bridge --all-targets -- -D warnings
+cargo clippy -p litellm-ai-gateway --all-targets --all-features -- -D warnings
 cargo test --workspace
+cargo test -p litellm-core --features bedrock-auth
+# the `auth`, `routes`, `state` and `realtime` tests only exist under `server`
+cargo test -p litellm-ai-gateway --features server
 ```
 
 When a Rust path is exposed through Python, add Python parity tests that compare

litellm-rust/Cargo.lock

@@ -1435,14 +1435,17 @@ dependencies = [
  "aws-sigv4",
  "aws-smithy-runtime-api",
  "aws-types",
+ "base64",
  "rand 0.8.7",
  "reqwest",
+ "rstest",
  "serde",
  "serde_json",
  "sha2 0.10.9",
  "thiserror 2.0.19",
  "tokio",
  "tracing",
+ "tracing-subscriber",
 ]
 
 [[package]]
@@ -1461,7 +1464,6 @@ dependencies = [
  "tokio",
  "tokio-tungstenite",
  "tracing",
- "tracing-subscriber",
 ]
 
 [[package]]
@@ -1511,6 +1513,16 @@ version = "0.3.17"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a"
 
+[[package]]
+name = "mime_guess"
+version = "2.0.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f7c44f8e672c00fe5308fa235f821cb4198414e1c77935c1ab6948d3fd78550e"
+dependencies = [
+ "mime",
+ "unicase",
+]
+
 [[package]]
 name = "mio"
 version = "1.2.2"
@@ -1969,6 +1981,7 @@ dependencies = [
  "hyper-util",
  "js-sys",
  "log",
+ "mime_guess",
  "percent-encoding",
  "pin-project-lite",
  "quinn",
@@ -2736,6 +2749,12 @@ version = "1.20.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20"
 
+[[package]]
+name = "unicase"
+version = "2.9.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "dbc4bc3a9f746d862c45cb89d705aa10f187bb96c76001afab07a0d35ce60142"
+
 [[package]]
 name = "unicode-ident"
 version = "1.0.24"

litellm-rust/Cargo.toml

@@ -24,10 +24,10 @@ pyo3 = "0.29.2"
 pyo3-async-runtimes = { version = "0.29.0", features = ["tokio-runtime"] }
 pythonize = "0.29.0"
 rand = "0.8"
-reqwest = { version = "0.12", default-features = false, features = ["blocking", "json", "rustls-tls", "http2", "stream"] }
+reqwest = { version = "0.12", default-features = false, features = ["blocking", "json", "multipart", "rustls-tls", "http2", "stream"] }
 rstest = "0.26.1"
 serde = { version = "1.0", features = ["derive"] }
-serde_json = "1.0"
+serde_json = { version = "1.0", features = ["float_roundtrip"] }
 sha2 = "0.10"
 subtle = "2"
 thiserror = "2.0"

litellm-rust/README.md

@@ -49,11 +49,6 @@ function per top-level route, mirroring the core entrypoints.
 
 ## Checks
 
-Run these before pushing Rust changes. GitHub Actions runs the same checks for
-changes under `litellm-rust/`.
-
-```bash
-cargo fmt --check
-cargo clippy --workspace --all-targets -- -D warnings
-cargo test --workspace
-```
+Run the commands under "Checks" in [CLAUDE.md](CLAUDE.md) before pushing Rust
+changes. That list is the single source of truth and matches what GitHub Actions
+runs for changes under `litellm-rust/`.

litellm-rust/crates/CODING_STANDARDS/PROVIDER_CODING_STANDARDS.md

@@ -49,11 +49,5 @@ Rules for adding or changing an LLM provider/route in `litellm-rust`. `messages`
 
 ## Checks before push
 
-25. Run, and keep green:
-    ```bash
-    cd litellm-rust
-    cargo fmt --check
-    cargo clippy -p litellm-ai-gateway --all-targets --features server -- -D warnings
-    cargo clippy -p litellm-core -p litellm-python-interop -p litellm-python-bridge --all-targets -- -D warnings
-    cargo test --workspace
-    ```
+25. Run, and keep green, the commands under "Checks" in `litellm-rust/CLAUDE.md`.
+    That list is the single source of truth and matches what GitHub Actions runs.

litellm-rust/crates/ai-gateway/Cargo.toml

@@ -32,6 +32,7 @@ subtle = { workspace = true, optional = true }
 # SHA-256 hash_token) so the plaintext credential never enters a log payload.
 sha2 = { workspace = true, optional = true }
 pyo3 = { workspace = true, features = ["auto-initialize"], optional = true }
+tower = { version = "0.5.3", features = ["util"], optional = true }
 
 [features]
 default = []
@@ -39,6 +40,7 @@ server = ["dep:axum", "dep:subtle", "dep:sha2"]
 # Build the gateway's config from the proxy YAML via an embedded Python
 # interpreter (links libpython; requires `litellm` importable at runtime).
 python-config = ["dep:pyo3"]
+trace-parity = ["server", "dep:tower", "litellm-core/observability"]
 
 [dev-dependencies]
 futures-channel = "0.3"

litellm-rust/crates/ai-gateway/Dockerfile

@@ -36,12 +36,12 @@ FROM chef AS builder
 # whenever only gateway source changes.
 COPY --from=planner /build/litellm-rust/recipe.json recipe.json
 RUN cargo chef cook --locked --release \
-        -p litellm-ai-gateway --features python-config \
+        -p litellm-ai-gateway --features server,python-config \
         --recipe-path recipe.json
 # Now copy the real sources and build the gateway binary. Deps are already cooked
 # above, so this step only recompiles the gateway crate.
 COPY litellm-rust/ .
-RUN cargo build --locked --release -p litellm-ai-gateway --features python-config
+RUN cargo build --locked --release -p litellm-ai-gateway --bin litellm-ai-gateway --features server,python-config
 
 # ---- Runtime ----------------------------------------------------------------
 # python:3.11-slim-bookworm ships libpython3.11, matching the builder's PyO3

litellm-rust/crates/ai-gateway/README.md

@@ -100,7 +100,7 @@ Worker tuning, rarely needed: `LITELLM_LOG_CHANNEL_CAPACITY` (4096),
 
 ## Build & run with Docker
 
-The image is built `--features python-config` and installs litellm **from this
+The image is built `--features server,python-config` and installs litellm **from this
 repo's source** (the config reader is newer than any PyPI release), so the build
 **context is the repo root**:
 
@@ -135,10 +135,10 @@ docker run --rm -p 4001:4001 \
 ```bash
 # config.yaml mode — needs litellm importable in the active python env
 LITELLM_CONFIG_PATH=./crates/ai-gateway/config.yaml \
-  cargo run --release -p litellm-ai-gateway --features python-config
+  cargo run --release -p litellm-ai-gateway --features server,python-config
 
 # env stand-in mode — no python, no config
-cargo run --release -p litellm-ai-gateway
+cargo run --release -p litellm-ai-gateway --features server
 ```
 
 ## Deploy on Render

litellm-rust/crates/ai-gateway/src/lib.rs

@@ -28,6 +28,8 @@ pub mod auth;
 pub mod routes;
 #[cfg(feature = "server")]
 pub mod state;
+#[cfg(feature = "trace-parity")]
+pub mod trace_parity;
 
 mod constants;
 pub mod integrations;

litellm-rust/crates/ai-gateway/src/ocr/common_utils.rs

@@ -12,6 +12,7 @@ use litellm_core::providers::azure_ai::ocr::transformation::{
     AZURE_AI_OCR_CONFIG, AZURE_DOCUMENT_INTELLIGENCE_OCR_CONFIG,
 };
 use litellm_core::providers::mistral::ocr::transformation::MISTRAL_OCR_CONFIG;
+use litellm_core::providers::reducto::ocr::transformation as reducto;
 use litellm_core::providers::vertex_ai::ocr::transformation as vertex_ai;
 use litellm_core::providers::vertex_ai::ocr::transformation::{
     VERTEX_AI_DEEPSEEK_OCR_CONFIG, VERTEX_AI_OCR_CONFIG,
@@ -39,6 +40,7 @@ pub(super) fn ocr_provider_config(
 ) -> Option<&'static dyn OcrProviderConfig> {
     match provider {
         "mistral" => Some(&MISTRAL_OCR_CONFIG),
+        "reducto" => reducto::config_for_model(model),
         "azure_ai" if is_azure_document_intelligence_model(model) => {
             Some(&AZURE_DOCUMENT_INTELLIGENCE_OCR_CONFIG)
         }
@@ -334,6 +336,7 @@ fn operation_status(response_json: &Value) -> Result<&str, Error> {
     }
 }
 
+#[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)]
 pub(super) async fn poll_document_intelligence(
     operation_url: &str,
     original_url: &str,
@@ -392,9 +395,11 @@ pub(super) async fn poll_document_intelligence(
 
 #[cfg(test)]
 mod tests {
-    use super::*;
+    use litellm_core::ocr::transformation::OcrResponseHandling;
     use serde_json::json;
 
+    use super::*;
+
     #[test]
     fn blocks_private_and_metadata_ips() {
         assert!(is_blocked_ip("127.0.0.1".parse().unwrap()));
@@ -438,4 +443,87 @@ mod tests {
 
         assert_eq!(transformed, document);
     }
+
+    #[test]
+    fn truncate_error_body_passes_short_strings_through() {
+        let body = "Unauthorized";
+        assert_eq!(truncate_error_body(body), "Unauthorized");
+    }
+
+    #[test]
+    fn truncate_error_body_caps_long_payloads() {
+        let body = "x".repeat(306);
+        let truncated = truncate_error_body(&body);
+
+        assert!(truncated.ends_with("... (truncated)"));
+        let prefix_chars = truncated
+            .strip_suffix("... (truncated)")
+            .expect("truncated marker present")
+            .chars()
+            .count();
+        assert_eq!(prefix_chars, 256);
+    }
+
+    #[test]
+    fn truncate_error_body_does_not_split_multibyte_chars() {
+        let body = "é".repeat(266);
+        let truncated = truncate_error_body(&body);
+        assert!(truncated.is_char_boundary(truncated.len()));
+    }
+
+    #[test]
+    fn ocr_dispatch_supports_migrated_providers() {
+        assert!(ocr_provider_config("mistral", "mistral-ocr-latest").is_some());
+        assert!(
+            ocr_provider_config("azure_ai", "pixtral-12b-2409")
+                .expect("azure ai config resolves")
+                .requires_data_uri_document()
+        );
+        assert_eq!(
+            ocr_provider_config("azure_ai", "doc-intelligence/prebuilt-read")
+                .expect("document intelligence config resolves")
+                .response_handling(),
+            OcrResponseHandling::AzureDocumentIntelligencePoll
+        );
+        assert!(
+            ocr_provider_config("vertex_ai", "deepseek-ocr-maas")
+                .expect("vertex deepseek config resolves")
+                .supported_ocr_params()
+                .contains(&"temperature")
+        );
+        assert!(ocr_provider_config("openai", "gpt-4o").is_none());
+    }
+
+    #[test]
+    fn string_headers_accepts_string_values() {
+        let headers = json!({
+            "x-trace-id": "trace-1"
+        })
+        .as_object()
+        .unwrap()
+        .clone();
+
+        assert_eq!(
+            string_headers(Some(headers)).expect("string headers accepted"),
+            vec![("x-trace-id".to_string(), "trace-1".to_string())]
+        );
+    }
+
+    #[test]
+    fn string_headers_rejects_non_string_values() {
+        let headers = json!({
+            "x-retry-count": 3
+        })
+        .as_object()
+        .unwrap()
+        .clone();
+
+        let err = string_headers(Some(headers)).expect_err("non-string header rejected");
+        assert_eq!(
+            err,
+            Error::InvalidRequest(
+                "OCR extra_headers.x-retry-count must be a string, got number".to_string()
+            )
+        );
+    }
 }

litellm-rust/crates/ai-gateway/src/ocr/handler.rs

@@ -50,7 +50,11 @@ pub(crate) async fn execute_ocr_provider_call(
         .await?;
         return Ok(request
             .config
-            .transform_ocr_response(&request.model, response_json)?
+            .transform_ocr_response_with_params(
+                &request.model,
+                response_json,
+                &request.optional_params,
+            )?
             .into_json());
     }
 
@@ -71,6 +75,10 @@ pub(crate) async fn execute_ocr_provider_call(
 
     Ok(request
         .config
-        .transform_ocr_response(&request.model, response_json)?
+        .transform_ocr_response_with_params(
+            &request.model,
+            response_json,
+            &request.optional_params,
+        )?
         .into_json())
 }

litellm-rust/crates/ai-gateway/src/ocr/hooks.rs

@@ -1,11 +1,15 @@
 use litellm_core::call_lifecycle::{CallLifecycleContext, CallLifecycleHooks, CallLifecycleTiming};
 use litellm_core::error::Error;
+use litellm_core::providers::reducto::ocr::transformation::{
+    build_upload_request, extract_document_source, extract_upload_file_id,
+};
 use serde_json::{Map, Value, json};
 use std::future::Future;
 use std::pin::Pin;
 
-use super::common_utils::{convert_document_url_to_data_uri, string_headers};
+use super::common_utils::{convert_document_url_to_data_uri, string_headers, truncate_error_body};
 use super::types::{PreparedOcrRequest, ProviderOcrRequest};
+use crate::client::http_client;
 use crate::integrations::custom_guardrail::{
     CustomGuardrailRunner, GuardrailContext, GuardrailError, GuardrailRequest,
 };
@@ -89,22 +93,39 @@ impl OcrLifecycleHooks {
         )?;
         let model = request.model.clone();
         let custom_llm_provider = request.custom_llm_provider.clone();
-        let document = if config.requires_data_uri_document() {
+        let is_reducto = custom_llm_provider == "reducto";
+        let document = if is_reducto {
+            let guarded_document = self
+                .run_during_call_guardrails(&model, &custom_llm_provider, &url, request.document)
+                .await?;
+            upload_reducto_document(
+                &guarded_document,
+                request.api_base.as_deref(),
+                request.timeout,
+                &upstream_headers,
+            )
+            .await?
+        } else if config.requires_data_uri_document() {
             convert_document_url_to_data_uri(request.document).await?
         } else {
             request.document
         };
+        let optional_params = request.optional_params;
         let body = config
-            .transform_ocr_request(&request.model, document, request.optional_params)?
+            .transform_ocr_request(&request.model, document, optional_params.clone())?
             .data;
-        let body = self
-            .run_during_call_guardrails(&model, &custom_llm_provider, &url, body)
-            .await?;
+        let body = if is_reducto {
+            body
+        } else {
+            self.run_during_call_guardrails(&model, &custom_llm_provider, &url, body)
+                .await?
+        };
         Ok(ProviderOcrRequest {
             model,
             config,
             url,
             body,
+            optional_params,
             upstream_headers,
             timeout: request.timeout,
         })
@@ -165,6 +186,63 @@ impl OcrLifecycleHooks {
     }
 }
 
+async fn upload_reducto_document(
+    document: &Value,
+    api_base: Option<&str>,
+    timeout: Option<std::time::Duration>,
+    upstream_headers: &[(String, String)],
+) -> Result<Value, Error> {
+    let source = extract_document_source(document)?;
+    let Some(authorization) = upstream_headers
+        .iter()
+        .find(|(name, _)| name.eq_ignore_ascii_case("authorization"))
+        .map(|(_, value)| value.as_str())
+    else {
+        return Err(Error::Auth(
+            "Reducto upload requires an Authorization header".to_string(),
+        ));
+    };
+    let Some(upload) = build_upload_request(source, authorization, api_base) else {
+        return Ok(document.clone());
+    };
+    let part = reqwest::multipart::Part::bytes(upload.bytes)
+        .file_name(upload.file_name)
+        .mime_str(&upload.mime_type)
+        .map_err(|error| Error::InvalidRequest(error.to_string()))?;
+    let form = reqwest::multipart::Form::new().part("file", part);
+    let mut request_builder = http_client().post(upload.url).multipart(form);
+    for (name, value) in upstream_headers {
+        if !name.eq_ignore_ascii_case("content-type")
+            && !name.eq_ignore_ascii_case("content-length")
+        {
+            request_builder = request_builder.header(name, value);
+        }
+    }
+    if let Some(timeout) = timeout {
+        request_builder = request_builder.timeout(timeout);
+    }
+    let response = request_builder
+        .send()
+        .await
+        .map_err(|error| Error::Network(error.to_string()))?;
+    let status = response.status();
+    let body = response
+        .text()
+        .await
+        .map_err(|error| Error::Network(error.to_string()))?;
+    if !status.is_success() {
+        return Err(Error::Http {
+            status: status.as_u16(),
+            body: truncate_error_body(&body),
+        });
+    }
+    let response_json: Value = serde_json::from_str(&body).map_err(|error| {
+        Error::InvalidResponse(format!("invalid Reducto upload response JSON: {error}"))
+    })?;
+    let file_id = extract_upload_file_id(&response_json)?;
+    Ok(json!({"type": "document_url", "document_url": file_id}))
+}
+
 impl CallLifecycleHooks<PreparedOcrRequest, PreparedOcrRequest, Value> for OcrLifecycleHooks {
     type PreCallFuture<'a> = OcrFuture<'a, PreparedOcrRequest>;
     type DuringCallFuture<'a> = OcrFuture<'a, PreparedOcrRequest>;

litellm-rust/crates/ai-gateway/src/ocr/mod.rs

@@ -24,4 +24,151 @@ pub async fn ocr(request: OcrRequest<'_>) -> Result<Value, Error> {
 }
 
 #[cfg(test)]
-mod tests;
+mod tests {
+    use serde_json::{Map, json};
+    use tokio::io::{AsyncReadExt, AsyncWriteExt};
+    use tokio::net::{TcpListener, TcpStream};
+
+    use super::{OcrRequest, ocr};
+    use crate::integrations::types::RequestMetadata;
+
+    async fn read_http_request(socket: &mut TcpStream) -> String {
+        let mut request = Vec::new();
+        let mut buffer = [0_u8; 1024];
+        let header_end = loop {
+            let n = socket.read(&mut buffer).await.expect("reads request");
+            if n == 0 {
+                break request.len();
+            }
+            request.extend_from_slice(&buffer[..n]);
+            if let Some(position) = request.windows(4).position(|window| window == b"\r\n\r\n") {
+                break position + 4;
+            }
+        };
+        let headers = String::from_utf8_lossy(&request[..header_end]);
+        let content_length = headers
+            .lines()
+            .find_map(|line| {
+                let (name, value) = line.split_once(':')?;
+                name.eq_ignore_ascii_case("content-length")
+                    .then(|| value.trim().parse::<usize>().ok())
+                    .flatten()
+            })
+            .unwrap_or(0);
+        while request.len().saturating_sub(header_end) < content_length {
+            let n = socket.read(&mut buffer).await.expect("reads body");
+            if n == 0 {
+                break;
+            }
+            request.extend_from_slice(&buffer[..n]);
+        }
+        String::from_utf8(request).expect("request is utf8")
+    }
+
+    fn base_ocr_request(model: &str) -> OcrRequest<'_> {
+        OcrRequest {
+            model,
+            document: json!({
+                "type": "document_url",
+                "document_url": "https://example.com/doc.pdf"
+            }),
+            api_key: Some("sk-test"),
+            api_base: None,
+            custom_llm_provider: None,
+            extra_headers: None,
+            optional_params: Map::new(),
+            timeout: None,
+            callbacks: Vec::new(),
+            guardrails: Vec::new(),
+            request_metadata: RequestMetadata::default(),
+            litellm_call_id: None,
+        }
+    }
+
+    #[tokio::test]
+    async fn reducto_file_upload_then_parse_maps_response() {
+        let listener = TcpListener::bind("127.0.0.1:0")
+            .await
+            .expect("test listener binds");
+        let address = listener.local_addr().expect("listener has local address");
+        let server = tokio::spawn(async move {
+            let (mut upload_socket, _) = listener.accept().await.expect("accepts upload request");
+            let upload_request = read_http_request(&mut upload_socket).await;
+            let upload_body = r#"{"file_id":"reducto://uploaded.pdf"}"#;
+            let upload_response = format!(
+                "HTTP/1.1 200 OK\r\ncontent-type: application/json\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{}",
+                upload_body.len(),
+                upload_body
+            );
+            upload_socket
+                .write_all(upload_response.as_bytes())
+                .await
+                .expect("writes upload response");
+
+            let (mut parse_socket, _) = listener.accept().await.expect("accepts parse request");
+            let parse_request = read_http_request(&mut parse_socket).await;
+            let parse_body = r#"{"job_id":"job_123","usage":{"num_pages":3,"credits":3},"result":{"chunks":[{"content":"Page 1 block A","blocks":[{"content":"Page 1 block A","bbox":{"page":1},"kind":"text"}]},{"content":"Page 2 block A","blocks":[{"content":"Page 2 block A","bbox":{"page":2},"kind":"table"}]},{"content":"Page 1 block B","blocks":[{"content":"Page 1 block B","bbox":{"page":1},"kind":"text"}]},{"content":"Page 3 block A","blocks":[{"content":"Page 3 block A","bbox":{"page":3},"kind":"figure"}]}]}}"#;
+            let parse_response = format!(
+                "HTTP/1.1 200 OK\r\ncontent-type: application/json\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{}",
+                parse_body.len(),
+                parse_body
+            );
+            parse_socket
+                .write_all(parse_response.as_bytes())
+                .await
+                .expect("writes parse response");
+            (upload_request, parse_request)
+        });
+        let api_base = format!("http://{address}");
+        let mut request = base_ocr_request("reducto/parse-v3");
+        request.api_base = Some(&api_base);
+        request.api_key = None;
+        request.extra_headers = Some(Map::from_iter([
+            ("Authorization".to_string(), json!("Bearer test-key")),
+            ("x-trace-id".to_string(), json!("trace-1")),
+        ]));
+        request.document = json!({
+            "type": "document_url",
+            "document_url": "data:application/pdf;base64,JVBERi0xLjQ="
+        });
+        request.optional_params = Map::from_iter([
+            (
+                "formatting".to_string(),
+                json!({"table_output_format": "html"}),
+            ),
+            ("retrieval".to_string(), json!({"chunk_mode": "section"})),
+            ("settings".to_string(), json!({"ocr_system": "standard"})),
+        ]);
+
+        let response = ocr(request).await.expect("Reducto OCR succeeds");
+
+        assert_eq!(response["pages"].as_array().map(Vec::len), Some(3));
+        assert_eq!(
+            response["pages"][0]["markdown"],
+            "Page 1 block A\n\nPage 1 block B"
+        );
+        assert_eq!(response["pages"][1]["markdown"], "Page 2 block A");
+        assert_eq!(response["pages"][2]["markdown"], "Page 3 block A");
+        assert_eq!(response["usage_info"]["pages_processed"], 3);
+        assert_eq!(response["usage_info"]["credits"], 3);
+        assert_eq!(response["provider_native_response"]["job_id"], "job_123");
+        let (upload_request, parse_request) = server.await.expect("server task completes");
+        assert!(
+            upload_request
+                .to_ascii_lowercase()
+                .contains("authorization: bearer test-key")
+        );
+        assert!(upload_request.contains("application/pdf"));
+        assert!(upload_request.contains("%PDF-1.4"));
+        assert!(upload_request.contains("x-trace-id: trace-1"));
+        assert!(
+            parse_request
+                .to_ascii_lowercase()
+                .contains("authorization: bearer test-key")
+        );
+        assert!(parse_request.contains(r#""input":"reducto://uploaded.pdf""#));
+        assert!(parse_request.contains(r#""table_output_format":"html""#));
+        assert!(parse_request.contains(r#""chunk_mode":"section""#));
+        assert!(parse_request.contains(r#""ocr_system":"standard""#));
+    }
+}

litellm-rust/crates/ai-gateway/src/ocr/prepare.rs

@@ -2,6 +2,7 @@ use std::sync::atomic::{AtomicU64, Ordering};
 use std::time::{SystemTime, UNIX_EPOCH};
 
 use litellm_core::routing_utils::provider::{CustomLlmProvider, get_custom_llm_provider};
+use serde_json::{Map, Value};
 
 use super::common_utils::ocr_provider_config;
 use super::hooks::OcrLifecycleHooks;
@@ -28,17 +29,33 @@ pub(crate) fn prepare_ocr_call(request: OcrRequest<'_>) -> PreparedOcrCall {
     let model = provider_info.model.to_string();
     let custom_llm_provider = provider_info.custom_llm_provider.to_string();
     let config = ocr_provider_config(&custom_llm_provider, &model)
-        .ok_or_else(|| litellm_core::Error::InvalidProvider(custom_llm_provider.clone()));
+        .ok_or_else(|| litellm_core::Error::InvalidProvider(custom_llm_provider.clone()))
+        .and_then(|config| {
+            validate_request_format(config, &request.optional_params, &custom_llm_provider)?;
+            Ok(config)
+        });
     let optional_params = match &config {
         Ok(config) => {
             let supported = config.supported_ocr_params();
-            config.map_ocr_params(
+            let mut mapped = config.map_ocr_params(
                 &request
                     .optional_params
-                    .into_iter()
+                    .iter()
                     .filter(|(name, _)| supported.contains(&name.as_str()))
+                    .map(|(name, value)| (name.clone(), value.clone()))
                     .collect(),
-            )
+            );
+            for name in [
+                "vertex_project",
+                "vertex_ai_project",
+                "vertex_location",
+                "vertex_ai_location",
+            ] {
+                if let Some(value) = request.optional_params.get(name) {
+                    mapped.insert(name.to_string(), value.clone());
+                }
+            }
+            mapped
         }
         Err(_) => request.optional_params,
     };
@@ -64,6 +81,26 @@ pub(crate) fn prepare_ocr_call(request: OcrRequest<'_>) -> PreparedOcrCall {
     }
 }
 
+fn validate_request_format(
+    config: &'static dyn litellm_core::ocr::transformation::OcrProviderConfig,
+    optional_params: &Map<String, Value>,
+    provider: &str,
+) -> Result<(), litellm_core::Error> {
+    let Some(format) = optional_params.get("req_format") else {
+        return Ok(());
+    };
+    match format.as_str() {
+        Some("litellm") => Ok(()),
+        Some("native") if config.supported_ocr_params().contains(&"req_format") => Ok(()),
+        Some("native") => Err(litellm_core::Error::InvalidRequest(format!(
+            "`req_format=native` is not supported for provider {provider}"
+        ))),
+        _ => Err(litellm_core::Error::InvalidRequest(format!(
+            "Invalid `req_format`: {format}. Expected `litellm` or `native`"
+        ))),
+    }
+}
+
 fn new_ocr_call_id() -> String {
     static COUNTER: AtomicU64 = AtomicU64::new(1);
     let sequence = COUNTER.fetch_add(1, Ordering::Relaxed);
@@ -73,3 +110,54 @@ fn new_ocr_call_id() -> String {
         .unwrap_or(0);
     format!("ocr-{timestamp}-{sequence}")
 }
+
+#[cfg(test)]
+mod tests {
+    use litellm_core::error::Error;
+    use serde_json::{Map, json};
+
+    use super::{OcrRequest, prepare_ocr_call};
+    use crate::integrations::types::RequestMetadata;
+
+    fn base_ocr_request(model: &str) -> OcrRequest<'_> {
+        OcrRequest {
+            model,
+            document: json!({
+                "type": "document_url",
+                "document_url": "https://example.com/doc.pdf"
+            }),
+            api_key: Some("sk-test"),
+            api_base: None,
+            custom_llm_provider: None,
+            extra_headers: None,
+            optional_params: Map::new(),
+            timeout: None,
+            callbacks: Vec::new(),
+            guardrails: Vec::new(),
+            request_metadata: RequestMetadata::default(),
+            litellm_call_id: None,
+        }
+    }
+
+    fn request_with_format(format: &str) -> OcrRequest<'_> {
+        let mut request = base_ocr_request("mistral/mistral-ocr-latest");
+        request.optional_params = Map::from_iter([("req_format".to_string(), json!(format))]);
+        request
+    }
+
+    #[test]
+    fn native_format_rejected_for_provider_without_support_as_bad_request() {
+        let prepared = prepare_ocr_call(request_with_format("native"));
+        assert!(
+            matches!(prepared.request.config, Err(Error::InvalidRequest(message)) if message.contains("not supported for provider"))
+        );
+    }
+
+    #[test]
+    fn unknown_format_rejected_for_provider_without_support_as_bad_request() {
+        let prepared = prepare_ocr_call(request_with_format("raw"));
+        assert!(
+            matches!(prepared.request.config, Err(Error::InvalidRequest(message)) if message.contains("Invalid `req_format`"))
+        );
+    }
+}

litellm-rust/crates/ai-gateway/src/ocr/types.rs

@@ -53,6 +53,7 @@ pub(crate) struct ProviderOcrRequest {
     pub(crate) config: &'static dyn OcrProviderConfig,
     pub(crate) url: String,
     pub(crate) body: Value,
+    pub(crate) optional_params: Map<String, Value>,
     pub(crate) upstream_headers: Vec<(String, String)>,
     pub(crate) timeout: Option<Duration>,
 }

litellm-rust/crates/ai-gateway/src/realtime/streaming.rs

@@ -106,16 +106,16 @@ impl RealTimeStreaming {
     /// `litellm_call_id`, replacing the gateway-generated fallback.
     fn on_session(&mut self, event: &RealtimeEvent) {
         let session = event.data.get("session").and_then(Value::as_object);
-        if let Some(id) = session.and_then(|s| s.get("id")).and_then(Value::as_str) {
-            if !id.is_empty() {
-                self.id = id.to_string();
-                self.litellm_call_id = id.to_string();
-            }
+        if let Some(id) = session.and_then(|s| s.get("id")).and_then(Value::as_str)
+            && !id.is_empty()
+        {
+            self.id = id.to_string();
+            self.litellm_call_id = id.to_string();
         }
-        if let Some(model) = session.and_then(|s| s.get("model")).and_then(Value::as_str) {
-            if !model.is_empty() {
-                self.model = model.to_string();
-            }
+        if let Some(model) = session.and_then(|s| s.get("model")).and_then(Value::as_str)
+            && !model.is_empty()
+        {
+            self.model = model.to_string();
         }
     }
 
@@ -323,6 +323,32 @@ mod tests {
         assert_eq!(streaming.dropped(), 0);
     }
 
+    #[test]
+    fn blank_session_id_and_model_keep_the_gateway_fallbacks() {
+        let mut streaming = RealTimeStreaming::new(
+            Vec::new(),
+            "call_fallback".to_string(),
+            "gpt-realtime".to_string(),
+            RequestMetadata::default(),
+        );
+
+        streaming.observe(&event(
+            r#"{"type":"session.created","session":{"id":"","model":""}}"#,
+        ));
+        let payload = streaming.build_payload();
+        assert_eq!(payload.id, "call_fallback");
+        assert_eq!(payload.litellm_call_id, "call_fallback");
+        assert_eq!(payload.model, "gpt-realtime");
+
+        streaming.observe(&event(
+            r#"{"type":"session.updated","session":{"id":"sess_002","model":""}}"#,
+        ));
+        let payload = streaming.build_payload();
+        assert_eq!(payload.id, "sess_002");
+        assert_eq!(payload.litellm_call_id, "sess_002");
+        assert_eq!(payload.model, "gpt-realtime");
+    }
+
     #[test]
     fn payload_serializes_with_camelcase_times_and_realtime_call_type() {
         let mut streaming = RealTimeStreaming::new(

litellm-rust/crates/ai-gateway/src/routes/messages/mod.rs

@@ -21,6 +21,12 @@ pub fn router() -> Router<AppState> {
     Router::new().route(MESSAGES_ROUTE_PATH, post(handle))
 }
 
+#[tracing::instrument(
+    name = "messages_gateway_route",
+    target = "litellm::function_trace",
+    level = "trace",
+    skip_all
+)]
 async fn handle(
     _auth: RequireMasterKey,
     State(state): State<AppState>,

litellm-rust/crates/ai-gateway/src/routes/messages/service.rs

@@ -12,6 +12,12 @@ pub(crate) enum MessagesResponse {
     Stream(reqwest::Response),
 }
 
+#[tracing::instrument(
+    name = "messages_gateway_service",
+    target = "litellm::function_trace",
+    level = "trace",
+    skip_all
+)]
 pub async fn run(
     router: &Arc<Router>,
     body: Value,

litellm-rust/crates/ai-gateway/src/routes/realtime/service.rs

@@ -50,18 +50,17 @@ where
         provider_model,
         params.api_key.as_deref(),
         params.api_base.as_deref(),
-    ) {
-        if let Some(handoff) = pool.take(&key) {
-            return crate::io::realtime::realtime_warm(
-                provider_model,
-                handoff,
-                idle_timeout,
-                observe,
-                client_in,
-                client_out,
-            )
-            .await;
-        }
+    ) && let Some(handoff) = pool.take(&key)
+    {
+        return crate::io::realtime::realtime_warm(
+            provider_model,
+            handoff,
+            idle_timeout,
+            observe,
+            client_in,
+            client_out,
+        )
+        .await;
     }
 
     // Cold path: fresh dial (the original behavior).

litellm-rust/crates/ai-gateway/src/trace_parity.rs

@@ -0,0 +1,65 @@
+//! Harness-only in-process adapters. Never mounted as production routes.
+
+use std::sync::Arc;
+
+use axum::body::{Body, to_bytes};
+use axum::http::header::{AUTHORIZATION, CONTENT_TYPE};
+use axum::http::{Request, StatusCode};
+use litellm_core::Error;
+use litellm_core::router::{Deployment, LiteLLMParams, Router as ModelRouter};
+use serde::Serialize;
+use serde_json::Value;
+use tower::ServiceExt;
+
+use crate::io::realtime_pool::RealtimePool;
+use crate::routes;
+use crate::state::AppState;
+
+#[derive(Debug, Serialize)]
+pub struct GatewayResponse {
+    pub status: u16,
+    pub body: Value,
+}
+
+pub async fn messages_request(
+    model_alias: String,
+    provider_model: String,
+    api_base: String,
+    body: Value,
+) -> Result<GatewayResponse, Error> {
+    let state = AppState {
+        router: Arc::new(ModelRouter::new(vec![Deployment {
+            model_name: model_alias,
+            litellm_params: LiteLLMParams {
+                model: provider_model,
+                api_key: Some("trace-provider-key".to_string()),
+                api_base: Some(api_base),
+            },
+        }])),
+        master_key: Some(Arc::from("trace-master-key")),
+        loggers: Arc::new(Vec::new()),
+        realtime_pool: RealtimePool::disabled(),
+    };
+    let request = Request::builder()
+        .method("POST")
+        .uri("/v1/messages")
+        .header(AUTHORIZATION, "Bearer trace-master-key")
+        .header(CONTENT_TYPE, "application/json")
+        .body(Body::from(body.to_string()))
+        .map_err(|error| Error::InvalidRequest(error.to_string()))?;
+    let response = routes::app(state)
+        .oneshot(request)
+        .await
+        .map_err(|error| match error {})?;
+    let status: StatusCode = response.status();
+    let bytes = to_bytes(response.into_body(), usize::MAX)
+        .await
+        .map_err(|error| Error::InvalidResponse(error.to_string()))?;
+    let body = serde_json::from_slice(&bytes).map_err(|error| {
+        Error::InvalidResponse(format!("gateway returned invalid JSON: {error}"))
+    })?;
+    Ok(GatewayResponse {
+        status: status.as_u16(),
+        body,
+    })
+}

litellm-rust/crates/ai-gateway/tests/ocr_lifecycle.rs

@@ -1,23 +1,19 @@
 use std::sync::{Arc, Mutex};
 use std::time::Duration;
 
-use litellm_core::error::Error;
-use litellm_core::http_utils::has_header;
-use litellm_core::ocr::transformation::OcrResponseHandling;
-use serde_json::{Map, Value, json};
-use tokio::io::{AsyncReadExt, AsyncWriteExt};
-use tokio::net::{TcpListener, TcpStream};
-
-use super::common_utils::{ocr_provider_config, string_headers, truncate_error_body};
-use super::{OcrRequest, ocr};
-use crate::integrations::custom_guardrail::{
+use litellm_ai_gateway::integrations::custom_guardrail::{
     CustomGuardrail, GuardrailContext, GuardrailDecision, GuardrailError, GuardrailEventHook,
     GuardrailFuture, GuardrailRequest,
 };
-use crate::integrations::custom_logger::{
+use litellm_ai_gateway::integrations::custom_logger::{
     CallbackTiming, CallbackValue, CustomLogger, LogFuture, ModelCallDetails,
 };
-use crate::integrations::types::RequestMetadata;
+use litellm_ai_gateway::integrations::types::RequestMetadata;
+use litellm_ai_gateway::ocr::{OcrRequest, ocr};
+use litellm_core::error::Error;
+use serde_json::{Map, Value, json};
+use tokio::io::{AsyncReadExt, AsyncWriteExt};
+use tokio::net::{TcpListener, TcpStream};
 
 async fn read_http_headers(socket: &mut TcpStream) -> String {
     let mut request = Vec::new();
@@ -136,6 +132,7 @@ struct RecordingOcrGuardrail {
     hooks: Vec<GuardrailEventHook>,
     events: Mutex<Vec<&'static str>>,
     block_pre_call: bool,
+    block_during_call: bool,
 }
 
 impl RecordingOcrGuardrail {
@@ -144,6 +141,7 @@ impl RecordingOcrGuardrail {
             hooks,
             events: Mutex::new(Vec::new()),
             block_pre_call: false,
+            block_during_call: false,
         }
     }
 
@@ -152,6 +150,16 @@ impl RecordingOcrGuardrail {
             hooks: vec![GuardrailEventHook::PreCall],
             events: Mutex::new(Vec::new()),
             block_pre_call: true,
+            block_during_call: false,
+        }
+    }
+
+    fn blocking_during_call() -> Self {
+        Self {
+            hooks: vec![GuardrailEventHook::DuringCall],
+            events: Mutex::new(Vec::new()),
+            block_pre_call: false,
+            block_during_call: true,
         }
     }
 
@@ -193,91 +201,95 @@ impl CustomGuardrail for RecordingOcrGuardrail {
     ) -> GuardrailFuture<'a> {
         Box::pin(async move {
             self.events.lock().unwrap().push("async_moderation_hook");
+            if self.block_during_call {
+                return Ok(GuardrailDecision::Block(GuardrailError::blocked(
+                    "blocked before provider",
+                )));
+            }
             request.data["body"]["guarded_during"] = json!(true);
             Ok(GuardrailDecision::Mask(request))
         })
     }
 }
 
-#[test]
-fn truncate_error_body_passes_short_strings_through() {
-    let body = "Unauthorized";
-    assert_eq!(truncate_error_body(body), "Unauthorized");
+fn base_ocr_request(model: &str) -> OcrRequest<'_> {
+    OcrRequest {
+        model,
+        document: json!({
+            "type": "document_url",
+            "document_url": "https://example.com/doc.pdf"
+        }),
+        api_key: Some("sk-test"),
+        api_base: None,
+        custom_llm_provider: None,
+        extra_headers: None,
+        optional_params: Map::new(),
+        timeout: None,
+        callbacks: Vec::new(),
+        guardrails: Vec::new(),
+        request_metadata: RequestMetadata::default(),
+        litellm_call_id: None,
+    }
 }
 
-#[test]
-fn truncate_error_body_caps_long_payloads() {
-    let body = "x".repeat(306);
-    let truncated = truncate_error_body(&body);
+#[tokio::test]
+async fn reducto_during_call_guardrail_blocks_before_upload() {
+    let listener = TcpListener::bind("127.0.0.1:0")
+        .await
+        .expect("test listener binds");
+    let address = listener.local_addr().expect("listener has local address");
+    let api_base = format!("http://{address}");
+    let guardrail = Arc::new(RecordingOcrGuardrail::blocking_during_call());
+    let mut request = base_ocr_request("reducto/parse-v3");
+    request.api_base = Some(&api_base);
+    request.document = json!({
+        "type": "document_url",
+        "document_url": "data:application/pdf;base64,JVBERi0xLjQ="
+    });
+    request.guardrails = vec![guardrail.clone()];
 
-    assert!(truncated.ends_with("... (truncated)"));
-    let prefix_chars = truncated
-        .strip_suffix("... (truncated)")
-        .expect("truncated marker present")
-        .chars()
-        .count();
-    assert_eq!(prefix_chars, 256);
+    let error = ocr(request).await.expect_err("guardrail blocks upload");
+
+    assert!(matches!(error, Error::InvalidRequest(_)));
+    assert_eq!(guardrail.events(), vec!["async_moderation_hook"]);
+    let accepted = tokio::time::timeout(Duration::from_millis(100), listener.accept()).await;
+    assert!(accepted.is_err(), "upload socket should not be touched");
 }
 
-#[test]
-fn truncate_error_body_does_not_split_multibyte_chars() {
-    let body = "é".repeat(266);
-    let truncated = truncate_error_body(&body);
-    assert!(truncated.is_char_boundary(truncated.len()));
-}
+#[tokio::test]
+async fn reducto_upload_error_body_is_truncated() {
+    let listener = TcpListener::bind("127.0.0.1:0")
+        .await
+        .expect("test listener binds");
+    let address = listener.local_addr().expect("listener has local address");
+    let server = tokio::spawn(async move {
+        let (mut socket, _) = listener.accept().await.expect("accepts upload request");
+        let _request = read_http_request(&mut socket).await;
+        let body = "x".repeat(300);
+        let response = format!(
+            "HTTP/1.1 500 Internal Server Error\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{}",
+            body.len(),
+            body
+        );
+        socket
+            .write_all(response.as_bytes())
+            .await
+            .expect("writes upload response");
+    });
+    let api_base = format!("http://{address}");
+    let mut request = base_ocr_request("reducto/parse-v3");
+    request.api_base = Some(&api_base);
+    request.document = json!({
+        "type": "document_url",
+        "document_url": "data:application/pdf;base64,JVBERi0xLjQ="
+    });
+
+    let error = ocr(request).await.expect_err("upload should fail");
 
-#[test]
-fn ocr_dispatch_supports_migrated_providers() {
-    assert!(ocr_provider_config("mistral", "mistral-ocr-latest").is_some());
     assert!(
-        ocr_provider_config("azure_ai", "pixtral-12b-2409")
-            .expect("azure ai config resolves")
-            .requires_data_uri_document()
+        matches!(error, Error::Http { status: 500, body } if body.chars().count() < 300 && body.ends_with("... (truncated)"))
     );
-    assert_eq!(
-        ocr_provider_config("azure_ai", "doc-intelligence/prebuilt-read")
-            .expect("document intelligence config resolves")
-            .response_handling(),
-        OcrResponseHandling::AzureDocumentIntelligencePoll
-    );
-    assert!(
-        ocr_provider_config("vertex_ai", "deepseek-ocr-maas")
-            .expect("vertex deepseek config resolves")
-            .supported_ocr_params()
-            .contains(&"temperature")
-    );
-    assert!(ocr_provider_config("openai", "gpt-4o").is_none());
-}
-
-#[test]
-fn string_headers_accepts_string_values() {
-    let headers = json!({
-        "x-trace-id": "trace-1"
-    })
-    .as_object()
-    .unwrap()
-    .clone();
-
-    assert_eq!(
-        string_headers(Some(headers)).expect("string headers accepted"),
-        vec![("x-trace-id".to_string(), "trace-1".to_string())]
-    );
-}
-
-#[test]
-fn auth_header_detection_is_case_insensitive() {
-    let headers = vec![
-        ("x-trace-id".to_string(), "trace-1".to_string()),
-        ("authorization".to_string(), "Bearer sk-test".to_string()),
-    ];
-
-    assert!(has_header(&headers, "authorization"));
-
-    let headers = vec![("Authorization".to_string(), "Bearer sk-test".to_string())];
-    assert!(has_header(&headers, "authorization"));
-
-    let headers = vec![("x-trace-id".to_string(), "trace-1".to_string())];
-    assert!(!has_header(&headers, "authorization"));
+    server.await.expect("server task completes");
 }
 
 #[tokio::test]
@@ -595,21 +607,3 @@ async fn document_intelligence_poll_uses_resolved_subscription_key() {
         "{poll_request}"
     );
 }
-
-#[test]
-fn string_headers_rejects_non_string_values() {
-    let headers = json!({
-        "x-retry-count": 3
-    })
-    .as_object()
-    .unwrap()
-    .clone();
-
-    let err = string_headers(Some(headers)).expect_err("non-string header rejected");
-    assert_eq!(
-        err,
-        Error::InvalidRequest(
-            "OCR extra_headers.x-retry-count must be a string, got number".to_string()
-        )
-    );
-}

litellm-rust/crates/core/Cargo.toml

@@ -6,12 +6,14 @@ license.workspace = true
 repository.workspace = true
 
 [dependencies]
+base64.workspace = true
 rand.workspace = true
 reqwest.workspace = true
 serde.workspace = true
 serde_json.workspace = true
 thiserror.workspace = true
 tracing.workspace = true
+tracing-subscriber = { workspace = true, optional = true }
 sha2.workspace = true
 aws-config = { version = "1.9.0", default-features = false, features = ["rustls", "rt-tokio"], optional = true }
 aws-credential-types = { version = "1.3.0", features = ["hardcoded-credentials"], optional = true }
@@ -30,6 +32,9 @@ bedrock-auth = [
     "dep:aws-types",
     "dep:aws-smithy-runtime-api",
 ]
+observability = ["dep:tracing-subscriber"]
 
 [dev-dependencies]
+rstest.workspace = true
 tokio = { workspace = true, features = ["macros", "rt-multi-thread"] }
+tracing-subscriber.workspace = true

litellm-rust/crates/core/src/constants.rs

@@ -41,3 +41,5 @@ pub const CHAT_COMPLETION_OBJECT: &str = "chat.completion";
 /// `litellm/litellm_core_utils/prompt_templates/factory.py`.
 pub const EMPTY_TEXT_PLACEHOLDER: &str =
     "[System: Empty message content sanitised to satisfy protocol]";
+
+pub const FUNCTION_TRACE_TARGET: &str = "litellm::function_trace";

litellm-rust/crates/core/src/http_utils.rs

@@ -101,6 +101,23 @@ mod tests {
         assert!(!has_header(&headers, "authorization"));
     }
 
+    #[test]
+    fn auth_header_detection_is_case_insensitive() {
+        let headers = vec![
+            ("x-trace-id".to_string(), "trace-1".to_string()),
+            ("authorization".to_string(), "Bearer sk-test".to_string()),
+        ];
+
+        assert!(has_header(&headers, "authorization"));
+
+        let headers = vec![("Authorization".to_string(), "Bearer sk-test".to_string())];
+
+        assert!(has_header(&headers, "authorization"));
+
+        let headers = vec![("x-trace-id".to_string(), "trace-1".to_string())];
+        assert!(!has_header(&headers, "authorization"));
+    }
+
     #[test]
     fn bearer_detection_requires_a_non_empty_token() {
         assert!(has_bearer_auth(&[(

litellm-rust/crates/core/src/lib.rs

@@ -6,6 +6,8 @@ pub mod constants;
 pub mod error;
 pub mod http_utils;
 pub mod messages;
+#[cfg(any(feature = "observability", test))]
+pub mod observability;
 pub mod ocr;
 pub mod providers;
 pub mod realtime;

litellm-rust/crates/core/src/observability/function_trace.rs

@@ -0,0 +1,215 @@
+use std::collections::HashMap;
+use std::sync::{Arc, Mutex};
+
+use serde::Serialize;
+use tracing::span::{Attributes, Id};
+use tracing::{Dispatch, Subscriber};
+use tracing_subscriber::layer::Context;
+use tracing_subscriber::prelude::*;
+use tracing_subscriber::registry::LookupSpan;
+use tracing_subscriber::{Layer, Registry};
+
+use super::function_trace_filter;
+
+#[derive(Clone, Debug, PartialEq, Serialize)]
+pub struct FunctionTraceEvent {
+    pub id: usize,
+    pub parent_id: Option<usize>,
+    pub function: &'static str,
+    pub module_path: Option<&'static str>,
+    pub file: Option<&'static str>,
+    pub line: Option<u32>,
+}
+
+#[derive(Clone, Default)]
+pub struct FunctionTrace {
+    events: Arc<Mutex<Vec<FunctionTraceEvent>>>,
+    span_events: Arc<Mutex<HashMap<Id, usize>>>,
+}
+
+impl FunctionTrace {
+    pub fn dispatcher(&self) -> Dispatch {
+        Dispatch::new(
+            Registry::default().with(
+                FunctionTraceLayer {
+                    trace: self.clone(),
+                }
+                .with_filter(function_trace_filter()),
+            ),
+        )
+    }
+
+    pub fn events(&self) -> Vec<FunctionTraceEvent> {
+        self.events
+            .lock()
+            .unwrap_or_else(|error| error.into_inner())
+            .clone()
+    }
+}
+
+struct FunctionTraceLayer {
+    trace: FunctionTrace,
+}
+
+impl<S> Layer<S> for FunctionTraceLayer
+where
+    S: Subscriber + for<'lookup> LookupSpan<'lookup>,
+{
+    fn on_new_span(&self, attributes: &Attributes<'_>, id: &Id, context: Context<'_, S>) {
+        let parent_id = context.span(id).and_then(|span| {
+            let span_events = self
+                .trace
+                .span_events
+                .lock()
+                .unwrap_or_else(|error| error.into_inner());
+            span.scope()
+                .skip(1)
+                .find_map(|ancestor| span_events.get(&ancestor.id()).copied())
+        });
+        let mut events = self
+            .trace
+            .events
+            .lock()
+            .unwrap_or_else(|error| error.into_inner());
+        let event_id = events.len();
+        events.push(FunctionTraceEvent {
+            id: event_id,
+            parent_id,
+            function: attributes.metadata().name(),
+            module_path: attributes.metadata().module_path(),
+            file: attributes.metadata().file(),
+            line: attributes.metadata().line(),
+        });
+        self.trace
+            .span_events
+            .lock()
+            .unwrap_or_else(|error| error.into_inner())
+            .insert(id.clone(), event_id);
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use crate::constants::FUNCTION_TRACE_TARGET;
+
+    use super::*;
+
+    fn event(
+        id: usize,
+        parent_id: Option<usize>,
+        function: &'static str,
+    ) -> (usize, Option<usize>, &'static str) {
+        (id, parent_id, function)
+    }
+
+    fn structural_events(
+        events: &[FunctionTraceEvent],
+    ) -> Vec<(usize, Option<usize>, &'static str)> {
+        events
+            .iter()
+            .map(|event| (event.id, event.parent_id, event.function))
+            .collect()
+    }
+
+    #[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)]
+    async fn outer() {
+        tokio::task::yield_now().await;
+        inner().await;
+    }
+
+    #[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)]
+    async fn inner() {
+        tokio::task::yield_now().await;
+    }
+
+    #[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)]
+    async fn concurrent_parent() {
+        tokio::join!(inner(), inner());
+    }
+
+    #[tokio::test]
+    async fn concurrent_futures_keep_separate_traces_across_yields() {
+        use tracing::instrument::WithSubscriber;
+
+        let first = FunctionTrace::default();
+        let second = FunctionTrace::default();
+        let outside = FunctionTrace::default();
+
+        async {
+            tokio::join!(
+                outer().with_subscriber(first.dispatcher()),
+                inner().with_subscriber(second.dispatcher()),
+            );
+            inner().await;
+        }
+        .with_subscriber(outside.dispatcher())
+        .await;
+
+        assert_eq!(
+            structural_events(&first.events()),
+            vec![event(0, None, "outer"), event(1, Some(0), "inner")],
+        );
+        assert_eq!(
+            structural_events(&second.events()),
+            vec![event(0, None, "inner")],
+        );
+        assert_eq!(
+            structural_events(&outside.events()),
+            vec![event(0, None, "inner")],
+        );
+    }
+
+    #[tokio::test]
+    async fn concurrent_siblings_keep_the_same_parent() {
+        use tracing::instrument::WithSubscriber;
+
+        let trace = FunctionTrace::default();
+        concurrent_parent()
+            .with_subscriber(trace.dispatcher())
+            .await;
+
+        assert_eq!(
+            structural_events(&trace.events()),
+            vec![
+                event(0, None, "concurrent_parent"),
+                event(1, Some(0), "inner"),
+                event(2, Some(0), "inner"),
+            ]
+        );
+    }
+
+    #[test]
+    fn records_matching_spans_in_creation_order() {
+        let trace = FunctionTrace::default();
+        let dispatch = trace.dispatcher();
+
+        tracing::dispatcher::with_default(&dispatch, || {
+            let _ignored = tracing::trace_span!(target: "other", "ignored");
+            let _first = tracing::trace_span!(target: FUNCTION_TRACE_TARGET, "same_name");
+            let _wrong_level = tracing::debug_span!(target: FUNCTION_TRACE_TARGET, "wrong_level");
+            let _second = tracing::trace_span!(target: FUNCTION_TRACE_TARGET, "same_name");
+        });
+
+        assert_eq!(
+            structural_events(&trace.events()),
+            vec![event(0, None, "same_name"), event(1, None, "same_name")]
+        );
+    }
+
+    #[test]
+    fn records_matching_span_nesting_depth() {
+        let trace = FunctionTrace::default();
+        let dispatch = trace.dispatcher();
+
+        tracing::dispatcher::with_default(&dispatch, || {
+            let outer = tracing::trace_span!(target: FUNCTION_TRACE_TARGET, "outer");
+            let _outer_guard = outer.enter();
+            let _inner = tracing::trace_span!(target: FUNCTION_TRACE_TARGET, "inner");
+        });
+
+        assert_eq!(
+            structural_events(&trace.events()),
+            vec![event(0, None, "outer"), event(1, Some(0), "inner")]
+        );
+    }
+}

litellm-rust/crates/core/src/observability/mod.rs

@@ -0,0 +1,59 @@
+use tracing::span::Id;
+use tracing::{Level, Metadata, Subscriber};
+use tracing_subscriber::filter::{FilterFn, LevelFilter, filter_fn};
+use tracing_subscriber::layer::Context;
+use tracing_subscriber::registry::LookupSpan;
+
+use crate::constants::FUNCTION_TRACE_TARGET;
+
+pub mod function_trace;
+
+pub use function_trace::{FunctionTrace, FunctionTraceEvent};
+
+pub fn function_trace_filter() -> FilterFn<impl Fn(&Metadata<'_>) -> bool> {
+    filter_fn(|metadata| {
+        metadata.is_span()
+            && metadata.target() == FUNCTION_TRACE_TARGET
+            && *metadata.level() == Level::TRACE
+    })
+    .with_max_level_hint(LevelFilter::TRACE)
+}
+
+pub fn span_depth<S>(context: &Context<'_, S>, id: &Id) -> usize
+where
+    S: Subscriber + for<'lookup> LookupSpan<'lookup>,
+{
+    context
+        .span(id)
+        .map(|span| span.scope().skip(1).count())
+        .unwrap_or_default()
+}
+
+#[cfg(test)]
+mod tests {
+    use tracing::instrument::WithSubscriber;
+
+    use super::*;
+
+    #[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)]
+    async fn instrumented_with_literal_target() {}
+
+    #[tokio::test]
+    async fn literal_instrument_target_matches_filter_constant() {
+        assert_eq!(FUNCTION_TRACE_TARGET, "litellm::function_trace");
+
+        let trace = FunctionTrace::default();
+        instrumented_with_literal_target()
+            .with_subscriber(trace.dispatcher())
+            .await;
+
+        let events = trace.events();
+        assert_eq!(events.len(), 1);
+        assert_eq!(events[0].id, 0);
+        assert_eq!(events[0].parent_id, None);
+        assert_eq!(events[0].function, "instrumented_with_literal_target");
+        assert_eq!(events[0].module_path, Some(module_path!()));
+        assert_eq!(events[0].file, Some(file!()));
+        assert!(events[0].line.is_some());
+    }
+}

litellm-rust/crates/core/src/ocr/transformation.rs

@@ -51,6 +51,15 @@ pub trait OcrProviderConfig: Sync {
         response_json: Value,
     ) -> Result<OcrResponseData, Error>;
 
+    fn transform_ocr_response_with_params(
+        &self,
+        model: &str,
+        response_json: Value,
+        _optional_params: &Map<String, Value>,
+    ) -> Result<OcrResponseData, Error> {
+        self.transform_ocr_response(model, response_json)
+    }
+
     fn complete_url(
         &self,
         api_base: Option<&str>,

litellm-rust/crates/core/src/ocr/types.rs

@@ -1,5 +1,5 @@
 use serde::{Deserialize, Serialize};
-use serde_json::Value;
+use serde_json::{Map, Value};
 
 #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
 pub struct OcrRequestData {
@@ -14,16 +14,25 @@ pub struct OcrResponseData {
     pub document_annotation: Option<Value>,
     pub usage_info: Option<Value>,
     pub object: String,
+    pub extra_fields: Map<String, Value>,
+    pub provider_native_response: Option<Value>,
 }
 
 impl OcrResponseData {
     pub fn into_json(self) -> Value {
-        serde_json::json!({
+        let mut response = serde_json::json!({
             "pages": self.pages,
             "model": self.model,
             "document_annotation": self.document_annotation,
             "usage_info": self.usage_info,
             "object": self.object,
-        })
+        });
+        if let Value::Object(object) = &mut response {
+            object.extend(self.extra_fields);
+            if let Some(native_response) = self.provider_native_response {
+                object.insert("provider_native_response".to_string(), native_response);
+            }
+        }
+        response
     }
 }

litellm-rust/crates/core/src/providers/mistral/ocr/transformation.rs

@@ -134,6 +134,8 @@ impl OcrProviderConfig for MistralOcrConfig {
             document_annotation,
             usage_info,
             object: "ocr".to_string(),
+            extra_fields: Map::new(),
+            provider_native_response: None,
         })
     }
 

litellm-rust/crates/core/src/providers/mod.rs

@@ -4,4 +4,5 @@ pub mod azure_ai;
 pub mod bedrock;
 pub mod mistral;
 pub mod openai;
+pub mod reducto;
 pub mod vertex_ai;

litellm-rust/crates/core/src/providers/reducto/mod.rs

@@ -0,0 +1 @@
+pub mod ocr;

litellm-rust/crates/core/src/providers/reducto/ocr/mod.rs

@@ -0,0 +1,4 @@
+pub mod transformation;
+
+#[cfg(test)]
+mod tests;

litellm-rust/crates/core/src/providers/reducto/ocr/tests.rs

@@ -0,0 +1,202 @@
+use rstest::{fixture, rstest};
+use serde_json::{Value, json};
+
+use super::transformation::*;
+use crate::ocr::transformation::OcrProviderConfig;
+
+#[fixture]
+fn parse_response() -> Value {
+    json!({
+        "job_id": "job_123",
+        "usage": {"num_pages": 3, "credits": 3},
+        "result": {
+            "chunks": [
+                {
+                    "content": "Page 1 block A",
+                    "blocks": [{
+                        "content": "Page 1 block A",
+                        "bbox": {"page": 1},
+                        "kind": "text",
+                    }],
+                },
+                {
+                    "content": "Page 2 block A",
+                    "blocks": [{
+                        "content": "Page 2 block A",
+                        "bbox": {"page": 2},
+                        "kind": "table",
+                    }],
+                },
+                {
+                    "content": "Page 1 block B",
+                    "blocks": [{
+                        "content": "Page 1 block B",
+                        "bbox": {"page": 1},
+                        "kind": "text",
+                    }],
+                },
+                {
+                    "content": "Page 3 block A",
+                    "blocks": [{
+                        "content": "Page 3 block A",
+                        "bbox": {"page": 3},
+                        "kind": "figure",
+                    }],
+                },
+            ],
+        },
+    })
+}
+
+#[rstest]
+fn test_parse_v3_file_upload_and_response_mapping(parse_response: Value) {
+    let source = classify_document_source("data:application/pdf;base64,JVBERi0xLjQ=")
+        .expect("PDF data URI should be valid");
+    let upload = build_upload_request(
+        source,
+        "Bearer test-key",
+        Some("https://platform.reducto.ai"),
+    )
+    .expect("data URI should require upload");
+    assert_eq!(upload.url, "https://platform.reducto.ai/upload");
+    assert_eq!(upload.authorization, "Bearer test-key");
+    assert_eq!(upload.file_name, "document");
+    assert_eq!(upload.mime_type, "application/pdf");
+    assert_eq!(upload.bytes, b"%PDF-1.4");
+
+    let optional_params = json!({
+        "formatting": {"table_output_format": "html"},
+        "retrieval": {"chunk_mode": "section"},
+        "settings": {"ocr_system": "standard"},
+    })
+    .as_object()
+    .expect("params should be an object")
+    .clone();
+    let request = build_parse_v3_request("reducto://uploaded.pdf", optional_params);
+    assert_eq!(
+        request.data,
+        json!({
+            "input": "reducto://uploaded.pdf",
+            "formatting": {"table_output_format": "html"},
+            "retrieval": {"chunk_mode": "section"},
+            "settings": {"ocr_system": "standard"},
+        })
+    );
+
+    let transformed = transform_reducto_response("parse-v3", parse_response.clone())
+        .expect("response should transform");
+    assert_eq!(
+        transformed.usage_info,
+        Some(json!({"pages_processed": 3, "credits": 3}))
+    );
+    assert_eq!(transformed.pages.len(), 3);
+    assert_eq!(
+        transformed.pages[0],
+        json!({
+            "index": 0,
+            "markdown": "Page 1 block A\n\nPage 1 block B",
+            "blocks": [
+                {"content": "Page 1 block A", "bbox": {"page": 1}, "kind": "text"},
+                {"content": "Page 1 block B", "bbox": {"page": 1}, "kind": "text"},
+            ],
+        })
+    );
+    assert_eq!(transformed.pages[1]["markdown"], "Page 2 block A");
+    assert_eq!(transformed.pages[2]["markdown"], "Page 3 block A");
+    assert_eq!(transformed.provider_native_response, Some(parse_response));
+}
+
+#[rstest]
+fn test_parse_v3_reducto_id_passthrough_skips_upload(parse_response: Value) {
+    let document = json!({
+        "type": "document_url",
+        "document_url": "reducto://already-uploaded.pdf",
+    });
+    let source = extract_document_source(&document).expect("Reducto ID should be valid");
+    assert!(build_upload_request(source.clone(), "Bearer test-key", None).is_none());
+    assert_eq!(
+        source,
+        ReductoDocumentSource::FileId("reducto://already-uploaded.pdf".to_string())
+    );
+
+    let request = REDUCTO_PARSE_V3_CONFIG
+        .transform_ocr_request(
+            "parse-v3",
+            document,
+            json!({"retrieval": {"chunk_mode": "section"}})
+                .as_object()
+                .expect("params should be object")
+                .clone(),
+        )
+        .expect("direct ID should transform");
+    assert_eq!(request.data["input"], "reducto://already-uploaded.pdf");
+    assert_eq!(request.data["retrieval"]["chunk_mode"], "section");
+
+    let response = REDUCTO_PARSE_V3_CONFIG
+        .transform_ocr_response("parse-v3", parse_response)
+        .expect("response should transform");
+    assert!(
+        response.pages[0]["markdown"]
+            .as_str()
+            .expect("markdown should be string")
+            .starts_with("Page 1 block A")
+    );
+}
+
+#[rstest]
+fn test_parse_legacy_wraps_enhance_under_options() {
+    let request = build_parse_legacy_request(
+        "reducto://legacy.pdf",
+        json!({"enhance": {"agentic": [{"type": "table"}]}})
+            .as_object()
+            .expect("params should be object"),
+    );
+    assert_eq!(
+        request.data,
+        json!({
+            "document_url": "reducto://legacy.pdf",
+            "options": {"enhance": {"agentic": [{"type": "table"}]}},
+        })
+    );
+}
+
+#[rstest]
+fn test_parse_v3_image_data_uri_upload_uses_image_mime() {
+    let source = classify_document_source("data:image/png;base64,iVBORw0KGgo=")
+        .expect("PNG data URI should be valid");
+    let upload = build_upload_request(
+        source,
+        "Bearer programmatic-key",
+        Some("https://custom.reducto.test/"),
+    )
+    .expect("data URI should require upload");
+    assert_eq!(upload.url, "https://custom.reducto.test/upload");
+    assert_eq!(upload.authorization, "Bearer programmatic-key");
+    assert_eq!(upload.mime_type, "image/png");
+    assert_eq!(upload.bytes, b"\x89PNG\r\n\x1a\n");
+}
+
+#[rstest]
+#[case::http("http://example.com/document.pdf")]
+#[case::https("https://example.com/document.pdf")]
+fn test_parse_v3_rejects_plain_http_urls(#[case] source: &str) {
+    let error = classify_document_source(source).expect_err("plain URL should be rejected");
+    assert!(error.to_string().contains("upload the file first"));
+}
+
+#[rstest]
+fn test_parse_v3_uses_programmatic_api_key_over_env() {
+    let key = resolve_api_key(Some("passed-key"), &|_| Some("env-reducto-key".to_string()))
+        .expect("explicit key should resolve");
+    assert_eq!(key, "passed-key");
+
+    let headers = REDUCTO_PARSE_V3_CONFIG
+        .validate_environment(Vec::new(), Some("passed-key"), &|_| {
+            Some("env-reducto-key".to_string())
+        })
+        .expect("headers should validate");
+    assert_eq!(
+        headers,
+        vec![("Authorization".to_string(), "Bearer passed-key".to_string())]
+    );
+}

litellm-rust/crates/core/src/providers/reducto/ocr/transformation.rs

@@ -0,0 +1,407 @@
+use std::collections::BTreeMap;
+
+use base64::Engine;
+use base64::engine::general_purpose::STANDARD as BASE64_STANDARD;
+use serde_json::{Map, Value, json};
+
+use crate::error::{Error, json_type_name};
+use crate::ocr::transformation::OcrProviderConfig;
+use crate::ocr::types::{OcrRequestData, OcrResponseData};
+
+pub const REDUCTO_API_BASE: &str = "https://platform.reducto.ai";
+pub const REDUCTO_API_KEY_ENV: &str = "REDUCTO_API_KEY";
+pub const REDUCTO_ID_PREFIX: &str = "reducto://";
+
+const PARSE_V3_SUPPORTED_OCR_PARAMS: &[&str] = &["formatting", "retrieval", "settings"];
+const PARSE_LEGACY_SUPPORTED_OCR_PARAMS: &[&str] = &["enhance"];
+const MISSING_KEY_MESSAGE: &str = "Missing REDUCTO_API_KEY - set it in the environment or pass api_key to litellm.ocr()/litellm.aocr()";
+const DATA_URI_UPLOAD_REQUIRED: &str =
+    "Reducto data URI upload must complete before OCR request transformation";
+
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub enum ReductoDocumentSource {
+    FileId(String),
+    Upload { bytes: Vec<u8>, mime_type: String },
+}
+
+#[derive(Clone, PartialEq, Eq)]
+pub struct ReductoUploadRequest {
+    pub url: String,
+    pub authorization: String,
+    pub file_name: &'static str,
+    pub bytes: Vec<u8>,
+    pub mime_type: String,
+}
+
+pub struct ReductoParseV3Config;
+pub struct ReductoParseLegacyConfig;
+
+pub const REDUCTO_PARSE_V3_CONFIG: ReductoParseV3Config = ReductoParseV3Config;
+pub const REDUCTO_PARSE_LEGACY_CONFIG: ReductoParseLegacyConfig = ReductoParseLegacyConfig;
+
+pub fn config_for_model(model: &str) -> Option<&'static dyn OcrProviderConfig> {
+    match model {
+        "parse-v3" => Some(&REDUCTO_PARSE_V3_CONFIG),
+        "parse-legacy" => Some(&REDUCTO_PARSE_LEGACY_CONFIG),
+        _ => None,
+    }
+}
+
+pub fn normalize_api_base(api_base: Option<&str>) -> String {
+    api_base
+        .map(str::trim)
+        .filter(|base| !base.is_empty())
+        .unwrap_or(REDUCTO_API_BASE)
+        .trim_end_matches('/')
+        .to_string()
+}
+
+pub fn parse_url(api_base: Option<&str>) -> String {
+    format!("{}/parse", normalize_api_base(api_base))
+}
+
+pub fn upload_url(api_base: Option<&str>) -> String {
+    format!("{}/upload", normalize_api_base(api_base))
+}
+
+pub fn resolve_api_key(
+    api_key: Option<&str>,
+    env_lookup: &dyn Fn(&str) -> Option<String>,
+) -> Result<String, Error> {
+    api_key
+        .map(str::trim)
+        .filter(|key| !key.is_empty())
+        .map(str::to_string)
+        .or_else(|| {
+            env_lookup(REDUCTO_API_KEY_ENV)
+                .map(|key| key.trim().to_string())
+                .filter(|key| !key.is_empty())
+        })
+        .ok_or_else(|| Error::Auth(MISSING_KEY_MESSAGE.to_string()))
+}
+
+pub fn extract_document_source(document: &Value) -> Result<ReductoDocumentSource, Error> {
+    let document = document.as_object().ok_or_else(|| Error::InvalidType {
+        expected: "object",
+        actual: json_type_name(document),
+    })?;
+    let source = document
+        .get("document_url")
+        .and_then(Value::as_str)
+        .filter(|source| !source.is_empty())
+        .or_else(|| document.get("image_url").and_then(Value::as_str))
+        .ok_or_else(|| {
+            Error::InvalidRequest(
+                "Reducto expected OCR preprocessing to produce document_url or image_url"
+                    .to_string(),
+            )
+        })?;
+    classify_document_source(source)
+}
+
+pub fn classify_document_source(source: &str) -> Result<ReductoDocumentSource, Error> {
+    if source.starts_with(REDUCTO_ID_PREFIX) {
+        return Ok(ReductoDocumentSource::FileId(source.to_string()));
+    }
+    if source.starts_with("http://") || source.starts_with("https://") {
+        return Err(Error::InvalidRequest(
+            "Reducto requires type='file' (auto-uploaded) or a reducto:// id. Plain http(s) URLs are not supported; upload the file first."
+                .to_string(),
+        ));
+    }
+    if !source.starts_with("data:") {
+        return Err(Error::InvalidRequest(
+            "Reducto requires a reducto:// id or a base64 data URI after OCR preprocessing."
+                .to_string(),
+        ));
+    }
+
+    let (header, encoded) = source
+        .split_once(',')
+        .ok_or_else(|| Error::InvalidRequest("Invalid Reducto data URI provided.".to_string()))?;
+    if !header.split(';').any(|part| part == "base64") {
+        return Err(Error::InvalidRequest(
+            "Reducto only supports base64-encoded data URIs.".to_string(),
+        ));
+    }
+
+    let mime_type = header
+        .strip_prefix("data:")
+        .and_then(|header| header.split(';').next())
+        .filter(|mime| !mime.is_empty())
+        .unwrap_or("application/octet-stream")
+        .to_string();
+    let bytes = BASE64_STANDARD.decode(encoded).map_err(|_| {
+        Error::InvalidRequest("Invalid Reducto base64 payload provided.".to_string())
+    })?;
+
+    Ok(ReductoDocumentSource::Upload { bytes, mime_type })
+}
+
+pub fn build_upload_request(
+    source: ReductoDocumentSource,
+    authorization: &str,
+    api_base: Option<&str>,
+) -> Option<ReductoUploadRequest> {
+    let ReductoDocumentSource::Upload { bytes, mime_type } = source else {
+        return None;
+    };
+
+    Some(ReductoUploadRequest {
+        url: upload_url(api_base),
+        authorization: authorization.to_string(),
+        file_name: "document",
+        bytes,
+        mime_type,
+    })
+}
+
+pub fn extract_upload_file_id(response_json: &Value) -> Result<&str, Error> {
+    response_json
+        .as_object()
+        .and_then(|response| response.get("file_id"))
+        .and_then(Value::as_str)
+        .filter(|file_id| !file_id.is_empty())
+        .ok_or_else(|| {
+            Error::InvalidResponse(format!(
+                "Reducto /upload returned 200 without a file_id; got payload={response_json}"
+            ))
+        })
+}
+
+pub fn build_parse_v3_request(
+    file_id: &str,
+    optional_params: Map<String, Value>,
+) -> OcrRequestData {
+    let data = std::iter::once(("input".to_string(), Value::String(file_id.to_string())))
+        .chain(optional_params)
+        .collect();
+    OcrRequestData {
+        data: Value::Object(data),
+        files: None,
+    }
+}
+
+pub fn build_parse_legacy_request(
+    file_id: &str,
+    optional_params: &Map<String, Value>,
+) -> OcrRequestData {
+    let options = optional_params
+        .get("enhance")
+        .filter(|enhance| !enhance.is_null())
+        .map(|enhance| json!({"options": {"enhance": enhance}}));
+    let data = match options {
+        Some(Value::Object(options)) => std::iter::once((
+            "document_url".to_string(),
+            Value::String(file_id.to_string()),
+        ))
+        .chain(options)
+        .collect(),
+        _ => Map::from_iter([(
+            "document_url".to_string(),
+            Value::String(file_id.to_string()),
+        )]),
+    };
+    OcrRequestData {
+        data: Value::Object(data),
+        files: None,
+    }
+}
+
+fn source_file_id(document: &Value) -> Result<String, Error> {
+    match extract_document_source(document)? {
+        ReductoDocumentSource::FileId(file_id) => Ok(file_id),
+        ReductoDocumentSource::Upload { .. } => Err(Error::Unsupported(DATA_URI_UPLOAD_REQUIRED)),
+    }
+}
+
+fn page_number(block: &Map<String, Value>) -> Option<i64> {
+    let page = block.get("bbox")?.as_object()?.get("page")?;
+    page.as_i64()
+        .or_else(|| page.as_u64().and_then(|page| i64::try_from(page).ok()))
+        .or_else(|| page.as_str().and_then(|page| page.parse().ok()))
+}
+
+fn chunks(result: &Map<String, Value>) -> &[Value] {
+    result
+        .get("chunks")
+        .and_then(Value::as_array)
+        .map(Vec::as_slice)
+        .unwrap_or_default()
+}
+
+fn build_pages(result: &Map<String, Value>) -> Vec<Value> {
+    let blocks_by_page = chunks(result)
+        .iter()
+        .filter_map(Value::as_object)
+        .filter_map(|chunk| chunk.get("blocks").and_then(Value::as_array))
+        .flatten()
+        .filter_map(|block| block.as_object().map(|object| (block, object)))
+        .filter_map(|(block, object)| page_number(object).map(|page| (page, block.clone())))
+        .fold(
+            BTreeMap::<i64, Vec<Value>>::new(),
+            |mut pages, (page, block)| {
+                pages.entry(page).or_default().push(block);
+                pages
+            },
+        );
+
+    if blocks_by_page.is_empty() {
+        let markdown = chunks(result)
+            .iter()
+            .filter_map(Value::as_object)
+            .filter_map(|chunk| chunk.get("content").and_then(Value::as_str))
+            .filter(|content| !content.is_empty())
+            .collect::<Vec<_>>()
+            .join("\n\n");
+        return if markdown.is_empty() {
+            Vec::new()
+        } else {
+            vec![json!({"index": 0, "markdown": markdown})]
+        };
+    }
+
+    blocks_by_page
+        .into_iter()
+        .map(|(page, blocks)| {
+            let markdown = blocks
+                .iter()
+                .filter_map(Value::as_object)
+                .filter_map(|block| block.get("content").and_then(Value::as_str))
+                .filter(|content| !content.is_empty())
+                .collect::<Vec<_>>()
+                .join("\n\n");
+            json!({
+                "index": page.saturating_sub(1).max(0),
+                "markdown": markdown,
+                "blocks": blocks,
+            })
+        })
+        .collect()
+}
+
+pub fn transform_reducto_response(
+    model: &str,
+    response_json: Value,
+) -> Result<OcrResponseData, Error> {
+    let response = response_json
+        .as_object()
+        .ok_or_else(|| Error::InvalidType {
+            expected: "object",
+            actual: json_type_name(&response_json),
+        })?;
+    let empty_result = Map::new();
+    let result = match response.get("result") {
+        Some(Value::Object(result)) => result,
+        Some(Value::Null) => &empty_result,
+        Some(_) => {
+            return Err(Error::InvalidResponse(
+                "Reducto result must be an object".to_string(),
+            ));
+        }
+        None => response,
+    };
+    let usage = response
+        .get("usage")
+        .and_then(Value::as_object)
+        .cloned()
+        .unwrap_or_default();
+    let usage_info = Some(json!({
+        "pages_processed": usage.get("num_pages").cloned().unwrap_or(Value::Null),
+        "credits": usage.get("credits").cloned().unwrap_or(Value::Null),
+    }));
+
+    Ok(OcrResponseData {
+        pages: build_pages(result),
+        model: model.to_string(),
+        document_annotation: None,
+        usage_info,
+        object: "ocr".to_string(),
+        extra_fields: Map::new(),
+        provider_native_response: Some(response_json),
+    })
+}
+
+impl OcrProviderConfig for ReductoParseV3Config {
+    fn supported_ocr_params(&self) -> &'static [&'static str] {
+        PARSE_V3_SUPPORTED_OCR_PARAMS
+    }
+
+    fn transform_ocr_request(
+        &self,
+        _model: &str,
+        document: Value,
+        optional_params: Map<String, Value>,
+    ) -> Result<OcrRequestData, Error> {
+        let file_id = source_file_id(&document)?;
+        Ok(build_parse_v3_request(&file_id, optional_params))
+    }
+
+    fn transform_ocr_response(
+        &self,
+        model: &str,
+        response_json: Value,
+    ) -> Result<OcrResponseData, Error> {
+        transform_reducto_response(model, response_json)
+    }
+
+    fn complete_url(
+        &self,
+        api_base: Option<&str>,
+        _model: &str,
+        _optional_params: &Map<String, Value>,
+        _env_lookup: &dyn Fn(&str) -> Option<String>,
+    ) -> Result<String, Error> {
+        Ok(parse_url(api_base))
+    }
+
+    fn resolve_api_key(
+        &self,
+        api_key: Option<&str>,
+        env_lookup: &dyn Fn(&str) -> Option<String>,
+    ) -> Result<String, Error> {
+        resolve_api_key(api_key, env_lookup)
+    }
+}
+
+impl OcrProviderConfig for ReductoParseLegacyConfig {
+    fn supported_ocr_params(&self) -> &'static [&'static str] {
+        PARSE_LEGACY_SUPPORTED_OCR_PARAMS
+    }
+
+    fn transform_ocr_request(
+        &self,
+        _model: &str,
+        document: Value,
+        optional_params: Map<String, Value>,
+    ) -> Result<OcrRequestData, Error> {
+        let file_id = source_file_id(&document)?;
+        Ok(build_parse_legacy_request(&file_id, &optional_params))
+    }
+
+    fn transform_ocr_response(
+        &self,
+        model: &str,
+        response_json: Value,
+    ) -> Result<OcrResponseData, Error> {
+        transform_reducto_response(model, response_json)
+    }
+
+    fn complete_url(
+        &self,
+        api_base: Option<&str>,
+        _model: &str,
+        _optional_params: &Map<String, Value>,
+        _env_lookup: &dyn Fn(&str) -> Option<String>,
+    ) -> Result<String, Error> {
+        Ok(parse_url(api_base))
+    }
+
+    fn resolve_api_key(
+        &self,
+        api_key: Option<&str>,
+        env_lookup: &dyn Fn(&str) -> Option<String>,
+    ) -> Result<String, Error> {
+        resolve_api_key(api_key, env_lookup)
+    }
+}

litellm-rust/crates/core/src/providers/vertex_ai/ocr/transformation.rs

@@ -212,6 +212,7 @@ impl OcrProviderConfig for VertexAiOcrConfig {
         MISTRAL_OCR_CONFIG.supported_ocr_params()
     }
 
+    #[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)]
     fn transform_ocr_request(
         &self,
         model: &str,
@@ -229,6 +230,7 @@ impl OcrProviderConfig for VertexAiOcrConfig {
         MISTRAL_OCR_CONFIG.transform_ocr_response(model, response_json)
     }
 
+    #[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)]
     fn complete_url(
         &self,
         api_base: Option<&str>,
@@ -253,10 +255,21 @@ impl OcrProviderConfig for VertexAiOcrConfig {
 }
 
 impl OcrProviderConfig for VertexAiDeepSeekOcrConfig {
+    #[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)]
     fn supported_ocr_params(&self) -> &'static [&'static str] {
         DEEPSEEK_SUPPORTED_OCR_PARAMS
     }
 
+    #[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)]
+    fn map_ocr_params(&self, non_default_params: &Map<String, Value>) -> Map<String, Value> {
+        non_default_params
+            .iter()
+            .filter(|(name, _)| DEEPSEEK_SUPPORTED_OCR_PARAMS.contains(&name.as_str()))
+            .map(|(name, value)| (name.clone(), value.clone()))
+            .collect()
+    }
+
+    #[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)]
     fn transform_ocr_request(
         &self,
         model: &str,
@@ -283,6 +296,7 @@ impl OcrProviderConfig for VertexAiDeepSeekOcrConfig {
         })
     }
 
+    #[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)]
     fn transform_ocr_response(
         &self,
         model: &str,
@@ -335,9 +349,12 @@ impl OcrProviderConfig for VertexAiDeepSeekOcrConfig {
             document_annotation: object.get("document_annotation").cloned(),
             usage_info,
             object: "ocr".to_string(),
+            extra_fields: Map::new(),
+            provider_native_response: None,
         })
     }
 
+    #[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)]
     fn complete_url(
         &self,
         api_base: Option<&str>,
@@ -360,6 +377,7 @@ impl OcrProviderConfig for VertexAiDeepSeekOcrConfig {
 #[cfg(test)]
 mod tests {
     use super::*;
+    use rstest::rstest;
 
     #[test]
     fn vertex_mistral_url_uses_project_location_and_model() {
@@ -411,6 +429,22 @@ mod tests {
         );
     }
 
+    #[rstest]
+    #[case::bare_model("deepseek-ocr-maas")]
+    #[case::namespaced_model("deepseek-ai/deepseek-ocr-maas")]
+    fn vertex_deepseek_request_uses_single_provider_namespace(#[case] model: &str) {
+        let body = VERTEX_AI_DEEPSEEK_OCR_CONFIG
+            .transform_ocr_request(
+                model,
+                json!({"type": "image_url", "image_url": "data:image/png;base64,AA=="}),
+                Map::new(),
+            )
+            .expect("request transforms")
+            .data;
+
+        assert_eq!(body["model"], "deepseek-ai/deepseek-ocr-maas");
+    }
+
     #[test]
     fn vertex_deepseek_response_wraps_markdown_content() {
         let response = VERTEX_AI_DEEPSEEK_OCR_CONFIG

litellm-rust/crates/python-bridge/Cargo.toml

@@ -14,11 +14,15 @@ default = ["abi3"]
 abi3 = ["pyo3/abi3-py310"]
 extension-module = ["pyo3/extension-module"]
 panic-test = []
+trace-parity = [
+    "dep:tracing",
+    "litellm-core/observability",
+    "litellm-ai-gateway/trace-parity",
+]
 
 [dependencies]
 futures-util.workspace = true
-tracing.workspace = true
-tracing-subscriber.workspace = true
+tracing = { workspace = true, optional = true }
 litellm-core = { workspace = true, features = ["bedrock-auth"] }
 litellm-ai-gateway = { workspace = true, default-features = false }
 litellm-python-interop.workspace = true
@@ -31,6 +35,7 @@ tokio.workspace = true
 [dev-dependencies]
 criterion = "0.8.2"
 tokio-tungstenite.workspace = true
+tracing.workspace = true
 
 [[bench]]
 name = "serialization"

litellm-rust/crates/python-bridge/src/constants.rs

@@ -1 +0,0 @@
-pub(crate) const FUNCTION_TRACE_TARGET: &str = "litellm::function_trace";

litellm-rust/crates/python-bridge/src/errors.rs

@@ -59,3 +59,41 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> {
     module.add("RustBridgeDeclined", py.get_type::<RustBridgeDeclined>())?;
     module.add("RustUpstreamError", py.get_type::<RustUpstreamError>())
 }
+
+pub(crate) fn ocr_error_to_pyerr(err: Error) -> PyErr {
+    match err {
+        Error::MissingField("document_url" | "image_url") => {
+            PyValueError::new_err("Document URL is required")
+        }
+        Error::Http { status, body } => RustUpstreamError::new_err((status, body)),
+        other => core_error_to_pyerr(other),
+    }
+}
+
+#[cfg(test)]
+mod ocr_error_tests {
+    use super::*;
+
+    #[test]
+    fn ocr_errors_preserve_python_validation_and_provider_details() {
+        Python::initialize();
+        Python::attach(|py| {
+            for field in ["document_url", "image_url"] {
+                let mapped = ocr_error_to_pyerr(Error::MissingField(field));
+                assert!(mapped.is_instance_of::<PyValueError>(py));
+                assert_eq!(mapped.value(py).to_string(), "Document URL is required");
+            }
+            let mapped = ocr_error_to_pyerr(Error::Http {
+                status: 429,
+                body: r#"{"message":"rate limited"}"#.to_string(),
+            });
+            assert!(mapped.is_instance_of::<RustUpstreamError>(py));
+            let args: (u16, String) = mapped
+                .value(py)
+                .getattr("args")
+                .and_then(|args| args.extract())
+                .expect("OCR failures retain status and unprefixed provider message");
+            assert_eq!(args, (429, r#"{"message":"rate limited"}"#.to_string()));
+        });
+    }
+}

litellm-rust/crates/python-bridge/src/function_trace.rs

@@ -1,216 +1,22 @@
 use std::future::Future;
-use std::sync::{Arc, Mutex};
 
+use litellm_core::observability::{FunctionTrace, FunctionTraceEvent};
 use serde::Serialize;
 use tracing::instrument::WithSubscriber;
-use tracing::span::{Attributes, Id};
-use tracing::{Dispatch, Level, Subscriber};
-use tracing_subscriber::filter::{LevelFilter, filter_fn};
-use tracing_subscriber::layer::Context;
-use tracing_subscriber::prelude::*;
-use tracing_subscriber::registry::LookupSpan;
-use tracing_subscriber::{Layer, Registry};
-
-use crate::constants::FUNCTION_TRACE_TARGET;
 
 #[derive(Serialize)]
-#[serde(untagged)]
-pub(crate) enum TraceResponse<T> {
-    Plain(T),
-    Traced {
-        response: T,
-        trace: Vec<FunctionTraceEvent>,
-    },
+pub(crate) struct TracedResponse<T> {
+    response: T,
+    trace: Vec<FunctionTraceEvent>,
 }
 
-pub(crate) async fn trace_call<T, E>(
+pub(crate) async fn capture<T, E>(
     future: impl Future<Output = Result<T, E>>,
-    enabled: bool,
-) -> Result<TraceResponse<T>, E> {
-    if !enabled {
-        return future.await.map(TraceResponse::Plain);
-    }
+) -> Result<TracedResponse<T>, E> {
     let trace = FunctionTrace::default();
     let response = future.with_subscriber(trace.dispatcher()).await?;
-    Ok(TraceResponse::Traced {
+    Ok(TracedResponse {
         response,
         trace: trace.events(),
     })
 }
-
-#[derive(Clone, Debug, PartialEq, Serialize)]
-pub struct FunctionTraceEvent {
-    pub function: &'static str,
-    pub depth: usize,
-}
-
-#[derive(Clone, Default)]
-pub struct FunctionTrace {
-    events: Arc<Mutex<Vec<FunctionTraceEvent>>>,
-}
-
-impl FunctionTrace {
-    pub fn dispatcher(&self) -> Dispatch {
-        let filter = filter_fn(|metadata| {
-            metadata.is_span()
-                && metadata.target() == FUNCTION_TRACE_TARGET
-                && *metadata.level() == Level::TRACE
-        })
-        .with_max_level_hint(LevelFilter::TRACE);
-        Dispatch::new(
-            Registry::default().with(
-                FunctionTraceLayer {
-                    trace: self.clone(),
-                }
-                .with_filter(filter),
-            ),
-        )
-    }
-
-    pub fn events(&self) -> Vec<FunctionTraceEvent> {
-        self.events
-            .lock()
-            .unwrap_or_else(|error| error.into_inner())
-            .clone()
-    }
-}
-
-struct FunctionTraceLayer {
-    trace: FunctionTrace,
-}
-
-impl<S> Layer<S> for FunctionTraceLayer
-where
-    S: Subscriber + for<'lookup> LookupSpan<'lookup>,
-{
-    fn on_new_span(&self, attributes: &Attributes<'_>, id: &Id, context: Context<'_, S>) {
-        let depth = context
-            .span(id)
-            .map(|span| span.scope().skip(1).count())
-            .unwrap_or_default();
-        self.trace
-            .events
-            .lock()
-            .unwrap_or_else(|error| error.into_inner())
-            .push(FunctionTraceEvent {
-                function: attributes.metadata().name(),
-                depth,
-            });
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-
-    #[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)]
-    async fn outer() {
-        tokio::task::yield_now().await;
-        inner().await;
-    }
-
-    #[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)]
-    async fn inner() {
-        tokio::task::yield_now().await;
-    }
-
-    #[tokio::test]
-    async fn concurrent_futures_keep_separate_traces_across_yields() {
-        use tracing::instrument::WithSubscriber;
-
-        let first = FunctionTrace::default();
-        let second = FunctionTrace::default();
-        let outside = FunctionTrace::default();
-
-        async {
-            tokio::join!(
-                outer().with_subscriber(first.dispatcher()),
-                inner().with_subscriber(second.dispatcher()),
-            );
-            inner().await;
-        }
-        .with_subscriber(outside.dispatcher())
-        .await;
-
-        assert_eq!(
-            first.events(),
-            vec![
-                FunctionTraceEvent {
-                    function: "outer",
-                    depth: 0
-                },
-                FunctionTraceEvent {
-                    function: "inner",
-                    depth: 1
-                },
-            ],
-        );
-        assert_eq!(
-            second.events(),
-            vec![FunctionTraceEvent {
-                function: "inner",
-                depth: 0
-            }],
-        );
-        assert_eq!(
-            outside.events(),
-            vec![FunctionTraceEvent {
-                function: "inner",
-                depth: 0
-            }],
-        );
-    }
-
-    #[test]
-    fn records_matching_spans_in_creation_order() {
-        let trace = FunctionTrace::default();
-        let dispatch = trace.dispatcher();
-
-        tracing::dispatcher::with_default(&dispatch, || {
-            let _ignored = tracing::trace_span!(target: "other", "ignored");
-            let _first = tracing::trace_span!(target: FUNCTION_TRACE_TARGET, "same_name");
-            let _wrong_level = tracing::debug_span!(target: FUNCTION_TRACE_TARGET, "wrong_level");
-            let _second = tracing::trace_span!(target: FUNCTION_TRACE_TARGET, "same_name");
-        });
-
-        assert_eq!(
-            trace.events(),
-            vec![
-                FunctionTraceEvent {
-                    function: "same_name",
-                    depth: 0,
-                },
-                FunctionTraceEvent {
-                    function: "same_name",
-                    depth: 0,
-                },
-            ]
-        );
-    }
-
-    #[test]
-    fn records_matching_span_nesting_depth() {
-        let trace = FunctionTrace::default();
-        let dispatch = trace.dispatcher();
-
-        tracing::dispatcher::with_default(&dispatch, || {
-            let outer = tracing::trace_span!(target: FUNCTION_TRACE_TARGET, "outer");
-            let _outer_guard = outer.enter();
-            let _inner = tracing::trace_span!(target: FUNCTION_TRACE_TARGET, "inner");
-        });
-
-        assert_eq!(
-            trace.events(),
-            vec![
-                FunctionTraceEvent {
-                    function: "outer",
-                    depth: 0,
-                },
-                FunctionTraceEvent {
-                    function: "inner",
-                    depth: 1,
-                },
-            ]
-        );
-    }
-}

litellm-rust/crates/python-bridge/src/lib.rs

@@ -1,8 +1,8 @@
-mod constants;
 mod diagnostics;
 mod errors;
 mod execution;
-pub mod function_trace;
+#[cfg(feature = "trace-parity")]
+mod function_trace;
 mod marshal;
 mod routes;
 
@@ -115,9 +115,43 @@ mod tests {
                 .extract::<Vec<String>>()
                 .expect("module names should be strings")
                 .into_iter()
-                .filter(|name| !name.starts_with("__"))
+                .filter(|name| !name.starts_with('_'))
                 .collect();
             assert_eq!(public_names, expected);
+
+            #[cfg(not(feature = "trace-parity"))]
+            assert!(!module.hasattr("_trace").expect("module lookup should work"));
+
+            #[cfg(feature = "trace-parity")]
+            {
+                let trace = module
+                    .getattr("_trace")
+                    .expect("trace build should expose its diagnostic namespace");
+                let trace_names: Vec<String> = trace
+                    .cast::<PyModule>()
+                    .expect("trace namespace should be a module")
+                    .dict()
+                    .keys()
+                    .extract::<Vec<String>>()
+                    .expect("trace names should be strings")
+                    .into_iter()
+                    .filter(|name| !name.starts_with("__"))
+                    .collect();
+                assert_eq!(
+                    trace_names,
+                    [
+                        "ocr",
+                        "aocr",
+                        "transcription",
+                        "atranscription",
+                        "messages",
+                        "amessages",
+                        "chat_completions",
+                        "achat_completions",
+                        "gateway_messages",
+                    ]
+                );
+            }
         });
     }
 

litellm-rust/crates/python-bridge/src/routes/audio_transcription.rs

@@ -54,16 +54,16 @@ bridge_route! {
     required = {
         model: String,
         #[pyo3(from_py_with = litellm_python_interop::from_py)]
-        audio: Value,
+        audio: serde_json::Value,
     },
     optional = {
         api_key: Option<String>,
         api_base: Option<String>,
         custom_llm_provider: Option<String>,
         #[pyo3(from_py_with = litellm_python_interop::from_py)]
-        extra_headers: Option<Value>,
+        extra_headers: Option<serde_json::Value>,
         #[pyo3(from_py_with = litellm_python_interop::from_py)]
-        optional_params: Option<Value>,
+        optional_params: Option<serde_json::Value>,
         timeout_seconds: Option<f64>,
     },
     prepare = prepare_transcription,

litellm-rust/crates/python-bridge/src/routes/chat_completions.rs

@@ -73,16 +73,16 @@ bridge_route! {
     required = {
         model: String,
         #[pyo3(from_py_with = litellm_python_interop::from_py)]
-        messages: Value,
+        messages: serde_json::Value,
     },
     optional = {
         #[pyo3(from_py_with = litellm_python_interop::from_py)]
-        optional_params: Option<Value>,
+        optional_params: Option<serde_json::Value>,
         api_key: Option<String>,
         api_base: Option<String>,
         custom_llm_provider: Option<String>,
         #[pyo3(from_py_with = litellm_python_interop::from_py)]
-        extra_headers: Option<Value>,
+        extra_headers: Option<serde_json::Value>,
         timeout_seconds: Option<f64>,
     },
     prepare = prepare_chat_completions,

litellm-rust/crates/python-bridge/src/routes/definition.rs

@@ -20,43 +20,33 @@ macro_rules! bridge_route {
         }
 
         #[pyfunction]
-        #[pyo3(signature = ($($required_name),*, $($optional_name=None,)* trace=false))]
+        #[pyo3(signature = ($($required_name),*, $($optional_name=None),*))]
         #[allow(clippy::too_many_arguments)]
         fn $sync_name(
             py: pyo3::Python<'_>,
             $($(#[$required_attr])* $required_name: $required_type,)*
             $($(#[$optional_attr])* $optional_name: $optional_type,)*
-            trace: bool,
         ) -> pyo3::PyResult<pyo3::Py<pyo3::PyAny>> {
             let future = $prepare($inputs {
                 $($required_name,)*
                 $($optional_name),*
             })?;
-            $crate::execution::run_sync(
-                py,
-                $crate::function_trace::trace_call(future, trace),
-                $map_error,
-            )
+            $crate::execution::run_sync(py, future, $map_error)
         }
 
         #[pyfunction]
-        #[pyo3(signature = ($($required_name),*, $($optional_name=None,)* trace=false))]
+        #[pyo3(signature = ($($required_name),*, $($optional_name=None),*))]
         #[allow(clippy::too_many_arguments)]
         fn $async_name(
             py: pyo3::Python<'_>,
             $($(#[$required_attr])* $required_name: $required_type,)*
             $($(#[$optional_attr])* $optional_name: $optional_type,)*
-            trace: bool,
         ) -> pyo3::PyResult<pyo3::Bound<'_, pyo3::PyAny>> {
             let future = $prepare($inputs {
                 $($required_name,)*
                 $($optional_name),*
             })?;
-            $crate::execution::run_async(
-                py,
-                $crate::function_trace::trace_call(future, trace),
-                $map_error,
-            )
+            $crate::execution::run_async(py, future, $map_error)
         }
 
         pub(super) fn register(
@@ -67,6 +57,71 @@ macro_rules! bridge_route {
             $crate::routes::definition::add_function(module, pyo3::wrap_pyfunction!($async_name, module)?)?;
             Ok(())
         }
+
+        #[cfg(feature = "trace-parity")]
+        mod trace {
+            use pyo3::prelude::*;
+            use super::{$inputs, $map_error, $prepare};
+
+            #[pyfunction]
+            #[pyo3(signature = ($($required_name),*, $($optional_name=None),*))]
+            #[allow(clippy::too_many_arguments)]
+            fn $sync_name(
+                py: pyo3::Python<'_>,
+                $($(#[$required_attr])* $required_name: $required_type,)*
+                $($(#[$optional_attr])* $optional_name: $optional_type,)*
+            ) -> pyo3::PyResult<pyo3::Py<pyo3::PyAny>> {
+                let future = $prepare($inputs {
+                    $($required_name,)*
+                    $($optional_name),*
+                })?;
+                $crate::execution::run_sync(
+                    py,
+                    $crate::function_trace::capture(future),
+                    $map_error,
+                )
+            }
+
+            #[pyfunction]
+            #[pyo3(signature = ($($required_name),*, $($optional_name=None),*))]
+            #[allow(clippy::too_many_arguments)]
+            fn $async_name(
+                py: pyo3::Python<'_>,
+                $($(#[$required_attr])* $required_name: $required_type,)*
+                $($(#[$optional_attr])* $optional_name: $optional_type,)*
+            ) -> pyo3::PyResult<pyo3::Bound<'_, pyo3::PyAny>> {
+                let future = $prepare($inputs {
+                    $($required_name,)*
+                    $($optional_name),*
+                })?;
+                $crate::execution::run_async(
+                    py,
+                    $crate::function_trace::capture(future),
+                    $map_error,
+                )
+            }
+
+            pub(super) fn register(
+                module: &pyo3::Bound<'_, pyo3::types::PyModule>,
+            ) -> pyo3::PyResult<()> {
+                $crate::routes::definition::add_function(
+                    module,
+                    pyo3::wrap_pyfunction!($sync_name, module)?,
+                )?;
+                $crate::routes::definition::add_function(
+                    module,
+                    pyo3::wrap_pyfunction!($async_name, module)?,
+                )?;
+                Ok(())
+            }
+        }
+
+        #[cfg(feature = "trace-parity")]
+        pub(super) fn register_trace(
+            module: &pyo3::Bound<'_, pyo3::types::PyModule>,
+        ) -> pyo3::PyResult<()> {
+            trace::register(module)
+        }
     };
 }
 
@@ -130,20 +185,26 @@ mod tests {
         ) -> PyResult<impl Future<Output = Result<String, Error>> + Send + 'static> {
             FUTURE_DROPPED.store(false, Ordering::SeqCst);
             let drop_guard = (inputs.value == "pending").then_some(DropGuard);
-            Ok(async move {
-                let _drop_guard = drop_guard;
-                tokio::task::yield_now().await;
-                match inputs.value.as_str() {
-                    "error" => Err(Error::InvalidRequest("synthetic error".to_string())),
-                    "map_panic" => Err(Error::InvalidRequest("panic in mapper".to_string())),
-                    "panic" => panic!("synthetic panic"),
-                    "pending" => {
-                        pending::<()>().await;
-                        unreachable!()
-                    }
-                    _ => Ok(inputs.value),
+            Ok(execute_echo(inputs, drop_guard))
+        }
+
+        #[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)]
+        async fn execute_echo(
+            inputs: EchoInputs,
+            drop_guard: Option<DropGuard>,
+        ) -> Result<String, Error> {
+            let _drop_guard = drop_guard;
+            tokio::task::yield_now().await;
+            match inputs.value.as_str() {
+                "error" => Err(Error::InvalidRequest("synthetic error".to_string())),
+                "map_panic" => Err(Error::InvalidRequest("panic in mapper".to_string())),
+                "panic" => panic!("synthetic panic"),
+                "pending" => {
+                    pending::<()>().await;
+                    unreachable!()
                 }
-            })
+                _ => Ok(inputs.value),
+            }
         }
 
         fn map_error(error: Error) -> PyErr {
@@ -164,22 +225,22 @@ mod tests {
                 (
                     "ocr",
                     "aocr",
-                    "(model, document, api_key=None, api_base=None, custom_llm_provider=None, extra_headers=None, optional_params=None, timeout_seconds=None, trace=False)",
+                    "(model, document, api_key=None, api_base=None, custom_llm_provider=None, extra_headers=None, optional_params=None, timeout_seconds=None)",
                 ),
                 (
                     "transcription",
                     "atranscription",
-                    "(model, audio, api_key=None, api_base=None, custom_llm_provider=None, extra_headers=None, optional_params=None, timeout_seconds=None, trace=False)",
+                    "(model, audio, api_key=None, api_base=None, custom_llm_provider=None, extra_headers=None, optional_params=None, timeout_seconds=None)",
                 ),
                 (
                     "messages",
                     "amessages",
-                    "(model, body, api_key=None, api_base=None, custom_llm_provider=None, extra_headers=None, timeout_seconds=None, trace=False)",
+                    "(model, body, api_key=None, api_base=None, custom_llm_provider=None, extra_headers=None, timeout_seconds=None)",
                 ),
                 (
                     "chat_completions",
                     "achat_completions",
-                    "(model, messages, optional_params=None, api_key=None, api_base=None, custom_llm_provider=None, extra_headers=None, timeout_seconds=None, trace=False)",
+                    "(model, messages, optional_params=None, api_key=None, api_base=None, custom_llm_provider=None, extra_headers=None, timeout_seconds=None)",
                 ),
             ];
 
@@ -411,6 +472,32 @@ asyncio.run(exercise())
         });
     }
 
+    #[cfg(feature = "trace-parity")]
+    #[test]
+    fn diagnostic_route_returns_the_response_and_filtered_trace() {
+        Python::initialize();
+        Python::attach(|py| {
+            let module = PyModule::new(py, "synthetic").expect("module should be created");
+            synthetic::register_trace(&module).expect("trace routes should register");
+            let locals = PyDict::new(py);
+            locals
+                .set_item("routes", &module)
+                .expect("module should enter Python locals");
+            let code = CString::new(
+                r#"
+result = routes.echo("traced")
+assert result == {
+    "response": "traced",
+    "trace": [{"function": "execute_echo", "depth": 0}],
+}
+"#,
+            )
+            .expect("Python source should not contain null bytes");
+            py.run(&code, Some(&locals), Some(&locals))
+                .expect("diagnostic route should return its response and trace");
+        });
+    }
+
     #[test]
     fn route_registration_rejects_duplicate_python_names() {
         Python::initialize();

litellm-rust/crates/python-bridge/src/routes/gateway_messages.rs

@@ -0,0 +1,29 @@
+use pyo3::prelude::*;
+use serde_json::Value;
+
+use crate::errors::core_error_to_pyerr;
+
+#[pyfunction]
+fn gateway_messages<'py>(
+    py: Python<'py>,
+    model_alias: String,
+    provider_model: String,
+    api_base: String,
+    #[pyo3(from_py_with = litellm_python_interop::from_py)] body: Value,
+) -> PyResult<Bound<'py, PyAny>> {
+    let future = litellm_ai_gateway::trace_parity::messages_request(
+        model_alias,
+        provider_model,
+        api_base,
+        body,
+    );
+    crate::execution::run_async(
+        py,
+        crate::function_trace::capture(future),
+        core_error_to_pyerr,
+    )
+}
+
+pub(super) fn register_trace(module: &Bound<'_, PyModule>) -> PyResult<()> {
+    super::definition::add_function(module, wrap_pyfunction!(gateway_messages, module)?)
+}

litellm-rust/crates/python-bridge/src/routes/messages.rs

@@ -50,14 +50,14 @@ bridge_route! {
     required = {
         model: String,
         #[pyo3(from_py_with = litellm_python_interop::from_py)]
-        body: Value,
+        body: serde_json::Value,
     },
     optional = {
         api_key: Option<String>,
         api_base: Option<String>,
         custom_llm_provider: Option<String>,
         #[pyo3(from_py_with = litellm_python_interop::from_py)]
-        extra_headers: Option<Value>,
+        extra_headers: Option<serde_json::Value>,
         timeout_seconds: Option<f64>,
     },
     prepare = prepare_messages,

litellm-rust/crates/python-bridge/src/routes/mod.rs

@@ -3,6 +3,9 @@ use pyo3::prelude::*;
 #[macro_use]
 mod definition;
 
+#[cfg(feature = "trace-parity")]
+mod gateway_messages;
+
 mod audio_transcription;
 mod chat_completions;
 mod messages;
@@ -12,5 +15,16 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> {
     ocr::register(module)?;
     audio_transcription::register(module)?;
     messages::register(module)?;
-    chat_completions::register(module)
+    chat_completions::register(module)?;
+    #[cfg(feature = "trace-parity")]
+    {
+        let trace = PyModule::new(module.py(), "_trace")?;
+        ocr::register_trace(&trace)?;
+        audio_transcription::register_trace(&trace)?;
+        messages::register_trace(&trace)?;
+        chat_completions::register_trace(&trace)?;
+        gateway_messages::register_trace(&trace)?;
+        module.add_submodule(&trace)?;
+    }
+    Ok(())
 }

litellm-rust/crates/python-bridge/src/routes/ocr.rs

@@ -5,7 +5,7 @@ use litellm_ai_gateway::io::ocr::{OcrRequest, ocr as run_ocr};
 use pyo3::prelude::*;
 use serde_json::Value;
 
-use crate::errors::core_error_to_pyerr;
+use crate::errors::ocr_error_to_pyerr;
 use crate::marshal::{RouteOptions, RouteOptionsInputs, object_or_empty};
 
 fn prepare_ocr(
@@ -56,18 +56,18 @@ bridge_route! {
     required = {
         model: String,
         #[pyo3(from_py_with = litellm_python_interop::from_py)]
-        document: Value,
+        document: serde_json::Value,
     },
     optional = {
         api_key: Option<String>,
         api_base: Option<String>,
         custom_llm_provider: Option<String>,
         #[pyo3(from_py_with = litellm_python_interop::from_py)]
-        extra_headers: Option<Value>,
+        extra_headers: Option<serde_json::Value>,
         #[pyo3(from_py_with = litellm_python_interop::from_py)]
-        optional_params: Option<Value>,
+        optional_params: Option<serde_json::Value>,
         timeout_seconds: Option<f64>,
     },
     prepare = prepare_ocr,
-    errors = core_error_to_pyerr,
+    errors = ocr_error_to_pyerr,
 }

litellm-rust/crates/python-bridge/src/routes/runtime.rs

@@ -1,423 +0,0 @@
-use std::future::Future;
-use std::panic::AssertUnwindSafe;
-use std::time::Duration;
-
-use futures_util::FutureExt;
-use litellm_core::error::Error;
-use litellm_python_interop::{Pythonized, panic_to_pyerr, release_gil};
-use pyo3::exceptions::PyRuntimeError;
-use pyo3::prelude::*;
-use serde::Serialize;
-use tokio::runtime::{Handle, Runtime};
-use tokio::time::{self, MissedTickBehavior};
-
-pub(super) fn run_sync<T, F>(
-    py: Python<'_>,
-    future: F,
-    map_error: fn(Error) -> PyErr,
-) -> PyResult<Py<PyAny>>
-where
-    T: Serialize + Send + 'static,
-    F: Future<Output = Result<T, Error>> + Send + 'static,
-{
-    run_sync_on(
-        py,
-        pyo3_async_runtimes::tokio::get_runtime(),
-        future,
-        map_error,
-    )
-}
-
-fn run_sync_on<T, F>(
-    py: Python<'_>,
-    runtime: &Runtime,
-    future: F,
-    map_error: fn(Error) -> PyErr,
-) -> PyResult<Py<PyAny>>
-where
-    T: Serialize + Send + 'static,
-    F: Future<Output = Result<T, Error>> + Send + 'static,
-{
-    if Handle::try_current().is_ok() {
-        return Err(PyRuntimeError::new_err(
-            "synchronous native routes cannot run from a Tokio context; use the async route",
-        ));
-    }
-
-    let result = release_gil(py, move || runtime.block_on(wait_for_sync_result(future)))?;
-    let result = map_core_result(result, map_error)?;
-    Pythonized(result).into_pyobject(py).map(Bound::unbind)
-}
-
-pub(super) fn run_async<T, F>(
-    py: Python<'_>,
-    future: F,
-    map_error: fn(Error) -> PyErr,
-) -> PyResult<Bound<'_, PyAny>>
-where
-    T: Serialize + Send + 'static,
-    F: Future<Output = Result<T, Error>> + Send + 'static,
-{
-    pyo3_async_runtimes::tokio::future_into_py(py, async move {
-        let result = catch_route_panic(future).await?;
-        let result = map_core_result(result, map_error)?;
-        Ok(Pythonized(result))
-    })
-}
-
-fn map_core_result<T>(result: Result<T, Error>, map_error: fn(Error) -> PyErr) -> PyResult<T> {
-    match result {
-        Ok(value) => Ok(value),
-        Err(error) => Err(
-            std::panic::catch_unwind(AssertUnwindSafe(|| map_error(error)))
-                .map_err(panic_to_pyerr)?,
-        ),
-    }
-}
-
-async fn catch_route_panic<T, F>(future: F) -> PyResult<Result<T, Error>>
-where
-    F: Future<Output = Result<T, Error>>,
-{
-    AssertUnwindSafe(future)
-        .catch_unwind()
-        .await
-        .map_err(panic_to_pyerr)
-}
-
-async fn wait_for_sync_result<T, F>(future: F) -> PyResult<Result<T, Error>>
-where
-    F: Future<Output = Result<T, Error>>,
-{
-    let future = catch_route_panic(future);
-    tokio::pin!(future);
-
-    let signal_interval = Duration::from_millis(50);
-    let mut signal_checks =
-        time::interval_at(time::Instant::now() + signal_interval, signal_interval);
-    signal_checks.set_missed_tick_behavior(MissedTickBehavior::Delay);
-    loop {
-        tokio::select! {
-            result = &mut future => return result,
-            _ = signal_checks.tick() => Python::attach(|py| py.check_signals())?,
-        }
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use std::ffi::CString;
-    use std::future::poll_fn;
-    use std::sync::atomic::{AtomicUsize, Ordering};
-    use std::sync::{Arc, mpsc};
-    use std::task::Poll;
-    use std::thread;
-    use std::time::Instant;
-
-    use pyo3::panic::PanicException;
-    use pyo3::types::{PyDict, PyModule};
-    use serde::Serializer;
-    use tokio::runtime::Builder;
-
-    use super::*;
-
-    fn runtime_error(error: Error) -> PyErr {
-        PyRuntimeError::new_err(error.to_string())
-    }
-
-    fn panicking_error_mapper(_error: Error) -> PyErr {
-        panic!("error mapper panicked")
-    }
-
-    struct PanickingOutput;
-
-    static ASYNC_PROBE_COMPLETED: AtomicUsize = AtomicUsize::new(0);
-
-    impl Serialize for PanickingOutput {
-        fn serialize<S>(&self, _serializer: S) -> Result<S::Ok, S::Error>
-        where
-            S: Serializer,
-        {
-            panic!("serializer panicked")
-        }
-    }
-
-    #[pyfunction]
-    fn async_serialization_panic(py: Python<'_>) -> PyResult<Bound<'_, PyAny>> {
-        run_async(py, async { Ok(PanickingOutput) }, runtime_error)
-    }
-
-    #[pyfunction]
-    fn async_runtime_probe(py: Python<'_>) -> PyResult<Bound<'_, PyAny>> {
-        run_async(
-            py,
-            async {
-                ASYNC_PROBE_COMPLETED.fetch_add(1, Ordering::SeqCst);
-                Ok(true)
-            },
-            runtime_error,
-        )
-    }
-
-    #[pyfunction]
-    fn runtime_worker_count() -> usize {
-        pyo3_async_runtimes::tokio::get_runtime()
-            .metrics()
-            .num_workers()
-    }
-
-    #[pyfunction]
-    fn runtime_is_responsive(_py: Python<'_>, expected_completions: usize) -> bool {
-        let completion_deadline = Instant::now() + Duration::from_secs(2);
-        while ASYNC_PROBE_COMPLETED.load(Ordering::SeqCst) < expected_completions {
-            if Instant::now() >= completion_deadline {
-                return false;
-            }
-            thread::sleep(Duration::from_millis(1));
-        }
-
-        let (heartbeat_tx, heartbeat_rx) = mpsc::sync_channel(1);
-        pyo3_async_runtimes::tokio::get_runtime().spawn(async move {
-            let _ = heartbeat_tx.send(());
-        });
-        heartbeat_rx.recv_timeout(Duration::from_secs(2)).is_ok()
-    }
-
-    fn extract_bool(py: Python<'_>, result: PyResult<Py<PyAny>>) -> bool {
-        result
-            .expect("route should complete")
-            .bind(py)
-            .extract()
-            .expect("result should convert")
-    }
-
-    #[test]
-    fn sync_runner_polls_future_on_the_caller_thread() {
-        Python::initialize();
-        Python::attach(|py| {
-            let caller_thread = std::thread::current().id();
-            let result = run_sync(
-                py,
-                async move { Ok(std::thread::current().id() == caller_thread) },
-                runtime_error,
-            );
-
-            assert!(extract_bool(py, result));
-        });
-    }
-
-    #[test]
-    fn sync_runner_releases_gil_while_waiting() {
-        Python::initialize();
-        Python::attach(|py| {
-            let result = run_sync(
-                py,
-                async {
-                    let gil_acquired = tokio::time::timeout(
-                        Duration::from_secs(2),
-                        tokio::task::spawn_blocking(|| Python::attach(|_| true)),
-                    )
-                    .await;
-                    Ok(matches!(gil_acquired, Ok(Ok(true))))
-                },
-                runtime_error,
-            );
-
-            assert!(extract_bool(py, result));
-        });
-    }
-
-    #[test]
-    fn sync_runner_rejects_calls_from_a_tokio_context() {
-        Python::initialize();
-        let runtime = Builder::new_current_thread()
-            .enable_all()
-            .build()
-            .expect("runtime should build");
-
-        let error = runtime.block_on(async {
-            Python::attach(|py| {
-                run_sync::<bool, _>(py, async { Ok(true) }, runtime_error)
-                    .expect_err("sync route should reject a nested Tokio runtime")
-            })
-        });
-
-        assert_eq!(
-            error.to_string(),
-            "RuntimeError: synchronous native routes cannot run from a Tokio context; use the async route"
-        );
-    }
-
-    #[test]
-    fn sync_runner_can_drive_a_current_thread_runtime() {
-        Python::initialize();
-        let runtime = Builder::new_current_thread()
-            .enable_all()
-            .build()
-            .expect("runtime should build");
-        Python::attach(|py| {
-            let result = run_sync_on(
-                py,
-                &runtime,
-                async {
-                    tokio::task::yield_now().await;
-                    Ok(true)
-                },
-                runtime_error,
-            );
-            assert!(extract_bool(py, result));
-        });
-    }
-
-    #[test]
-    fn sync_runner_maps_a_panicked_future() {
-        Python::initialize();
-        Python::attach(|py| {
-            let error = run_sync::<bool, _>(
-                py,
-                poll_fn(|_| -> Poll<Result<bool, Error>> { panic!("route future panicked") }),
-                runtime_error,
-            )
-            .expect_err("panicked route should become a Python exception");
-
-            assert!(error.is_instance_of::<PanicException>(py));
-            assert_eq!(error.to_string(), "PanicException: route future panicked");
-        });
-    }
-
-    #[test]
-    fn sync_runner_maps_a_panicked_error_mapper() {
-        Python::initialize();
-        Python::attach(|py| {
-            let error = run_sync::<bool, _>(
-                py,
-                async { Err(Error::InvalidRequest("invalid".to_string())) },
-                panicking_error_mapper,
-            )
-            .expect_err("panicked mapper should become a Python exception");
-
-            assert!(error.is_instance_of::<PanicException>(py));
-            assert_eq!(error.to_string(), "PanicException: error mapper panicked");
-        });
-    }
-
-    #[test]
-    fn sync_runner_surfaces_serializer_panics() {
-        Python::initialize();
-        Python::attach(|py| {
-            let error = run_sync(py, async { Ok(PanickingOutput) }, runtime_error)
-                .expect_err("serializer panic should become a Python exception");
-
-            assert!(error.is_instance_of::<PanicException>(py));
-            assert_eq!(error.to_string(), "PanicException: serializer panicked");
-        });
-    }
-
-    #[test]
-    fn sync_runner_supports_concurrent_callers_on_the_shared_runtime() {
-        Python::initialize();
-        let barrier = Arc::new(tokio::sync::Barrier::new(2));
-        let callers: Vec<_> = (0..2)
-            .map(|_| {
-                let barrier = Arc::clone(&barrier);
-                thread::spawn(move || {
-                    Python::attach(|py| {
-                        extract_bool(
-                            py,
-                            run_sync(
-                                py,
-                                async move {
-                                    Ok(tokio::time::timeout(Duration::from_secs(2), barrier.wait())
-                                        .await
-                                        .is_ok())
-                                },
-                                runtime_error,
-                            ),
-                        )
-                    })
-                })
-            })
-            .collect();
-        let results: Vec<_> = callers
-            .into_iter()
-            .map(|caller| caller.join().expect("caller should not panic"))
-            .collect();
-
-        assert_eq!(results, vec![true, true]);
-    }
-
-    #[test]
-    fn async_runner_surfaces_serializer_panics() {
-        Python::initialize();
-        Python::attach(|py| {
-            let module = PyModule::new(py, "runtime").expect("module should be created");
-            module
-                .add_function(
-                    wrap_pyfunction!(async_serialization_panic, &module)
-                        .expect("function should wrap"),
-                )
-                .expect("function should register");
-            let locals = PyDict::new(py);
-            locals
-                .set_item("runtime", &module)
-                .expect("module should enter Python locals");
-            let code = CString::new(
-                r#"
-import asyncio
-
-async def exercise():
-    try:
-        await runtime.async_serialization_panic()
-    except BaseException as error:
-        assert type(error).__name__ == "PanicException"
-        assert str(error) == "serializer panicked"
-    else:
-        raise AssertionError("serializer panic was not raised")
-
-asyncio.run(exercise())
-"#,
-            )
-            .expect("Python source should not contain null bytes");
-            py.run(&code, Some(&locals), Some(&locals))
-                .expect("serializer panic should reach the Python awaiter");
-        });
-    }
-
-    #[test]
-    fn async_result_delivery_does_not_stall_tokio_workers() {
-        Python::initialize();
-        ASYNC_PROBE_COMPLETED.store(0, Ordering::SeqCst);
-        Python::attach(|py| {
-            let module = PyModule::new(py, "runtime").expect("module should be created");
-            for function in [
-                wrap_pyfunction!(async_runtime_probe, &module).expect("function should wrap"),
-                wrap_pyfunction!(runtime_worker_count, &module).expect("function should wrap"),
-                wrap_pyfunction!(runtime_is_responsive, &module).expect("function should wrap"),
-            ] {
-                module
-                    .add_function(function)
-                    .expect("function should register");
-            }
-            let locals = PyDict::new(py);
-            locals
-                .set_item("runtime", &module)
-                .expect("module should enter Python locals");
-            let code = CString::new(
-                r#"
-import asyncio
-
-async def exercise():
-    worker_count = runtime.runtime_worker_count()
-    awaitables = [runtime.async_runtime_probe() for _ in range(worker_count)]
-    assert runtime.runtime_is_responsive(worker_count)
-    assert await asyncio.gather(*awaitables) == [True] * worker_count
-
-asyncio.run(exercise())
-"#,
-            )
-            .expect("Python source should not contain null bytes");
-            py.run(&code, Some(&locals), Some(&locals))
-                .expect("result delivery should leave Tokio workers responsive");
-        });
-    }
-}

litellm/_service_logger.py

@@ -1,4 +1,5 @@
 import asyncio
+from collections.abc import Callable, Coroutine
 from datetime import datetime, timedelta
 from typing import TYPE_CHECKING, Any, Final
 
@@ -83,6 +84,47 @@ class ServiceLogging(CustomLogger):
                 return open_telemetry_logger
         return None
 
+    @staticmethod
+    def _sync_dispatch_loop() -> asyncio.AbstractEventLoop | None:
+        """The event loop a blocking caller can dispatch on, or ``None`` if it has none."""
+        try:
+            loop: Final = asyncio.get_event_loop()
+        except RuntimeError:
+            return None
+        return None if loop.is_closed() else loop
+
+    @staticmethod
+    async def _emit_guarded(hook: Callable[[], Coroutine[object, object, None]]) -> None:
+        """Emit one service event, absorbing anything the callbacks raise.
+
+        Monitoring must not break the call it monitors. Sync callers are the ones that
+        swallow their own service failures (a Redis batch read returns an empty dict),
+        so an exception from a misconfigured callback would replace a Redis outage with
+        a callback error and skip the caller's fallback handling.
+        """
+        try:
+            await hook()
+        except Exception as e:
+            verbose_logger.exception("Error emitting service event - %s", e)
+
+    @staticmethod
+    def _dispatch_from_sync(hook: Callable[[], Coroutine[object, object, None]]) -> None:
+        """Run an async service hook from a blocking caller, whatever event loop it holds.
+
+        Takes a factory rather than a coroutine so the hook is built on the path that
+        runs it, and only ever once.
+        """
+        loop: Final = ServiceLogging._sync_dispatch_loop()
+        try:
+            if loop is None:
+                asyncio.run(ServiceLogging._emit_guarded(hook))
+            elif loop.is_running():
+                loop.create_task(ServiceLogging._emit_guarded(hook))
+            else:
+                loop.run_until_complete(ServiceLogging._emit_guarded(hook))
+        except Exception as e:
+            verbose_logger.exception("Error dispatching service event - %s", e)
+
     def service_success_hook(
         self,
         service: ServiceTypes,
@@ -99,54 +141,45 @@ class ServiceLogging(CustomLogger):
         if self.mock_testing:
             self.mock_testing_sync_success_hook += 1
 
-        try:
-            # Try to get the current event loop
-            loop: Final = asyncio.get_event_loop()
-            # Check if the loop is running
-            if loop.is_running():
-                # If we're in a running loop, create a task
-                loop.create_task(
-                    self.async_service_success_hook(
-                        service=service,
-                        duration=duration,
-                        call_type=call_type,
-                        parent_otel_span=parent_otel_span,
-                        start_time=start_time,
-                        end_time=end_time,
-                    )
-                )
-            else:
-                # Loop exists but not running, we can use run_until_complete
-                loop.run_until_complete(
-                    self.async_service_success_hook(
-                        service=service,
-                        duration=duration,
-                        call_type=call_type,
-                        parent_otel_span=parent_otel_span,
-                        start_time=start_time,
-                        end_time=end_time,
-                    )
-                )
-        except RuntimeError:
-            # No event loop exists, create a new one and run
-            asyncio.run(
-                self.async_service_success_hook(
-                    service=service,
-                    duration=duration,
-                    call_type=call_type,
-                    parent_otel_span=parent_otel_span,
-                    start_time=start_time,
-                    end_time=end_time,
-                )
+        self._dispatch_from_sync(
+            lambda: self.async_service_success_hook(
+                service=service,
+                duration=duration,
+                call_type=call_type,
+                parent_otel_span=parent_otel_span,
+                start_time=start_time,
+                end_time=end_time,
             )
+        )
 
-    def service_failure_hook(self, service: ServiceTypes, duration: float, error: Exception, call_type: str):
+    def service_failure_hook(
+        self,
+        service: ServiceTypes,
+        duration: float,
+        error: Exception,
+        call_type: str,
+        parent_otel_span: Span | None = None,
+        start_time: datetime | float | None = None,
+        end_time: float | datetime | None = None,
+    ):
         """
-        [TODO] Not implemented for sync calls yet. V0 is focused on async monitoring (used by proxy).
+        Handles both sync and async monitoring by checking for existing event loop.
         """
         if self.mock_testing:
             self.mock_testing_sync_failure_hook += 1
 
+        self._dispatch_from_sync(
+            lambda: self.async_service_failure_hook(
+                service=service,
+                duration=duration,
+                error=error,
+                call_type=call_type,
+                parent_otel_span=parent_otel_span,
+                start_time=start_time,
+                end_time=end_time,
+            )
+        )
+
     async def async_service_success_hook(
         self,
         service: ServiceTypes,

litellm/anthropic_interface/exceptions/exceptions.py

@@ -1,8 +1,9 @@
 """Anthropic error format type definitions."""
 
+from collections.abc import Mapping
 from typing import Literal
 
-from typing_extensions import Required, TypedDict
+from typing_extensions import NotRequired, ReadOnly, Required, TypedDict
 
 # Known Anthropic error types
 # Source: https://docs.anthropic.com/en/api/errors
@@ -23,6 +24,7 @@ class AnthropicErrorDetail(TypedDict):
 
     type: AnthropicErrorType
     message: str
+    provider_specific_fields: NotRequired[ReadOnly[Mapping[str, object]]]
 
 
 class AnthropicErrorResponse(TypedDict, total=False):

litellm/caching/caching.py

@@ -100,6 +100,7 @@ class Cache:
         qdrant_semantic_cache_vector_size: int | None = None,
         semantic_cache_embedding_max_input_tokens: int | None = None,
         semantic_cache_embedding_timeout: float | None = None,
+        semantic_cache_scope: str = SemanticCacheScope.KEY.value,
         # GCP IAM authentication parameters
         gcp_service_account: str | None = None,
         gcp_ssl_ca_certs: str | None = None,
@@ -127,6 +128,7 @@ class Cache:
             similarity_threshold (float, optional): The similarity threshold for semantic-caching, Required if type is "redis-semantic" or "qdrant-semantic".
             semantic_cache_embedding_max_input_tokens (int, optional): Truncate prompts to this many tokens before embedding them for semantic caching. Defaults to the embedding deployment's configured max_input_tokens.
             semantic_cache_embedding_timeout (float, optional): Seconds a semantic-cache lookup may spend embedding the prompt before it gives up and lets the request continue to the LLM. Defaults to SEMANTIC_CACHE_EMBEDDING_TIMEOUT_SECONDS.
+            semantic_cache_scope (str, optional): "key" isolates semantic-cache buckets per key/team/org. "end_user" additionally isolates per end user (falls back to the key scope when the request carries no end-user id). Defaults to "key".
 
             # Disk Cache Args
             disk_cache_dir (str, optional): The directory for the disk cache. Defaults to None.
@@ -274,6 +276,7 @@ class Cache:
         self.redis_flush_size = redis_flush_size
         self.ttl = ttl
         self.mode: CacheMode = mode or CacheMode.default_on
+        self.semantic_cache_scope: str = SemanticCacheScope(semantic_cache_scope).value
 
         if self.type == LiteLLMCacheType.LOCAL and default_in_memory_ttl is not None:
             self.ttl = default_in_memory_ttl
@@ -301,6 +304,7 @@ class Cache:
         "user_api_key_team_id",
         "user_api_key_org_id",
     )
+    _SEMANTIC_CACHE_END_USER_SCOPE_FIELD: Final = "user_api_key_end_user_id"
 
     def _is_semantic_cache(self) -> bool:
         return self.type in (
@@ -309,19 +313,21 @@ class Cache:
             LiteLLMCacheType.VALKEY_SEMANTIC,
         )
 
-    def _get_semantic_cache_tenant_scope(self, kwargs: dict) -> str:
-        metadata: Final[dict] = kwargs.get("metadata") or {}
-        litellm_params: Final[dict] = kwargs.get("litellm_params") or {}
-        metadata_in_litellm_params: Final[dict] = litellm_params.get("metadata") or {}
+    def _semantic_cache_scope_fields(self) -> tuple[str, ...]:
+        if self.semantic_cache_scope == SemanticCacheScope.END_USER:
+            return (*self._SEMANTIC_CACHE_TENANT_SCOPE_FIELDS, self._SEMANTIC_CACHE_END_USER_SCOPE_FIELD)
+        return self._SEMANTIC_CACHE_TENANT_SCOPE_FIELDS
 
-        scope = ""
-        for field in self._SEMANTIC_CACHE_TENANT_SCOPE_FIELDS:
-            value = metadata.get(field)
-            if value is None:
-                value = metadata_in_litellm_params.get(field)
-            if value is not None:
-                scope += f"{field}: {value}"
-        return scope
+    def _get_semantic_cache_tenant_scope(self, kwargs: dict) -> str:
+        litellm_params: Final[dict] = kwargs.get("litellm_params") or {}
+        metadata_sources: Final[tuple[dict, ...]] = tuple(
+            source.get(key) or {} for source in (kwargs, litellm_params) for key in ("metadata", "litellm_metadata")
+        )
+        scope_values: Final = (
+            (field, next((source[field] for source in metadata_sources if source.get(field) is not None), None))
+            for field in self._semantic_cache_scope_fields()
+        )
+        return "".join(f"{field}: {value}" for field, value in scope_values if value is not None)
 
     def get_cache_key(self, **kwargs) -> str:
         """

litellm/caching/dual_cache.py

@@ -8,10 +8,9 @@ Has 4 primary methods:
     - async_get_cache
 """
 
-import asyncio
 import time
 import traceback
-from concurrent.futures import ThreadPoolExecutor
+from collections.abc import Sequence
 from threading import Lock
 from typing import TYPE_CHECKING, Any, Final
 
@@ -188,31 +187,38 @@ class DualCache(BaseCache):
         local_only: bool = False,
         **kwargs,
     ):
-        received_args: Final = locals()
-        received_args.pop("self")
-
-        def run_in_new_loop():
-            """Run the coroutine in a new event loop within this thread."""
-            new_loop: Final = asyncio.new_event_loop()
-            try:
-                asyncio.set_event_loop(new_loop)
-                return new_loop.run_until_complete(self.async_batch_get_cache(**received_args))
-            finally:
-                new_loop.close()
-                asyncio.set_event_loop(None)
-
         try:
-            # First, try to get the current event loop
-            _ = asyncio.get_running_loop()
-            # If we're already in an event loop, run in a separate thread
-            # to avoid nested event loop issues
-            with ThreadPoolExecutor(max_workers=1) as executor:
-                future: Final = executor.submit(run_in_new_loop)
-                return future.result()
+            in_memory_result: Final = (
+                self.in_memory_cache.batch_get_cache(keys, **kwargs) if self.in_memory_cache is not None else None
+            )
+            result: Final = in_memory_result if in_memory_result is not None else tuple(None for _ in keys)
 
-        except RuntimeError:
-            # No running event loop, we can safely run in this thread
-            return run_in_new_loop()
+            if None not in result or self.redis_cache is None or local_only:
+                return result
+
+            sublist_keys, previous_access_times = self._reserve_redis_batch_keys(time.time(), keys, result)
+            if len(sublist_keys) == 0:
+                return result
+
+            try:
+                redis_result: Final = self.redis_cache.batch_get_cache(
+                    key_list=sublist_keys, parent_otel_span=parent_otel_span
+                )
+            except Exception:
+                # Do not throttle subsequent callers if the Redis read fails.
+                self._rollback_redis_batch_key_reservations(previous_access_times)
+                raise
+
+            if self.in_memory_cache is not None:
+                for key, value in redis_result.items():
+                    if value is not None:
+                        self.in_memory_cache.set_cache(key, value, **self._backfill_kwargs(kwargs))
+
+            return list(  # mutable-ok: public list contract
+                redis_result.get(key) if value is None else value for key, value in zip(keys, result)
+            )
+        except Exception:
+            verbose_logger.error(traceback.format_exc())
 
     async def async_get_cache(
         self,
@@ -251,7 +257,7 @@ class DualCache(BaseCache):
         self,
         current_time: float,
         keys: list[str],
-        result: list[Any],
+        result: Sequence[Any],
     ) -> tuple[list[str], dict[str, float | None]]:
         """
         Atomically choose keys to fetch from Redis and reserve their access time.

litellm/caching/redis_cache.py

@@ -27,6 +27,7 @@ from litellm.constants import (
     REDIS_CIRCUIT_BREAKER_ENABLED,
     REDIS_CIRCUIT_BREAKER_FAILURE_THRESHOLD,
     REDIS_CIRCUIT_BREAKER_RECOVERY_TIMEOUT,
+    REDIS_CIRCUIT_BREAKER_TIMEOUT_MIN_DURATION,
 )
 from litellm.litellm_core_utils.core_helpers import _get_parent_otel_span_from_kwargs
 from litellm.litellm_core_utils.coroutine_checker import coroutine_checker
@@ -41,6 +42,8 @@ from .base_cache import BaseCache
 
 if TYPE_CHECKING:
     from opentelemetry.trace import Span as _Span
+    from prometheus_client import Counter as _PromCounter
+    from prometheus_client import Gauge as _PromGauge
     from redis.asyncio import Redis, RedisCluster
     from redis.asyncio.client import Pipeline
     from redis.asyncio.cluster import ClusterPipeline
@@ -78,10 +81,18 @@ class _AsyncRedisCommands(Protocol):
     def pipeline(self, transaction: bool = True) -> "Pipeline[bytes]": ...
 
 
+_BREAKER_GUARD_FRAME_NAMES: Final = frozenset(
+    {"<lambda>", "wrapper", "_run_under_circuit_breaker", "_run_under_circuit_breaker_sync"}
+)
+
+
 def _get_call_stack_info(num_frames: int = 2) -> str:
     """
     Get the function names from the previous 1-2 functions in the call stack.
 
+    Frames belonging to this module's circuit-breaker guards are skipped so the
+    reported callers stay the real ones even on guarded methods.
+
     Args:
         num_frames: Number of previous frames to include (default: 2)
 
@@ -102,11 +113,11 @@ def _get_call_stack_info(num_frames: int = 2) -> str:
             return "unknown"
         function_names: Final = []
 
-        for _ in range(num_frames):
-            if frame is None:
-                break
-            func_name = frame.f_code.co_name
-            function_names.append(func_name)
+        while frame is not None and len(function_names) < num_frames:
+            if frame.f_code.co_name in _BREAKER_GUARD_FRAME_NAMES and frame.f_globals.get("__name__") == __name__:
+                frame = frame.f_back
+                continue
+            function_names.append(frame.f_code.co_name)
             frame = frame.f_back
 
         if not function_names:
@@ -127,10 +138,20 @@ class RedisCircuitBreaker:
       HALF_OPEN - recovery probe: allow one request through
 
     Transitions:
-      CLOSED    -> OPEN      after failure_threshold consecutive failures
+      CLOSED    -> OPEN      after failure_threshold consecutive hard connectivity
+                             failures, or after an unbroken run of timeout failures
+                             (no success or hard failure in between) that reaches
+                             failure_threshold and spans timeout_min_duration seconds
       OPEN      -> HALF_OPEN after recovery_timeout seconds
       HALF_OPEN -> CLOSED    on success
       HALF_OPEN -> OPEN      on failure (resets timer)
+
+    Timeouts are accounted separately from hard connectivity failures because the async
+    Redis timeout includes time waiting for the worker event loop to resume: one loop
+    stall makes every in-flight operation time out together, which satisfies a purely
+    consecutive threshold instantly even though Redis is healthy. Requiring a
+    timeout-only streak to also span timeout_min_duration filters such bursts while a
+    real outage that surfaces as timeouts still opens the breaker after that duration.
     """
 
     CLOSED = "closed"
@@ -142,13 +163,19 @@ class RedisCircuitBreaker:
         failure_threshold: int,
         recovery_timeout: int,
         enabled: bool = True,
+        timeout_min_duration: float = REDIS_CIRCUIT_BREAKER_TIMEOUT_MIN_DURATION,
     ) -> None:
         self.failure_threshold = failure_threshold
         self.recovery_timeout = recovery_timeout
         self.enabled = enabled
+        self.timeout_min_duration = timeout_min_duration
         self._failure_count = 0
+        self._hard_failure_count = 0
+        self._timeout_count = 0
+        self._timeout_streak_started_at: float | None = None
         self._opened_at: float | None = None
         self._state = self.CLOSED
+        _breaker_metrics().record_state_change(None, self._state)
 
     def is_open(self) -> bool:
         """Returns True if Redis calls should be skipped."""
@@ -161,24 +188,45 @@ class RedisCircuitBreaker:
             return True
         if self._state == self.OPEN:
             if time.time() - (self._opened_at or 0) > self.recovery_timeout:
-                self._state = self.HALF_OPEN
+                self._set_state(self.HALF_OPEN)
                 return False  # this caller is the designated probe
             return True
         return False
 
-    def record_failure(self) -> None:
+    def _should_open(self, now: float) -> bool:
+        if self._state == self.HALF_OPEN:
+            return True
+        if self._hard_failure_count >= self.failure_threshold:
+            return True
+        if self._timeout_count < self.failure_threshold:
+            return False
+        return now - (self._timeout_streak_started_at or now) >= self.timeout_min_duration
+
+    def record_failure(self, is_timeout: bool = False) -> None:
         if not self.enabled:
             return
+        now: Final = time.time()
         self._failure_count += 1
-        self._opened_at = time.time()
-        if self._failure_count >= self.failure_threshold:
+        if is_timeout:
+            self._timeout_count += 1
+            if self._timeout_streak_started_at is None:
+                self._timeout_streak_started_at = now
+        else:
+            self._hard_failure_count += 1
+            self._timeout_count = 0
+            self._timeout_streak_started_at = None
+        self._opened_at = now
+        _breaker_metrics().record_failure("timeout" if is_timeout else "connectivity")
+        if self._should_open(now):
             if self._state != self.OPEN:
                 verbose_logger.warning(
-                    "Redis circuit breaker OPENED after %d consecutive failures — fast-failing Redis calls for %ds",
+                    "Redis circuit breaker OPENED after %d consecutive failures"
+                    " (%d hard connectivity) — fast-failing Redis calls for %ds",
                     self._failure_count,
+                    self._hard_failure_count,
                     self.recovery_timeout,
                 )
-            self._state = self.OPEN
+            self._set_state(self.OPEN)
 
     def record_success(self) -> None:
         if not self.enabled:
@@ -186,7 +234,17 @@ class RedisCircuitBreaker:
         if self._state == self.HALF_OPEN:
             verbose_logger.info("Redis circuit breaker CLOSED — Redis recovered")
         self._failure_count = 0
-        self._state = self.CLOSED
+        self._hard_failure_count = 0
+        self._timeout_count = 0
+        self._timeout_streak_started_at = None
+        self._set_state(self.CLOSED)
+
+    def _set_state(self, state: str) -> None:
+        if state == self._state:
+            return
+        _breaker_metrics().record_transition(state)
+        _breaker_metrics().record_state_change(self._state, state)
+        self._state = state
 
 
 _RedisCallResult = TypeVar("_RedisCallResult")
@@ -226,6 +284,78 @@ def _is_redis_health_failure(exc: BaseException) -> bool:
         return True
 
 
+@functools.lru_cache(maxsize=1)
+def _redis_timeout_error_types() -> tuple[type, ...]:
+    """Health failures that are timeouts rather than unambiguous connectivity errors.
+
+    ``builtins.TimeoutError`` covers ``asyncio.TimeoutError`` and ``socket.timeout``
+    (aliases since py3.11 / py3.10). ``redis.exceptions.TimeoutError`` does not subclass
+    either, so it is listed explicitly.
+    """
+    try:
+        from redis.exceptions import TimeoutError as RedisTimeoutError
+    except ImportError:
+        return (TimeoutError,)
+    return (RedisTimeoutError, TimeoutError)
+
+
+def _is_redis_timeout_failure(exc: BaseException) -> bool:
+    return isinstance(exc, _redis_timeout_error_types())
+
+
+class _BreakerMetrics:
+    """Prometheus metrics for the Redis circuit breaker; no-ops when the client is absent.
+
+    Registered lazily on the default registry (which /metrics serves) via the module-level
+    ``_breaker_metrics`` singleton so repeated RedisCache construction never re-registers.
+    """
+
+    def __init__(self) -> None:
+        self._state_gauge: _PromGauge | None = None
+        self._transitions: _PromCounter | None = None
+        self._failures: _PromCounter | None = None
+        try:
+            from prometheus_client import Counter as PromCounter
+            from prometheus_client import Gauge
+        except ImportError:
+            return
+        self._state_gauge = Gauge(
+            "litellm_redis_circuit_breaker_state",
+            "Number of Redis circuit breakers currently in each state",
+            labelnames=("state",),
+        )
+        self._transitions = PromCounter(
+            "litellm_redis_circuit_breaker_transitions",
+            "Redis circuit breaker state transitions",
+            labelnames=("state",),
+        )
+        self._failures = PromCounter(
+            "litellm_redis_circuit_breaker_failures",
+            "Redis health failures counted by the circuit breaker",
+            labelnames=("failure_class",),
+        )
+
+    def record_state_change(self, old_state: str | None, new_state: str) -> None:
+        if self._state_gauge is None:
+            return
+        if old_state is not None:
+            self._state_gauge.labels(old_state).dec()
+        self._state_gauge.labels(new_state).inc()
+
+    def record_transition(self, state: str) -> None:
+        if self._transitions is not None:
+            self._transitions.labels(state).inc()
+
+    def record_failure(self, failure_class: str) -> None:
+        if self._failures is not None:
+            self._failures.labels(failure_class).inc()
+
+
+@functools.lru_cache(maxsize=1)
+def _breaker_metrics() -> _BreakerMetrics:
+    return _BreakerMetrics()
+
+
 def _record_swallowed_redis_failure(breaker: RedisCircuitBreaker, exc: BaseException) -> None:
     """Record a Redis failure that the calling method is about to swallow.
 
@@ -237,10 +367,27 @@ def _record_swallowed_redis_failure(breaker: RedisCircuitBreaker, exc: BaseExcep
     """
     if not _is_redis_health_failure(exc):
         return
-    breaker.record_failure()
+    breaker.record_failure(is_timeout=_is_redis_timeout_failure(exc))
     _swallowed_redis_failures.set(_swallowed_redis_failures.get() + 1)
 
 
+def _enter_circuit_breaker(breaker: RedisCircuitBreaker, name: str) -> int:
+    """Reject the call if the breaker is open, else return the swallowed-failure count to compare against."""
+    if breaker.is_open():
+        raise Exception(f"Redis circuit breaker is open — skipping {name}")
+    return _swallowed_redis_failures.get()
+
+
+def _exit_circuit_breaker(breaker: RedisCircuitBreaker, swallowed_before: int) -> None:
+    """Record success only when nothing failed while the call ran.
+
+    Several Redis methods catch their own connection errors and return a default, so a
+    method that returned is not on its own proof of a healthy Redis.
+    """
+    if _swallowed_redis_failures.get() == swallowed_before:
+        breaker.record_success()
+
+
 async def _run_under_circuit_breaker(
     breaker: RedisCircuitBreaker,
     name: str,
@@ -249,20 +396,33 @@ async def _run_under_circuit_breaker(
     """Run one Redis coroutine under a circuit breaker.
 
     Shared by the method decorator and the Lua script executor so both feed the same
-    health signal. Success is recorded only when nothing failed while ``call`` ran,
-    because several Redis methods catch their own connection errors and return a default.
+    health signal.
     """
-    if breaker.is_open():
-        raise Exception(f"Redis circuit breaker is open — skipping {name}")
-    swallowed_before: Final = _swallowed_redis_failures.get()
+    swallowed_before: Final = _enter_circuit_breaker(breaker, name)
     try:
         result: Final = await call()
+    except Exception as e:
+        if _is_redis_health_failure(e):
+            breaker.record_failure(is_timeout=_is_redis_timeout_failure(e))
+        raise
+    _exit_circuit_breaker(breaker, swallowed_before)
+    return result
+
+
+def _run_under_circuit_breaker_sync(
+    breaker: RedisCircuitBreaker,
+    name: str,
+    call: Callable[[], _RedisCallResult],
+) -> _RedisCallResult:
+    """Run one blocking Redis call under a circuit breaker, feeding the same health signal as the async path."""
+    swallowed_before: Final = _enter_circuit_breaker(breaker, name)
+    try:
+        result: Final = call()
     except Exception as e:
         if _is_redis_health_failure(e):
             breaker.record_failure()
         raise
-    if _swallowed_redis_failures.get() == swallowed_before:
-        breaker.record_success()
+    _exit_circuit_breaker(breaker, swallowed_before)
     return result
 
 
@@ -288,6 +448,14 @@ def _redis_circuit_breaker_guard(method):
     return wrapper
 
 
+def _redis_circuit_breaker_guard_sync(method: Callable[..., _RedisCallResult]) -> Callable[..., _RedisCallResult]:
+    return functools.wraps(method)(
+        lambda self, *args, **kwargs: _run_under_circuit_breaker_sync(
+            self._circuit_breaker, method.__name__, lambda: method(self, *args, **kwargs)
+        )
+    )
+
+
 class RedisCache(BaseCache):
     # if users don't provider one, use the default litellm cache
 
@@ -1146,14 +1314,13 @@ class RedisCache(BaseCache):
         """
         key_value_dict = {}
         _key_list: Final = [key for key in key_list if key is not None]
+        start_time: Final = time.time()
 
         try:
-            _keys: Final = []
-            for cache_key in _key_list:
-                cache_key = self.check_and_fix_namespace(key=cache_key or "")
-                _keys.append(cache_key)
-            start_time: Final = time.time()
+            swallowed_before: Final = _enter_circuit_breaker(self._circuit_breaker, "batch_get_cache")
+            _keys: Final = [self.check_and_fix_namespace(key=cache_key or "") for cache_key in _key_list]
             results: Final = self._run_redis_mget_operation(keys=_keys)
+            _exit_circuit_breaker(self._circuit_breaker, swallowed_before)
             end_time: Final = time.time()
             _duration: Final = end_time - start_time
             self.service_logger_obj.service_success_hook(
@@ -1178,7 +1345,18 @@ class RedisCache(BaseCache):
 
             return decoded_results
         except Exception as e:
+            failed_at: Final = time.time()
+            self.service_logger_obj.service_failure_hook(
+                service=ServiceTypes.REDIS,
+                duration=failed_at - start_time,
+                error=e,
+                call_type=f"batch_get_cache <- {_get_call_stack_info()}",
+                start_time=start_time,
+                end_time=failed_at,
+                parent_otel_span=parent_otel_span,
+            )
             verbose_logger.error("Error occurred in batch get cache - %s", e)
+            _record_swallowed_redis_failure(self._circuit_breaker, e)
             return key_value_dict
 
     @_redis_circuit_breaker_guard

litellm/caching/redis_cluster_node_isolation.py

@@ -19,13 +19,13 @@ connections untouched. Every other branch (MOVED, ASK, CLUSTERDOWN, slot-not-cov
 retry-exhaustion) is unchanged from upstream, since those already carry real evidence the
 topology changed.
 
-redis-py 8.x fixed this upstream with gentler machinery than this override's
-``node.disconnect()`` (which also kills connections other coroutines are mid-operation
-on, so one timeout cascades into a reconnect storm and, with TLS, a fresh handshake per
-killed connection): it marks in-use connections for reconnect only after their current
-operation completes, disconnects only the idle pooled ones, and defers reinitialization
-to the outer retry loop. When the installed ``ClusterNode`` has that per-connection
-recovery API, the factory returns the base ``RedisCluster`` unmodified.
+redis-py 8.x recovers connections per-connection, so the copied override is not used. Upstream
+still flips the shared ``_initialize`` flag on any node's timeout, funneling every concurrent
+caller through the reinit lock and, if ``CLUSTER SLOTS`` lands on the slow node, into a full
+teardown. For those versions the factory returns a thin wrapper around upstream's
+``_execute_command`` that clears the flag again after an isolated timeout (a ConnectionError,
+a third consecutive timeout on the same node, or a concurrent request from any other command
+or ``aclose()`` still reinits).
 """
 
 import asyncio
@@ -44,6 +44,8 @@ class _ClusterNodeAttrs(Protocol):
     mode; typing ``target_node`` as this Protocol at the one boundary keeps the override's
     own logic fully typed without a banned ``typing.cast``."""
 
+    name: str
+
     async def execute_command(
         self,
         *args: object,
@@ -78,18 +80,20 @@ class _ClusterAttrs(Protocol):
 #: this override can't see (Python won't error -- it'll just run our now-stale copy), so
 #: construction logs a loud warning rather than silently trusting an unverified copy.
 _VERIFIED_REDIS_VERSIONS: Final = frozenset({"5.3.1"})
+_CONSECUTIVE_TIMEOUTS_BEFORE_REINIT: Final = 3
 
 
-def get_litellm_async_redis_cluster_class(
+def get_litellm_async_redis_cluster_class(  # noqa: C901  # supports redis-py version-specific cluster implementations
     cluster_node_class: type | None = None,
+    base_cluster_class: type | None = None,
 ) -> type["_AsyncRedisClusterType"]:
-    """Returns the base ``RedisCluster`` when the installed redis-py already recovers a
-    node-level connection error per-connection (8.x+), else builds the ``RedisCluster``
-    subclass with the per-node isolation fix for older versions whose upstream branch
-    tears down the whole cluster client.
+    """Returns a timeout-tolerant ``RedisCluster`` subclass when installed redis-py already
+    recovers node-level connections per-connection (8.x+), else builds the ``RedisCluster``
+    subclass with the per-node isolation fix for older versions whose upstream branch tears
+    down the whole cluster client.
 
-    ``cluster_node_class`` exists for dependency injection in tests; production callers
-    leave it unset and the installed ``ClusterNode`` is used.
+    ``cluster_node_class`` and ``base_cluster_class`` exist for dependency injection in tests;
+    production callers leave them unset and the installed redis-py classes are used.
 
     Imported lazily because this module is reachable from a base ``import litellm`` while
     redis is not a base dependency. Cheap to call repeatedly: the underlying redis
@@ -118,13 +122,68 @@ def get_litellm_async_redis_cluster_class(
     from redis.exceptions import TimeoutError as _RedisTimeoutError
 
     node_class: Final = cluster_node_class if cluster_node_class is not None else _AsyncClusterNode
+    base_class: Final = base_cluster_class if base_cluster_class is not None else _BaseAsyncRedisCluster
     if hasattr(node_class, "update_active_connections_for_reconnect"):
         verbose_logger.debug(
-            "redis-py %s recovers a node-level connection error per-connection upstream; "
-            "using the base RedisCluster without litellm's node-isolation override.",
+            "redis-py %s recovers node connections per-connection upstream; using "
+            "LiteLLM's timeout-tolerant RedisCluster wrapper.",
             redis.__version__,
         )
-        return _BaseAsyncRedisCluster
+
+        class LiteLLMAsyncRedisClusterTimeoutTolerant(
+            base_class  # pyright: ignore[reportGeneralTypeIssues, reportUntypedBaseClass]  # the injected base class is selected at runtime
+        ):
+            def __init__(
+                self,
+                *args: object,
+                **kwargs: object,  # kwargs-ok: passes redis-py's constructor kwargs through untouched
+            ) -> None:
+                self._litellm_initialize = False
+                self._litellm_reinit_requests = 0
+                self._litellm_tolerated_timeouts = 0
+                super().__init__(*args, **kwargs)
+                self._litellm_consecutive_timeouts: dict[  # mutable-ok: per-node counter updated on the command hot path
+                    str, int
+                ] = {}
+
+            @property
+            def _initialize(self) -> bool:
+                return self._litellm_initialize
+
+            @_initialize.setter
+            def _initialize(self, value: bool) -> None:
+                if value:
+                    self._litellm_reinit_requests += 1
+                self._litellm_initialize = value
+
+            async def _execute_command(
+                self,
+                target_node: _ClusterNodeAttrs,
+                *args: object,
+                **kwargs: object,  # kwargs-ok: matches redis-py's own command dispatch signature
+            ) -> object:
+                outstanding_before: Final = self._litellm_reinit_requests - self._litellm_tolerated_timeouts
+                pending_before: Final = self._litellm_initialize
+                try:
+                    result: Final = await super()._execute_command(target_node, *args, **kwargs)
+                except _RedisTimeoutError:
+                    timeouts: Final = self._litellm_consecutive_timeouts.get(target_node.name, 0) + 1
+                    if timeouts >= _CONSECUTIVE_TIMEOUTS_BEFORE_REINIT:
+                        self._litellm_consecutive_timeouts.pop(target_node.name, None)
+                        raise
+                    self._litellm_consecutive_timeouts[target_node.name] = timeouts
+                    self._litellm_tolerated_timeouts += 1
+                    if (
+                        not pending_before
+                        and self._litellm_reinit_requests - self._litellm_tolerated_timeouts == outstanding_before
+                    ):
+                        self._initialize = False
+                    raise
+                if self._litellm_consecutive_timeouts:
+                    self._litellm_consecutive_timeouts.pop(target_node.name, None)
+                return result
+
+        return LiteLLMAsyncRedisClusterTimeoutTolerant
 
     if redis.__version__ not in _VERIFIED_REDIS_VERSIONS:
         verbose_logger.warning(

litellm/completion_extras/litellm_responses_transformation/transformation.py

@@ -932,7 +932,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
         return OpenAiResponsesToChatCompletionStreamIterator(streaming_response, sync_stream, json_mode)
 
     def _convert_content_str_to_input_text(self, content: str, role: str) -> dict[str, object]:
-        if role == "user" or role == "system" or role == "tool":
+        if role in ("user", "system", "developer", "tool"):
             return {"type": "input_text", "text": content}
         else:
             return {"type": "output_text", "text": content}

litellm/constants.py

@@ -7,6 +7,7 @@ from litellm.litellm_core_utils.env_utils import get_env_int, get_env_int_in_ran
 
 DEFAULT_HEALTH_CHECK_PROMPT: Final = str(os.getenv("DEFAULT_HEALTH_CHECK_PROMPT", "test from litellm"))
 AZURE_DEFAULT_RESPONSES_API_VERSION: Final = str(os.getenv("AZURE_DEFAULT_RESPONSES_API_VERSION", "preview"))
+AZURE_OPENAI_AUDIO_PROVIDERS: Final = frozenset({"azure", "azure_ai"})
 ROUTER_MAX_FALLBACKS: Final = int(os.getenv("ROUTER_MAX_FALLBACKS", 5))
 ROUTER_FALLBACK_ERROR_DETAIL_MAX_CHARS: Final = 2000
 RUNTIME_UPDATABLE_ROUTER_SETTINGS: Final[frozenset[str]] = frozenset(
@@ -39,6 +40,7 @@ ROUTER_SETTINGS_MANAGED_OUTSIDE_CONFIG: Final[frozenset[str]] = frozenset(
         "router_general_settings",
         "ignore_invalid_deployments",
         "fallback_access_check",
+        "heuristic_v2_router_limit",
     }
 )
 DEFAULT_BATCH_SIZE: Final = int(os.getenv("DEFAULT_BATCH_SIZE", 512))
@@ -149,6 +151,7 @@ DEFAULT_SEMANTIC_GUARD_SIMILARITY_THRESHOLD = float(os.getenv("DEFAULT_SEMANTIC_
 MCP_OAUTH2_TOKEN_EXPIRY_BUFFER_SECONDS: Final = int(os.getenv("MCP_OAUTH2_TOKEN_EXPIRY_BUFFER_SECONDS", "60"))
 MCP_OAUTH2_TOKEN_CACHE_MAX_SIZE: Final = int(os.getenv("MCP_OAUTH2_TOKEN_CACHE_MAX_SIZE", "200"))
 MCP_OAUTH2_TOKEN_CACHE_DEFAULT_TTL: Final = int(os.getenv("MCP_OAUTH2_TOKEN_CACHE_DEFAULT_TTL", "3600"))
+MCP_SSO_ASSERTION_CACHE_TTL_SECONDS: Final = int(os.getenv("MCP_SSO_ASSERTION_CACHE_TTL_SECONDS", "60"))
 
 # Default npm cache directory for STDIO MCP servers.
 # npm/npx needs a writable cache dir; in containers the default (~/.npm)
@@ -430,6 +433,9 @@ REDIS_CONNECTION_POOL_TIMEOUT: Final = int(os.getenv("REDIS_CONNECTION_POOL_TIME
 REDIS_CIRCUIT_BREAKER_FAILURE_THRESHOLD: Final = int(os.getenv("REDIS_CIRCUIT_BREAKER_FAILURE_THRESHOLD", 5))
 REDIS_CIRCUIT_BREAKER_RECOVERY_TIMEOUT: Final = int(os.getenv("REDIS_CIRCUIT_BREAKER_RECOVERY_TIMEOUT", 60))
 REDIS_CIRCUIT_BREAKER_ENABLED: Final = os.getenv("REDIS_CIRCUIT_BREAKER_ENABLED", "true").lower() == "true"
+# minimum seconds a timeout-only failure streak must span before it can open the breaker,
+# so one event-loop stall timing out many queued calls at once does not trip it
+REDIS_CIRCUIT_BREAKER_TIMEOUT_MIN_DURATION: Final = float(os.getenv("REDIS_CIRCUIT_BREAKER_TIMEOUT_MIN_DURATION", 5.0))
 # Seconds of idle before a Redis cluster connection is validated with a PING and
 # reconnected if dead, so a connection silently dropped by a cluster restart
 # (e.g. ElastiCache Serverless maintenance) is not reused while broken
@@ -1450,6 +1456,7 @@ SESSION_DEPLOYMENT_AFFINITY_TTL_METADATA_KEY: Final = "_session_deployment_affin
 CONSUMED_REQUEST_TAGS_METADATA_KEY: Final = "_consumed_request_tags"
 INTERNAL_CALL_ORIGIN_METADATA_KEY: Final = "internal_call_origin"
 SESSION_ID_GENERATED_METADATA_KEY: Final = "litellm_session_id_generated"
+SESSION_ID_OMITTED_METADATA_KEY: Final = "litellm_session_id_omitted"
 LITELLM_TRUNCATED_PAYLOAD_FIELD: Final = "litellm_truncated"
 LITELLM_TRUNCATION_DB_SAFEGUARD_NOTE: Final = (
     "Truncation is a DB storage safeguard. "

litellm/experimental_mcp_client/client.py

@@ -70,7 +70,7 @@ def to_basic_auth(auth_value: str) -> str:
 
 
 def strip_auth_scheme(auth_value: str, scheme: str) -> str:
-    """Return ``auth_value`` with a leading ``<scheme> `` removed, or unchanged when absent.
+    """Return ``auth_value`` with a leading ``<scheme>`` and separator removed, or unchanged when absent.
 
     Callers supply both a bare credential and a complete header value, so prefixing
     unconditionally yields ``Bearer Bearer <jwt>``. Scheme names are case-insensitive per
@@ -78,10 +78,9 @@ def strip_auth_scheme(auth_value: str, scheme: str) -> str:
     with the scheme text and a scheme with nothing behind it are returned untouched.
     Surrounding whitespace is left to ``_strip_header_whitespace`` at header-build time.
     """
-    scheme_name, _, remainder = auth_value.lstrip().partition(" ")
-    credential: Final = remainder.lstrip()
-    if credential and scheme_name.lower() == scheme.lower():
-        return credential
+    parts: Final = auth_value.split(None, 1)
+    if len(parts) == 2 and parts[0].lower() == scheme.lower():
+        return parts[1]
     return auth_value
 
 

litellm/integrations/azure_storage/azure_storage.py

@@ -36,7 +36,7 @@ AZURE_STORAGE_TOKEN_SCOPE: Final = "https://storage.azure.com/.default"
 def _cached_credential_chain_token_provider() -> Callable[[], str]:
     return get_azure_ad_token_provider(
         azure_scope=AZURE_STORAGE_TOKEN_SCOPE,
-        azure_credential=AzureCredentialType.DefaultAzureCredential,
+        azure_credential=AzureCredentialType.DeploymentIdentityCredential,
     )
 
 

litellm/integrations/custom_guardrail.py

@@ -831,10 +831,10 @@ class CustomGuardrail(CustomLogger):
         # should run guardrail
         litellm_guardrails: Final = request_data.get("guardrails")
         if litellm_guardrails is None or not isinstance(litellm_guardrails, list):
-            return response
+            return None
 
         if self.should_run_guardrail(data=request_data, event_type=GuardrailEventHooks.post_call) is not True:
-            return response
+            return None
 
         # CHECK IF GUARDRAIL REJECTS THE REQUEST
         result: Final = await self.async_post_call_success_hook(
@@ -850,7 +850,7 @@ class CustomGuardrail(CustomLogger):
         )
 
         if not self._is_valid_response_type(result):
-            return response
+            return None
 
         return result
 

litellm/integrations/vector_store_integrations/vector_store_pre_call_hook.py

@@ -6,7 +6,8 @@ It searches the vector store for relevant context and appends it to the messages
 """
 
 from collections.abc import Awaitable, Callable
-from typing import TYPE_CHECKING, Any, Final, cast
+from dataclasses import dataclass
+from typing import TYPE_CHECKING, Any, Final, Protocol, cast
 
 import litellm
 import litellm.vector_stores
@@ -24,10 +25,35 @@ from litellm.types.vector_stores import (
 
 if TYPE_CHECKING:
     from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
+    from litellm.proxy.utils import PrismaClient
+    from litellm.router import Router
 else:
     LiteLLMLoggingObj = Any
 
 
+class ProxyRuntime(Protocol):
+    def llm_router(self) -> "Router | None": ...
+
+    def prisma_client(self) -> "PrismaClient | None": ...
+
+
+@dataclass(frozen=True, slots=True)
+class ProxyServerRuntime:
+    def llm_router(self) -> "Router | None":
+        try:
+            from litellm.proxy.proxy_server import llm_router
+        except ImportError:
+            return None
+        return llm_router
+
+    def prisma_client(self) -> "PrismaClient | None":
+        try:
+            from litellm.proxy.proxy_server import prisma_client
+        except ImportError:
+            return None
+        return prisma_client
+
+
 class VectorStorePreCallHook(CustomLogger):
     CONTENT_PREFIX_STRING = "Context:\n\n"
     """
@@ -39,8 +65,9 @@ class VectorStorePreCallHook(CustomLogger):
     3. Appends the search results as context to the messages
     """
 
-    def __init__(self):
+    def __init__(self, proxy_runtime: ProxyRuntime | None = None):
         super().__init__()
+        self.proxy_runtime: Final[ProxyRuntime] = proxy_runtime or ProxyServerRuntime()
 
     async def async_get_chat_completion_prompt(
         self,
@@ -79,21 +106,8 @@ class VectorStorePreCallHook(CustomLogger):
             if litellm.vector_store_registry is None:
                 return model, messages, non_default_params
 
-            # Get prisma_client for database fallback
-            prisma_client = None
-            llm_router = None
-            try:
-                from litellm.proxy.proxy_server import (
-                    llm_router as _llm_router,
-                )
-                from litellm.proxy.proxy_server import (
-                    prisma_client as _prisma_client,
-                )
-
-                prisma_client = _prisma_client
-                llm_router = _llm_router
-            except ImportError:
-                pass
+            prisma_client: Final = self.proxy_runtime.prisma_client()
+            llm_router: Final = self.proxy_runtime.llm_router()
 
             # Use database fallback to ensure synchronization across instances
             vector_stores_to_run: list[
@@ -136,15 +150,23 @@ class VectorStorePreCallHook(CustomLogger):
                         Callable[..., Awaitable[VectorStoreSearchResponse]],
                         litellm.vector_stores.asearch,
                     )
-                search_response = await search_function(
-                    **{
-                        "vector_store_id": vector_store_id,
-                        "query": query,
-                        "custom_llm_provider": custom_llm_provider,
-                        "metadata": request_metadata,
-                        **litellm_params_for_vector_store,
-                    },
-                )
+                try:
+                    search_response = await search_function(
+                        **{
+                            "vector_store_id": vector_store_id,
+                            "query": query,
+                            "custom_llm_provider": custom_llm_provider,
+                            "metadata": request_metadata,
+                            **litellm_params_for_vector_store,
+                        },
+                    )
+                except Exception as search_error:
+                    verbose_logger.warning(
+                        "Vector store search failed for vector_store_id=%s, continuing without its context: %s",
+                        vector_store_id,
+                        search_error,
+                    )
+                    continue
 
                 verbose_logger.debug("search_response: %s", search_response)
 
@@ -153,7 +175,7 @@ class VectorStorePreCallHook(CustomLogger):
 
                 # Process search results and append as context
                 modified_messages = self._append_search_results_to_messages(
-                    messages=messages, search_response=search_response
+                    messages=modified_messages, search_response=search_response
                 )
 
                 # Get the number of results for logging

litellm/litellm_core_utils/internal_call_metadata.py

@@ -18,9 +18,11 @@ caller's identity metadata, minus two things that must never be forwarded as-is:
 from __future__ import annotations
 
 from collections.abc import Mapping
+from types import MappingProxyType
 from typing import Final
 
 from litellm.constants import INTERNAL_CALL_ORIGIN_METADATA_KEY, NON_INFERENCE_CALL_TYPES
+from litellm.litellm_core_utils.initialize_dynamic_callback_params import initialize_standard_callback_dynamic_params
 from litellm.types.utils import BACKGROUND_RESPONSE_COST_POLL_CALL_ORIGIN, InternalCallOrigin
 
 BUDGET_RESERVATION_METADATA_KEYS: Final = frozenset({"user_api_key_budget_reservation"})
@@ -142,6 +144,19 @@ def forwarded_internal_call_metadata(
     }
 
 
+def parent_session_kwargs(request_kwargs: Mapping[str, object] | None) -> Mapping[str, str]:
+    kwargs: Final = request_kwargs or MappingProxyType({})
+    return MappingProxyType(
+        {k: v for k in ("litellm_session_id", "litellm_trace_id") if isinstance(v := kwargs.get(k), str)}
+    )
+
+
+def effective_turn_off_message_logging(request_kwargs: Mapping[str, object] | None) -> bool | None:
+    return initialize_standard_callback_dynamic_params(dict(request_kwargs) if request_kwargs else None).get(
+        "turn_off_message_logging"
+    )
+
+
 def sanitized_forwardable_call_metadata(
     parent_metadata: Mapping[str, object],
     call_origin: InternalCallOrigin,

litellm/litellm_core_utils/litellm_logging.py

@@ -10,7 +10,7 @@ import subprocess
 import sys
 import time
 import traceback
-from collections.abc import Callable, Mapping, Sequence
+from collections.abc import Callable, Iterator, Mapping, Sequence
 from datetime import datetime as dt_object
 from functools import lru_cache
 from types import MappingProxyType, TracebackType
@@ -414,6 +414,11 @@ def _resolve_vertex_location_for_cost(
     return VertexBase.get_vertex_region(configured_location, model)
 
 
+def _provider_response_id(source: object) -> str | None:
+    candidate: Final = source.get("id") if isinstance(source, dict) else getattr(source, "id", None)
+    return candidate if isinstance(candidate, str) and candidate else None
+
+
 class Logging(LiteLLMLoggingBaseClass):
     global \
         supabaseClient, \
@@ -429,6 +434,7 @@ class Logging(LiteLLMLoggingBaseClass):
     custom_pricing: bool = False
     stream_options = None
     litellm_request_debug: bool = False
+    streamed_anthropic_message_id: str | None = None
 
     def __init__(
         self,
@@ -2136,7 +2142,7 @@ class Logging(LiteLLMLoggingBaseClass):
             self.model_call_details["cache_hit"] = cache_hit
 
             if self.call_type == CallTypes.anthropic_messages.value:
-                result = self._handle_anthropic_messages_response_logging(result=result)
+                result = self._anthropic_messages_logged_response(result=result)
             elif (
                 self.call_type == CallTypes.generate_content.value
                 or self.call_type == CallTypes.agenerate_content.value
@@ -3806,6 +3812,23 @@ class Logging(LiteLLMLoggingBaseClass):
             )
         return None
 
+    def record_streamed_anthropic_message_id(self, message_id: str) -> None:
+        self.streamed_anthropic_message_id = message_id
+
+    def _anthropic_messages_logged_response(self, result: Any) -> ModelResponse:
+        """
+        The ModelResponse a /v1/messages spend_logs row is built from.
+
+        A streaming call bridged onto the Responses API is the one case where the `msg_` id the
+        caller was served is minted locally rather than issued upstream, so it is absent from the
+        response the row would otherwise be keyed on and has to be carried over here.
+        """
+        logged: Final = self._handle_anthropic_messages_response_logging(result=result)
+        streamed_message_id: Final = self.streamed_anthropic_message_id
+        if streamed_message_id is None:
+            return logged
+        return logged.model_copy(update={"id": streamed_message_id})
+
     def _handle_anthropic_messages_response_logging(self, result: Any) -> ModelResponse:
         """
         Handles logging for Anthropic messages responses.
@@ -3832,11 +3855,12 @@ class Logging(LiteLLMLoggingBaseClass):
         if isinstance(result, ResponsesAPIResponse):
             return self._translate_responses_api_response_to_model_response(result)
 
+        provider_response_id: Final = _provider_response_id(result)
         httpx_response: Final = self.model_call_details.get("httpx_response", None)
         if httpx_response and isinstance(httpx_response, httpx.Response):
             result = litellm.AnthropicConfig().transform_response(
                 raw_response=httpx_response,
-                model_response=litellm.ModelResponse(),
+                model_response=litellm.ModelResponse(id=provider_response_id),
                 model=self.model,
                 messages=[],
                 logging_obj=self,
@@ -3859,7 +3883,7 @@ class Logging(LiteLLMLoggingBaseClass):
                     status_code=200,
                     headers={},
                 ),
-                model_response=litellm.ModelResponse(),
+                model_response=litellm.ModelResponse(id=provider_response_id),
                 json_mode=None,
                 speed=self.optional_params.get("speed") if self.optional_params else None,
             )
@@ -3882,7 +3906,7 @@ class Logging(LiteLLMLoggingBaseClass):
             return LiteLLMResponsesTransformationHandler().transform_response(
                 model=self.model,
                 raw_response=result,
-                model_response=litellm.ModelResponse(),
+                model_response=litellm.ModelResponse(id=_provider_response_id(result)),
                 logging_obj=self,
                 request_data={},
                 messages=[],
@@ -3897,7 +3921,7 @@ class Logging(LiteLLMLoggingBaseClass):
                 "usage-only ModelResponse to keep the spend_logs row.",
                 str(e),
             )
-            model_response: Final = litellm.ModelResponse()
+            model_response: Final = litellm.ModelResponse(id=_provider_response_id(result))
             model_response.model = self.model
             usage: Final = getattr(result, "usage", None)
             if usage is not None and ResponseAPILoggingUtils._is_response_api_usage(usage):
@@ -5854,14 +5878,28 @@ def _get_status_fields(
     #########################################################
     # Map - guardrail_information.guardrail_status to guardrail_status
     #########################################################
-    guardrail_status: GuardrailStatus = "not_run"
-    if guardrail_information and isinstance(guardrail_information, list):
-        for information in guardrail_information:
-            if isinstance(information, dict):
-                raw_status = information.get("guardrail_status", "not_run")
-                if raw_status != "not_run":
-                    guardrail_status = GUARDRAIL_STATUS_MAP.get(raw_status, "not_run")
-                    break
+    # Severity order, least severe first. The status aggregates across ALL
+    # guardrail entries rather than taking the first non-"not_run" one: a
+    # pre_call guardrail that passed (e.g. a mask) records its entry before a
+    # later guardrail's block, and first-wins would report a blocked request
+    # as "success".
+    GUARDRAIL_STATUS_SEVERITY: Final[tuple[GuardrailStatus, ...]] = (
+        "not_run",
+        "success",
+        "guardrail_failed_to_respond",
+        "guardrail_intervened",
+    )
+    entries: Final[Sequence[object]] = guardrail_information if isinstance(guardrail_information, list) else ()
+    raw_statuses: Final[Iterator[object]] = (
+        entry.get("guardrail_status", "not_run") for entry in entries if isinstance(entry, dict)
+    )
+    # A guardrail is free to write any value here, and an unhashable one would
+    # raise TypeError on the mapping lookup and drop the whole payload.
+    guardrail_status: Final[GuardrailStatus] = max(
+        (GUARDRAIL_STATUS_MAP.get(raw_status, "not_run") for raw_status in raw_statuses if isinstance(raw_status, str)),
+        key=GUARDRAIL_STATUS_SEVERITY.index,
+        default="not_run",
+    )
 
     return StandardLoggingPayloadStatusFields(llm_api_status=llm_api_status, guardrail_status=guardrail_status)
 

litellm/litellm_core_utils/llm_cost_calc/tool_call_cost_tracking.py

@@ -5,6 +5,8 @@ Helper utilities for tracking the cost of built-in tools.
 from collections.abc import Mapping
 from typing import Final, Literal
 
+from pydantic import ValidationError
+
 import litellm
 from litellm.constants import OPENAI_FILE_SEARCH_COST_PER_1K_CALLS
 from litellm.litellm_core_utils.llm_cost_calc.utils import (
@@ -13,6 +15,7 @@ from litellm.litellm_core_utils.llm_cost_calc.utils import (
 from litellm.types.llms.openai import (
     FileSearchTool,
     ResponsesAPIResponse,
+    ResponsesToolUsage,
     WebSearchOptions,
 )
 from litellm.types.utils import (
@@ -32,6 +35,17 @@ def _output_item_type(output_item: object) -> str | None:
     return item_type if isinstance(item_type, str) else None
 
 
+def _reported_web_search_requests(response_object: ResponsesAPIResponse) -> int | None:
+    tool_usage: Final = getattr(response_object, "tool_usage", None)
+    if tool_usage is None:
+        return None
+    try:
+        web_search: Final = ResponsesToolUsage.model_validate(tool_usage).web_search
+    except ValidationError:
+        return None
+    return None if web_search is None else web_search.num_requests
+
+
 def _usage_reports_server_side_web_search_calls(usage: Usage) -> bool:
     details: Final = getattr(usage, "server_side_tool_usage_details", None)
     if not isinstance(details, Mapping):
@@ -182,15 +196,19 @@ class StandardBuiltInToolCostTracking:
 
         Providers that report a request count in usage (gemini, anthropic, xai, vertex) are handled by
         get_cost_for_web_search_request and never reach here. This path prices per call, so it must count
-        the web_search_call items. Chat-completions responses only expose url_citation annotations with no
-        count, so they floor to a single billable search.
+        the web_search_call items, unless the response reports the billable count itself
+        (Bedrock's tool_usage.web_search.num_requests, which excludes open_page fetches). Chat-completions
+        responses only expose url_citation annotations with no count, so they floor to a single billable search.
         """
-        if isinstance(response_object, ResponsesAPIResponse):
-            count = sum(
-                1 for output_item in response_object.output if _output_item_type(output_item) == "web_search_call"
-            )
-            return max(count, 1)
-        return 1
+        if not isinstance(response_object, ResponsesAPIResponse):
+            return 1
+        reported: Final = _reported_web_search_requests(response_object)
+        if reported is not None:
+            return reported
+        count: Final = sum(
+            1 for output_item in response_object.output if _output_item_type(output_item) == "web_search_call"
+        )
+        return max(count, 1)
 
     @staticmethod
     def _handle_file_search_cost(

litellm/litellm_core_utils/llm_cost_calc/utils.py

@@ -415,40 +415,64 @@ def _is_off_peak(off_peak: Mapping[str, object], current_time: datetime | None =
     return False
 
 
-def _coerce_off_peak_rate(value: object, default: float) -> float:
+@dataclass(frozen=True, slots=True)
+class TokenRates:
+    input_rate: float
+    output_rate: float
+    cache_read_rate: float
+    cache_creation_rate: float
+    reasoning_rate: float | None
+
+    @property
+    def billed_reasoning_rate(self) -> float:
+        return self.output_rate if self.reasoning_rate is None else self.reasoning_rate
+
+
+def _parse_off_peak_rate(value: object) -> float | None:
     if isinstance(value, bool):
-        return default
+        return None
     if isinstance(value, (int, float)):
         return float(value)
     if isinstance(value, str):
         try:
             return float(value)
         except ValueError:
-            return default
-    return default
+            return None
+    return None
 
 
-def _apply_off_peak_pricing(
-    model_info: ModelInfo,
-    current_time: datetime | None,
-    prompt_base_cost: float,
-    completion_base_cost: float,
-    cache_read_cost: float,
-) -> tuple[float, float, float]:
+def _off_peak_rate(off_peak: Mapping[str, object], key: str, standard_rate: float) -> float:
+    parsed: Final = _parse_off_peak_rate(off_peak.get(key))
+    return standard_rate if parsed is None else parsed
+
+
+def _open_off_peak_block(model_info: ModelInfo, current_time: datetime | None) -> Mapping[str, object] | None:
+    off_peak: Final = model_info.get("off_peak_pricing")
+    if not isinstance(off_peak, Mapping) or not _is_off_peak(off_peak, current_time):
+        return None
+    return off_peak
+
+
+def apply_off_peak_pricing(model_info: ModelInfo, current_time: datetime | None, rates: TokenRates) -> TokenRates:
     """Swap in off-peak per-token rates when the current UTC time is inside one of the model's
     off_peak_pricing rules, the every-day hours_utc windows or a day-of-week-qualified entry in
     windows. An off-peak rate replaces the rate that would otherwise apply rather than
     discounting it, so a model that also has tiered or above-threshold pricing bills the flat
     off-peak rate for the whole request while the window is open. Any rate left unset in
-    off_peak_pricing falls back to the standard rate.
+    off_peak_pricing falls back to the standard rate, so a block without
+    output_cost_per_reasoning_token keeps the model's own reasoning rate, or its off-peak output
+    rate when reasoning has no dedicated rate at all.
     """
-    off_peak: Final = model_info.get("off_peak_pricing")
-    if not isinstance(off_peak, Mapping) or not _is_off_peak(off_peak, current_time):
-        return prompt_base_cost, completion_base_cost, cache_read_cost
-    return (
-        _coerce_off_peak_rate(off_peak.get("input_cost_per_token"), prompt_base_cost),
-        _coerce_off_peak_rate(off_peak.get("output_cost_per_token"), completion_base_cost),
-        _coerce_off_peak_rate(off_peak.get("cache_read_input_token_cost"), cache_read_cost),
+    off_peak: Final = _open_off_peak_block(model_info, current_time)
+    if off_peak is None:
+        return rates
+    off_peak_reasoning_rate: Final = _parse_off_peak_rate(off_peak.get("output_cost_per_reasoning_token"))
+    return TokenRates(
+        input_rate=_off_peak_rate(off_peak, "input_cost_per_token", rates.input_rate),
+        output_rate=_off_peak_rate(off_peak, "output_cost_per_token", rates.output_rate),
+        cache_read_rate=_off_peak_rate(off_peak, "cache_read_input_token_cost", rates.cache_read_rate),
+        cache_creation_rate=_off_peak_rate(off_peak, "cache_creation_input_token_cost", rates.cache_creation_rate),
+        reasoning_rate=rates.reasoning_rate if off_peak_reasoning_rate is None else off_peak_reasoning_rate,
     )
 
 
@@ -458,14 +482,28 @@ def _apply_off_peak_to_base_costs(
     base_costs: tuple[float, float, float, float, float],
 ) -> tuple[float, float, float, float, float]:
     """Apply off-peak rates to an already-resolved set of base costs, whichever pricing path
-    produced them. Cache-creation rates are passed through untouched, since off_peak_pricing
-    has no field for them.
+    produced them. The one-hour cache-creation rate passes through untouched, since
+    off_peak_pricing has no field for it, and reasoning is left to _resolve_billed_reasoning_rate.
     """
     prompt, completion, cache_creation, cache_creation_above_1hr, cache_read = base_costs
-    off_peak_prompt, off_peak_completion, off_peak_cache_read = _apply_off_peak_pricing(
-        model_info, current_time, prompt, completion, cache_read
+    rates: Final = apply_off_peak_pricing(
+        model_info,
+        current_time,
+        TokenRates(
+            input_rate=prompt,
+            output_rate=completion,
+            cache_read_rate=cache_read,
+            cache_creation_rate=cache_creation,
+            reasoning_rate=None,
+        ),
+    )
+    return (
+        rates.input_rate,
+        rates.output_rate,
+        rates.cache_creation_rate,
+        cache_creation_above_1hr,
+        rates.cache_read_rate,
     )
-    return (off_peak_prompt, off_peak_completion, cache_creation, cache_creation_above_1hr, off_peak_cache_read)
 
 
 def _get_token_base_cost(
@@ -1029,6 +1067,29 @@ def _resolve_reasoning_token_cost(
     return standard_reasoning_cost if standard_reasoning_cost is not None else completion_base_cost
 
 
+def _resolve_billed_reasoning_rate(
+    model_info: ModelInfo,
+    usage: Usage,
+    service_tier: str | None,
+    completion_base_cost: float,
+    current_time: datetime | None,
+) -> float:
+    off_peak: Final = _open_off_peak_block(model_info, current_time)
+    off_peak_reasoning_rate: Final = (
+        None if off_peak is None else _parse_off_peak_rate(off_peak.get("output_cost_per_reasoning_token"))
+    )
+    if off_peak_reasoning_rate is not None:
+        return off_peak_reasoning_rate
+    tiered_reasoning_rate: Final = _get_tiered_reasoning_rate(model_info=model_info, usage=usage)
+    if tiered_reasoning_rate is not None:
+        return tiered_reasoning_rate
+    return _resolve_reasoning_token_cost(
+        model_info=model_info,
+        service_tier=service_tier,
+        completion_base_cost=completion_base_cost,
+    )
+
+
 def generic_cost_per_token(
     model: str,
     usage: Usage,
@@ -1037,6 +1098,7 @@ def generic_cost_per_token(
     data_residency: str | None = None,
     model_info: ModelInfo | None = None,
     vertex_location: str | None = None,
+    current_time: datetime | None = None,
 ) -> tuple[float, float]:
     """
     Calculates the cost per token for a given model, prompt tokens, and completion tokens.
@@ -1051,6 +1113,7 @@ def generic_cost_per_token(
         - vertex_location: optional Vertex AI location the request was served from
           (e.g. "us-east5", "global"), used to apply the per-model
           regional-endpoint uplift multiplier when non-global.
+        - current_time: the moment the request is billed at, for off_peak_pricing; defaults to now, UTC
 
     Returns:
         Tuple[float, float] - prompt_cost_in_usd, completion_cost_in_usd
@@ -1117,6 +1180,7 @@ def generic_cost_per_token(
             usage.prompt_tokens - cache_hit - audio_tokens - cache_creation - image_tokens - video_tokens, 0
         )
 
+    billing_time: Final = current_time if current_time is not None else datetime.now(timezone.utc)
     (
         prompt_base_cost,
         completion_base_cost,
@@ -1127,6 +1191,7 @@ def generic_cost_per_token(
         model_info=model_info,
         usage=usage,
         service_tier=service_tier,
+        current_time=billing_time,
         threshold_is_inclusive=_uses_inclusive_token_thresholds(custom_llm_provider),
     )
 
@@ -1185,17 +1250,13 @@ def generic_cost_per_token(
 
     ## REASONING COST
     if not is_text_tokens_total and reasoning_tokens and reasoning_tokens > 0:
-        tiered_reasoning_rate: Final = _get_tiered_reasoning_rate(model_info=model_info, usage=usage)
-        _output_cost_per_reasoning_token = (
-            tiered_reasoning_rate
-            if tiered_reasoning_rate is not None
-            else _resolve_reasoning_token_cost(
-                model_info=model_info,
-                service_tier=service_tier,
-                completion_base_cost=completion_base_cost,
-            )
+        completion_cost += float(reasoning_tokens) * _resolve_billed_reasoning_rate(
+            model_info=model_info,
+            usage=usage,
+            service_tier=service_tier,
+            completion_base_cost=completion_base_cost,
+            current_time=billing_time,
         )
-        completion_cost += float(reasoning_tokens) * _output_cost_per_reasoning_token
 
     ## IMAGE COST
     if not is_text_tokens_total and image_tokens and image_tokens > 0:
@@ -1247,6 +1308,7 @@ def get_token_type_cost_breakdown(
     service_tier: str | None = None,
     data_residency: str | None = None,
     vertex_location: str | None = None,
+    current_time: datetime | None = None,
 ) -> TokenTypeCostBreakdown:
     """
     Provider-agnostic cost of reasoning and cache tokens, derived from the usage
@@ -1265,6 +1327,7 @@ def get_token_type_cost_breakdown(
     except Exception:
         return TokenTypeCostBreakdown(0.0, 0.0, 0.0)
 
+    billing_time: Final = current_time if current_time is not None else datetime.now(timezone.utc)
     (
         _prompt_base_cost,
         completion_base_cost,
@@ -1275,6 +1338,7 @@ def get_token_type_cost_breakdown(
         model_info=model_info,
         usage=usage,
         service_tier=service_tier,
+        current_time=billing_time,
         threshold_is_inclusive=_uses_inclusive_token_thresholds(custom_llm_provider),
     )
 
@@ -1284,18 +1348,12 @@ def get_token_type_cost_breakdown(
     if not reasoning_tokens:
         reasoning_tokens = _coerce_token_count(getattr(usage, "reasoning_tokens", 0))
 
-    # Reasoning is billed at the selected tier's reasoning rate for tiered models,
-    # else at the service-tier-aware per-reasoning-token rate - this mirrors how the
-    # total completion cost is computed, so the breakdown can never diverge from it.
-    tiered_reasoning_rate: Final = _get_tiered_reasoning_rate(model_info=model_info, usage=usage)
-    reasoning_rate: Final = (
-        tiered_reasoning_rate
-        if tiered_reasoning_rate is not None
-        else _resolve_reasoning_token_cost(
-            model_info=model_info,
-            service_tier=service_tier,
-            completion_base_cost=completion_base_cost,
-        )
+    reasoning_rate: Final = _resolve_billed_reasoning_rate(
+        model_info=model_info,
+        usage=usage,
+        service_tier=service_tier,
+        completion_base_cost=completion_base_cost,
+        current_time=billing_time,
     )
     reasoning_cost = float(reasoning_tokens) * reasoning_rate
 

litellm/litellm_core_utils/prompt_templates/common_utils.py

@@ -1554,6 +1554,22 @@ def with_prompt_cache_breakpoint(target: _MarkedT, marker: object) -> _MarkedT:
     return cast(_MarkedT, marked)  # cast-ok: same block shape as the input plus the marker key
 
 
+LITELLM_INTERNAL_MESSAGE_FIELDS: Final = frozenset({"thinking_blocks", "reasoning_content", "provider_specific_fields"})
+
+
+def strip_litellm_internal_message_fields(message: AllMessageValues) -> AllMessageValues:
+    """Drop the fields litellm attaches to assistant messages (e.g. when translating Anthropic thinking
+    blocks) that OpenAI-compatible endpoints with strict schemas reject as extra inputs."""
+    if LITELLM_INTERNAL_MESSAGE_FIELDS.isdisjoint(message):
+        return message
+    return cast(  # cast-ok: same TypedDict minus internal keys
+        AllMessageValues,
+        {  # mutable-ok: provider transforms mutate message dicts in place downstream
+            key: value for key, value in message.items() if key not in LITELLM_INTERNAL_MESSAGE_FIELDS
+        },
+    )
+
+
 def filter_value_from_dict(dictionary: dict, key: str, depth: int = 0) -> Any:
     """
     Filters a value from a dictionary

litellm/litellm_core_utils/secret_redaction.py

@@ -11,7 +11,7 @@ from typing import Final
 
 from litellm.constants import MINIMUM_CUSTOM_KEY_LENGTH
 
-_REDACTED: Final = "REDACTED"
+REDACTED: Final = "REDACTED"
 
 
 def _build_secret_patterns() -> "re.Pattern[str]":
@@ -89,7 +89,7 @@ _SECRET_RE: Final = _build_secret_patterns()
 
 def redact_string(value: str) -> str:
     """Scrub known secret/credential patterns from *value* and return the result."""
-    return _SECRET_RE.sub(_REDACTED, value)
+    return _SECRET_RE.sub(REDACTED, value)
 
 
 _UNIX_SYSTEM_PATH: Final = r"/(?:etc|var|opt|usr|home|root|private|Users|tmp|mnt|srv)/[^\s'\"\)\]}>,]+"
@@ -110,7 +110,7 @@ def redact_internal_details(value: str) -> str:
     on top of redact_string(). For client-facing messages only: server logs keep this detail."""
     marker_index: Final = value.find(_TRACEBACK_MARKER)
     without_traceback: Final = value[:marker_index].rstrip() if marker_index != -1 else value
-    return _INTERNAL_DETAIL_RE.sub(_REDACTED, redact_string(without_traceback))
+    return _INTERNAL_DETAIL_RE.sub(REDACTED, redact_string(without_traceback))
 
 
 def redact_structured_value(key: str | None, value: str) -> str:
@@ -126,4 +126,4 @@ def redact_structured_value(key: str | None, value: str) -> str:
     if scrubbed != value or key is None:
         return scrubbed
     rendered: Final = f"'{key}': '{value}'"
-    return _REDACTED if redact_string(rendered) != rendered else value
+    return REDACTED if redact_string(rendered) != rendered else value

litellm/litellm_core_utils/sensitive_data_masker.py

@@ -1,9 +1,10 @@
-from collections.abc import Mapping
+from collections.abc import Mapping, Sequence
 from typing import Any, Final
 
 from pydantic import BaseModel
 
-from litellm.constants import DEFAULT_MAX_RECURSE_DEPTH_SENSITIVE_DATA_MASKER
+from litellm.constants import DEFAULT_MAX_RECURSE_DEPTH, DEFAULT_MAX_RECURSE_DEPTH_SENSITIVE_DATA_MASKER
+from litellm.litellm_core_utils.secret_redaction import REDACTED
 
 
 class SensitiveDataMasker:
@@ -214,6 +215,46 @@ def mask_sensitive_keys(data: dict[str, Any], sensitive_fields: set[str]) -> dic
     return masked
 
 
+def redact_credentials_in_payload(data: Mapping[str, object]) -> Mapping[str, object]:
+    """Return a copy of ``data`` where every value under a credential-named key is
+    replaced by the shared ``REDACTED`` marker, nested mappings are recursed into,
+    and every other value is preserved by identity.
+
+    Sensitive-key detection is delegated to the shared :class:`SensitiveDataMasker`,
+    so the credential names stay in one place. Unlike
+    :func:`mask_credentials_in_payload`, no prefix or suffix of the secret survives
+    and non-string secrets are covered too, which is what a payload rendered
+    straight to stdout needs. ``None`` is preserved so an unset credential still
+    reads as unset, and lists and tuples are rebuilt element by element so a
+    credential nested inside one is caught as well. The walk is bounded only to stop
+    runaway recursion, and a container sitting at that bound is replaced wholesale
+    rather than passed through, so burying a credential deeper than the walk goes
+    hides it instead of exposing it.
+    """
+    return _redact_mapping(data, 0)
+
+
+def _redact_mapping(data: Mapping[str, object], depth: int) -> Mapping[str, object]:
+    return {key: _redact_entry(key, value, depth) for key, value in data.items()}
+
+
+def _redact_entry(key: str, value: object, depth: int) -> object:
+    if value is not None and _default_masker.is_sensitive_key(key):
+        return REDACTED
+    if not isinstance(value, (Mapping, list, tuple)):
+        return value
+    if depth >= DEFAULT_MAX_RECURSE_DEPTH:
+        return REDACTED
+    if isinstance(value, Mapping):
+        return _redact_mapping(value, depth + 1)
+    return _redact_sequence(value, depth + 1)
+
+
+def _redact_sequence(values: Sequence[object], depth: int) -> Sequence[object]:
+    redacted: Final = tuple(_redact_entry("", item, depth) for item in values)
+    return redacted if isinstance(values, tuple) else list(redacted)
+
+
 # Usage example:
 """
 masker = SensitiveDataMasker()

litellm/litellm_core_utils/streaming_handler.py

@@ -2337,6 +2337,9 @@ class CustomStreamWrapper:
         else:
             self.sent_last_chunk = True
             processed_chunk: Final = self.finish_reason_handler()
+            if self.stream_options is None:
+                usage: Final = calculate_total_usage(chunks=self.chunks)
+                processed_chunk._hidden_params["usage"] = usage  # pyright: ignore[reportPrivateUsage]  # sync parity
             # see sync __next__'s sibling branch: deliberately do NOT restore
             # here - this chunk is still this call's own data, and restoring
             # before returning it would corrupt the caller's own log

litellm/llms/anthropic/experimental_pass_through/adapters/handler.py

@@ -21,6 +21,7 @@ from litellm.llms.anthropic.experimental_pass_through.context_management import
 )
 from litellm.llms.anthropic.experimental_pass_through.utils import (
     is_reasoning_auto_summary_enabled,
+    litellm_logging_obj_from_kwargs,
     local_model_name,
 )
 from litellm.types.llms.anthropic_messages.anthropic_response import (
@@ -621,6 +622,7 @@ class LiteLLMMessagesToCompletionTransformationHandler:
                 tool_name_mapping=tool_name_mapping,
                 polyfill_result=polyfill_result,
                 is_async=True,
+                litellm_logging_obj=litellm_logging_obj_from_kwargs(kwargs),
             )
             if transformed_stream is not None:
                 return transformed_stream
@@ -755,6 +757,7 @@ class LiteLLMMessagesToCompletionTransformationHandler:
                 tool_name_mapping=tool_name_mapping,
                 polyfill_result=polyfill_result,
                 is_async=False,
+                litellm_logging_obj=litellm_logging_obj_from_kwargs(kwargs),
             )
             if transformed_stream is not None:
                 return transformed_stream

litellm/llms/anthropic/experimental_pass_through/adapters/streaming_iterator.py

@@ -31,6 +31,7 @@ from litellm.types.llms.anthropic import (
 from litellm.types.utils import AdapterCompletionStreamWrapper, Delta
 
 if TYPE_CHECKING:
+    from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObject
     from litellm.types.utils import ModelResponseStream
 
 
@@ -287,12 +288,16 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper):
         applied_edits: list[AppliedEdit] | None = None,
         compaction_block: CompactionBlock | None = None,
         iterations_usage: list[UsageIteration] | None = None,
+        litellm_logging_obj: "LiteLLMLoggingObject | None" = None,
     ):
         # Wrap the upstream stream so chunks that carry both content and a
         # finish_reason (fake-streamed providers) are split into two — see
         # _CombinedChunkSplitter.
         super().__init__(_CombinedChunkSplitter(completion_stream))
         self.model = model
+        self._message_id: str = f"msg_{uuid.uuid4()}"
+        if litellm_logging_obj is not None:
+            litellm_logging_obj.record_streamed_anthropic_message_id(self._message_id)
         # Mapping of truncated tool names to original names (for OpenAI's 64-char limit)
         self.tool_name_mapping = tool_name_mapping or {}
         # Polyfill applied_edits on final message_delta.
@@ -507,7 +512,7 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper):
                     {
                         "type": "message_start",
                         "message": {
-                            "id": f"msg_{uuid.uuid4()}",
+                            "id": self._message_id,
                             "type": "message",
                             "role": "assistant",
                             "content": [],
@@ -741,7 +746,7 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper):
                     {
                         "type": "message_start",
                         "message": {
-                            "id": f"msg_{uuid.uuid4()}",
+                            "id": self._message_id,
                             "type": "message",
                             "role": "assistant",
                             "content": [],

litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py

@@ -174,6 +174,7 @@ from litellm.types.utils import Choices, ModelResponse, StreamingChoices, Usage
 from .streaming_iterator import AnthropicStreamWrapper
 
 if TYPE_CHECKING:
+    from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObject
     from litellm.types.llms.anthropic import ContentBlockContentBlockDict
 
 ToolResultContent: TypeAlias = str | list[ToolMessageContentPart]
@@ -264,6 +265,7 @@ class AnthropicAdapter:
         tool_name_mapping: dict[str, str] | None = None,
         polyfill_result: PolyfillResult | None = None,
         is_async: bool = True,
+        litellm_logging_obj: "LiteLLMLoggingObject | None" = None,
     ) -> AsyncIterator[bytes] | Iterator[bytes] | None:
         """
         Translate OpenAI streaming response to Anthropic format.
@@ -290,6 +292,7 @@ class AnthropicAdapter:
             applied_edits=applied_edits,
             compaction_block=compaction_block,
             iterations_usage=iterations_usage,
+            litellm_logging_obj=litellm_logging_obj,
         )
         # Return the SSE-wrapped version for proper event formatting.
         if is_async:

litellm/llms/anthropic/experimental_pass_through/messages/utils.py

@@ -6,6 +6,7 @@ from litellm.types.llms.anthropic import AnthropicMessagesRequestOptionalParams
 from litellm.types.llms.anthropic_messages.anthropic_response import (
     AnthropicMessagesResponse,
 )
+from litellm.types.llms.openai import ChatCompletionSystemMessage
 
 if TYPE_CHECKING:
     from litellm.exceptions import ContentPolicyViolationError
@@ -36,6 +37,16 @@ def safeguard_refusal_error(model: str, stop_details: Mapping[str, object]) -> "
     )
 
 
+def anthropic_system_to_openai_message(system: object) -> ChatCompletionSystemMessage | None:
+    """
+    Return the Anthropic Messages top-level ``system`` (a string or a list of text
+    blocks) as an OpenAI-style system message, or None when the request has none.
+    """
+    if not isinstance(system, (str, list)) or not system:
+        return None
+    return ChatCompletionSystemMessage(role="system", content=system)
+
+
 @lru_cache(maxsize=1)
 def _anthropic_messages_optional_param_keys() -> frozenset[str]:
     """

litellm/llms/anthropic/experimental_pass_through/responses_adapters/handler.py

@@ -20,7 +20,7 @@ from litellm.types.llms.anthropic_messages.anthropic_response import (
 )
 from litellm.types.llms.openai import ResponsesAPIResponse
 
-from ..utils import local_model_name
+from ..utils import litellm_logging_obj_from_kwargs, local_model_name
 from .streaming_iterator import AnthropicResponsesStreamWrapper
 from .transformation import LiteLLMAnthropicToResponsesAPIAdapter
 
@@ -186,7 +186,9 @@ class LiteLLMMessagesToResponsesAPIHandler:
 
         if stream:
             wrapper: Final = AnthropicResponsesStreamWrapper(
-                responses_stream=result, model=local_model_name(model, kwargs.get("custom_llm_provider"))
+                responses_stream=result,
+                model=local_model_name(model, kwargs.get("custom_llm_provider")),
+                litellm_logging_obj=litellm_logging_obj_from_kwargs(responses_kwargs),
             )
             return wrapper.async_anthropic_sse_wrapper()
 
@@ -266,7 +268,9 @@ class LiteLLMMessagesToResponsesAPIHandler:
 
         if stream:
             wrapper: Final = AnthropicResponsesStreamWrapper(
-                responses_stream=result, model=local_model_name(model, kwargs.get("custom_llm_provider"))
+                responses_stream=result,
+                model=local_model_name(model, kwargs.get("custom_llm_provider")),
+                litellm_logging_obj=litellm_logging_obj_from_kwargs(responses_kwargs),
             )
             return wrapper.async_anthropic_sse_wrapper()
 

litellm/llms/anthropic/experimental_pass_through/responses_adapters/streaming_iterator.py

@@ -4,7 +4,7 @@ import json
 import traceback
 from collections import deque
 from collections.abc import AsyncIterator, Mapping
-from typing import Any, Final
+from typing import TYPE_CHECKING, Any, Final
 
 from litellm import verbose_logger
 from litellm._uuid import uuid
@@ -12,6 +12,9 @@ from litellm.types.llms.anthropic_messages.anthropic_response import AnthropicUs
 
 from .transformation import LiteLLMAnthropicToResponsesAPIAdapter
 
+if TYPE_CHECKING:
+    from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObject
+
 
 class AnthropicResponsesStreamWrapper:
     """
@@ -31,10 +34,13 @@ class AnthropicResponsesStreamWrapper:
         self,
         responses_stream: Any,
         model: str,
+        litellm_logging_obj: "LiteLLMLoggingObject | None" = None,
     ) -> None:
         self.responses_stream = responses_stream
         self.model = model
         self._message_id: str = f"msg_{uuid.uuid4()}"
+        if litellm_logging_obj is not None:
+            litellm_logging_obj.record_streamed_anthropic_message_id(self._message_id)
         self._current_block_index: int = -1
         # Map item_id -> content_block_index so we can stop the right block later
         self._item_id_to_block_index: dict[str, int] = {}

litellm/llms/anthropic/experimental_pass_through/utils.py

@@ -1,11 +1,14 @@
 import os
 from collections.abc import Mapping
 from types import MappingProxyType
-from typing import Final
+from typing import TYPE_CHECKING, Final
 
 import litellm
 from litellm.types.utils import ModelInfo
 
+if TYPE_CHECKING:
+    from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObject
+
 OPENAI_MAX_PROMPT_CACHE_KEY_LENGTH: Final = 64
 
 _EFFORT_DEGRADATION_CHAIN: Final[Mapping[str, tuple[str, ...]]] = MappingProxyType(
@@ -24,6 +27,14 @@ def prompt_cache_key_from_user_id(user_id: object) -> str | None:
     return str(user_id)[:OPENAI_MAX_PROMPT_CACHE_KEY_LENGTH] or None
 
 
+def litellm_logging_obj_from_kwargs(kwargs: Mapping[str, object]) -> "LiteLLMLoggingObject | None":
+    """The logging object the bridged call logs through, when the caller supplied one."""
+    from litellm.litellm_core_utils.litellm_logging import Logging
+
+    candidate: Final = kwargs.get("litellm_logging_obj")
+    return candidate if isinstance(candidate, Logging) else None
+
+
 def local_model_name(model: str, custom_llm_provider: object) -> str:
     """The id the provider itself knows, for reporting back to the caller in ``message_start``."""
     return model.removeprefix(f"{custom_llm_provider}/") if isinstance(custom_llm_provider, str) else model

litellm/llms/azure/chat/gpt_5_transformation.py

@@ -7,6 +7,7 @@ from litellm.exceptions import UnsupportedParamsError
 from litellm.llms.openai.chat.gpt_5_transformation import (
     OpenAIGPT5Config,
     _get_effort_level,
+    is_gpt_reasoning_series_name,
 )
 from litellm.types.llms.openai import AllMessageValues
 
@@ -35,26 +36,7 @@ class AzureOpenAIGPT5Config(AzureOpenAIConfig, OpenAIGPT5Config):
 
     @classmethod
     def is_model_gpt_5_model(cls, model: str) -> bool:
-        """Check if the Azure model string refers to a gpt-5 variant.
-
-        Accepts both explicit gpt-5 model names and the ``gpt5_series/`` prefix
-        used for manual routing.
-        """
-        # The gpt-5-chat* family (gpt-5-chat, gpt-5-chat-latest, gpt-5-chat-2025-08-07,
-        # …) are regular chat models: they support temperature and tool_choice but NOT
-        # reasoning_effort.  They must NOT be routed through the GPT-5 reasoning path.
-        #
-        # Versioned chat models such as gpt-5.3-chat and gpt-5.1-chat ARE reasoning
-        # models and must stay on the GPT-5 path.  The distinguishing feature is that
-        # the gpt-5-chat family has a literal "-chat" immediately after "gpt-5"
-        # (i.e. "gpt-5-chat…"), while versioned chat models interpose a minor version
-        # number (i.e. "gpt-5.<digit>-chat").
-        #
-        # Using a startswith("gpt-5-chat") prefix check on the normalized name (rather
-        # than a substring check) makes this boundary explicit and avoids any ambiguity
-        # if future model names coincidentally contain "gpt-5-chat" as an interior run.
-        _normalized: Final = model.split("/")[-1]  # strip provider prefix, e.g. "azure/"
-        return ("gpt-5" in model and not _normalized.startswith("gpt-5-chat")) or "gpt5_series" in model
+        return is_gpt_reasoning_series_name(model) or "gpt5_series" in model
 
     def get_supported_openai_params(self, model: str) -> list[str]:
         """Get supported parameters for Azure OpenAI GPT-5 models.