Merge remote-tracking branch 'berri/litellm_internal_staging' into litellm_batch_ui_logs

.github/workflows/test-litellm-ui-build.yml

@@ -19,9 +19,6 @@ jobs:
   build-ui:
     runs-on: ubuntu-latest
     timeout-minutes: 10
-    defaults:
-      run:
-        working-directory: ui/litellm-dashboard
 
     steps:
       - name: Checkout repository
@@ -35,18 +32,11 @@ jobs:
         with:
           category: ui
 
-      - name: Setup Node.js
+      # Built through the image stage rather than the checkout, because the
+      # stage copies ui/litellm-dashboard/ alone: an import reaching above the
+      # dashboard root resolves in a checkout and fails in every image we ship.
+      # Dockerfile, docker/Dockerfile.non_root and ui/Dockerfile share this
+      # stage verbatim, so building one covers all three.
+      - name: Build the dashboard as the shipped images build it
         if: steps.changes.outputs.decision != 'skip'
-        uses: actions/setup-node@a0853c24544627f65ddf259abe73b1d18a591444 # v5.0.0
-        with:
-          node-version-file: ui/litellm-dashboard/.nvmrc
-          cache: "npm"
-          cache-dependency-path: ui/litellm-dashboard/package-lock.json
-
-      - name: Install dependencies
-        if: steps.changes.outputs.decision != 'skip'
-        run: npm ci
-
-      - name: Build
-        if: steps.changes.outputs.decision != 'skip'
-        run: npm run build
+        run: docker build --target ui-builder -f Dockerfile .

.github/workflows/test-rust.yml

@@ -74,12 +74,19 @@ jobs:
       - name: Run Clippy with Bedrock auth
         run: cargo clippy -p litellm-core --all-targets --features bedrock-auth --locked -- -D warnings
 
+      - name: Run Clippy with all gateway features
+        run: cargo clippy -p litellm-ai-gateway --all-targets --all-features --locked -- -D warnings
+
       - name: Run Rust tests
         run: cargo test --workspace --locked
 
       - name: Run core tests with Bedrock auth
         run: cargo test -p litellm-core --features bedrock-auth --locked
 
+      # Not --all-features: python-config links libpython, which this job does not install.
+      - name: Run gateway tests with the server feature
+        run: cargo test -p litellm-ai-gateway --features server --locked
+
   release-wheel:
     name: release wheel
     runs-on: ubuntu-latest

.github/workflows/test-unit.yml

@@ -151,6 +151,7 @@ jobs:
               tests/test_litellm/proxy/google_endpoints
               tests/test_litellm/proxy/openai_files_endpoint
               tests/test_litellm/proxy/batches_endpoints
+              tests/test_litellm/proxy/container_endpoints
               tests/test_litellm/proxy/fine_tuning_endpoints
               tests/test_litellm/proxy/vector_store_files_endpoints
               tests/test_litellm/proxy/video_endpoints

basedpyright-code-budget.json

@@ -3,7 +3,7 @@
     "limit": 14074
   },
   "reportArgumentType": {
-    "limit": 2215
+    "limit": 2214
   },
   "reportAssignmentType": {
     "limit": 319
@@ -42,7 +42,7 @@
     "limit": 12
   },
   "reportIndexIssue": {
-    "limit": 25
+    "limit": 24
   },
   "reportInvalidTypeForm": {
     "limit": 34

docker/build_from_pip/Dockerfile.build_from_pip

@@ -55,7 +55,10 @@ RUN HOME=/opt/prisma XDG_CACHE_HOME=/opt/prisma/.cache PRISMA_BINARY_CACHE_DIR=/
 
 ENV PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries
 
+COPY docker/prod_entrypoint.sh /app/docker/prod_entrypoint.sh
+RUN sed -i 's/\r$//' /app/docker/prod_entrypoint.sh && chmod +x /app/docker/prod_entrypoint.sh
+
 EXPOSE 4000/tcp
 
-ENTRYPOINT ["litellm"]
+ENTRYPOINT ["/app/docker/prod_entrypoint.sh"]
 CMD ["--port", "4000"]

docker/component_entrypoint.sh

@@ -1,8 +1,10 @@
 #!/bin/sh
 
-if [ "$USE_DDTRACE" = "true" ]; then
-    export DD_TRACE_OPENAI_ENABLED="False"
-    exec ddtrace-run "$@"
-fi
+case "$USE_DDTRACE" in
+    [Tt][Rr][Uu][Ee])
+        export DD_TRACE_OPENAI_ENABLED="False"
+        exec ddtrace-run "$@"
+        ;;
+esac
 
 exec "$@"

docker/prod_entrypoint.sh

@@ -1,8 +1,10 @@
 #!/bin/sh
 
-if [ "$USE_DDTRACE" = "true" ]; then
-    export DD_TRACE_OPENAI_ENABLED="False"
-    exec ddtrace-run litellm "$@"
-else
-    exec litellm "$@"
-fi
+case "$USE_DDTRACE" in
+    [Tt][Rr][Uu][Ee])
+        export DD_TRACE_OPENAI_ENABLED="False"
+        exec ddtrace-run litellm "$@"
+        ;;
+esac
+
+exec litellm "$@"

enterprise/litellm_enterprise/proxy/audit_logging_endpoints.py

@@ -48,6 +48,18 @@ def _build_json_field_or_condition(json_key: str, value: str) -> dict[str, objec
     }
 
 
+def _build_search_condition(search: str) -> dict[str, object]:
+    """Match a row whose id, changed_by, object_id, or changed_by_api_key equals the search value."""
+    return {
+        "OR": (
+            {"id": search},
+            {"changed_by": search},
+            {"object_id": search},
+            {"changed_by_api_key": search},
+        )
+    }
+
+
 @router.get(
     "/audit",
     tags=["Audit Logging"],
@@ -83,6 +95,10 @@ async def get_audit_logs(
         None,
         description="Filter by token (key hash) present in before_value or updated_values JSON (PostgreSQL only)",
     ),
+    search: str | None = Query(
+        None,
+        description="Match a row whose id, object_id, changed_by, or changed_by_api_key equals this value",
+    ),
     # Sorting parameters
     sort_by: str | None = Query(
         None,
@@ -118,6 +134,11 @@ async def get_audit_logs(
         *([_build_json_field_or_condition("token", object_key_hash)] if object_key_hash else []),
     ]
 
+    and_conditions: Final[tuple[dict[str, object], ...]] = (
+        *json_field_conditions,
+        *((_build_search_condition(search),) if search else ()),
+    )
+
     # Build filter conditions
     where_conditions: Final[dict[str, object]] = {
         **({"changed_by": changed_by} if changed_by else {}),
@@ -126,14 +147,14 @@ async def get_audit_logs(
         **({"table_name": table_name} if table_name else {}),
         **({"object_id": object_id} if object_id else {}),
         **({"updated_at": date_filter} if start_date or end_date else {}),
-        **({"AND": json_field_conditions} if json_field_conditions else {}),
+        **({"AND": and_conditions} if and_conditions else {}),
     }
 
     order_by: Final[dict[str, str]] = (
         {sort_by: sort_order} if sort_by and isinstance(sort_by, str) else {"updated_at": sort_order}
     )
 
-    audit_log_table: Final[TableActions["prisma_models.LiteLLM_AuditLog"]] = AuditLogRepository(prisma_client).table
+    audit_log_table: Final[TableActions[prisma_models.LiteLLM_AuditLog]] = AuditLogRepository(prisma_client).table
 
     # Get paginated results
     audit_logs: Final = await audit_log_table.find_many(
@@ -195,7 +216,7 @@ async def get_audit_log_by_id(
             detail={"message": CommonProxyErrors.db_not_connected_error.value},
         )
 
-    audit_log_table: Final[TableActions["prisma_models.LiteLLM_AuditLog"]] = AuditLogRepository(prisma_client).table
+    audit_log_table: Final[TableActions[prisma_models.LiteLLM_AuditLog]] = AuditLogRepository(prisma_client).table
 
     # Get the audit log by ID
     audit_log: Final = await audit_log_table.find_unique(where={"id": id})

enterprise/litellm_enterprise/proxy/hooks/managed_files.py

@@ -50,10 +50,12 @@ from litellm.proxy.openai_files_endpoints.common_utils import (
     MAX_FILE_LIST_LIMIT,
     _is_base64_encoded_unified_file_id,
     apply_unified_file_ids,
+    decode_model_from_file_id,
     ensure_batch_response_managed_file_ids,
     get_batch_id_from_unified_batch_id,
     get_content_type_from_file_object,
     get_model_id_from_unified_batch_id,
+    get_original_file_id,
     map_raw_file_ids_to_unified,
     normalize_mime_type_for_provider,
     resolve_managed_output_file_model_name,
@@ -446,6 +448,103 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
             detail=f"Object not found: {unified_object_id}",
         )
 
+    async def enforce_batch_object_access(
+        self, object_id: str, user_api_key_dict: UserAPIKeyAuth
+    ) -> None:
+        """Deny access to a provider-format batch id owned by another caller.
+
+        Ids with no ownership row (batches created before ownership tracking,
+        or directly on the provider account) stay accessible so pass-through
+        reads keep working.
+        """
+        if self.prisma_client is None:
+            return
+        managed_object = (
+            await self.prisma_client.db.litellm_managedobjecttable.find_first(
+                where={"OR": [{"unified_object_id": object_id}, {"model_object_id": object_id}]}
+            )
+        )
+        if managed_object is None:
+            return
+        if not can_access_resource(
+            user_api_key_dict=user_api_key_dict,
+            created_by=managed_object.created_by,
+            resource_team_id=managed_object.team_id,
+        ):
+            raise HTTPException(
+                status_code=403,
+                detail=f"User {user_api_key_dict.user_id} does not have access to the object {object_id}",
+            )
+
+    async def enforce_provider_file_access(
+        self, file_id: str, user_api_key_dict: UserAPIKeyAuth
+    ) -> None:
+        """Deny access to a provider-format file id owned by another caller.
+
+        Ownership rows for provider-format ids are written when a managed
+        batch's output/error files are first synced; ids with no row stay
+        accessible so pass-through reads keep working.
+        """
+        if self.prisma_client is None:
+            return
+        managed_file = (
+            await self.prisma_client.db.litellm_managedfiletable.find_first(
+                where={"OR": [{"unified_file_id": file_id}, {"flat_model_file_ids": {"has": file_id}}]}
+            )
+        )
+        if managed_file is None:
+            return
+        if not can_access_resource(
+            user_api_key_dict=user_api_key_dict,
+            created_by=managed_file.created_by,
+            resource_team_id=managed_file.team_id,
+        ):
+            raise HTTPException(
+                status_code=403,
+                detail=f"User {user_api_key_dict.user_id} does not have access to the file {file_id}",
+            )
+
+    async def store_batch_output_file_ownership(
+        self, response: LiteLLMBatch, litellm_parent_otel_span: Optional[Span]
+    ) -> None:
+        """Record ownership rows for a batch's provider-format output/error
+        file ids, inherited from the owning batch row (never the caller), so
+        file reads can be isolation-checked."""
+        provider_file_ids = tuple(
+            file_id
+            for file_id in (
+                getattr(response, "output_file_id", None),
+                getattr(response, "error_file_id", None),
+            )
+            if file_id and not _is_base64_encoded_unified_file_id(file_id)
+        )
+        if not provider_file_ids:
+            return
+        if self.prisma_client is None:
+            return
+        batch_row = (
+            await self.prisma_client.db.litellm_managedobjecttable.find_first(
+                where={"unified_object_id": response.id}
+            )
+        )
+        if batch_row is None or (
+            batch_row.created_by is None and batch_row.team_id is None
+        ):
+            return
+        owner_identity = UserAPIKeyAuth(
+            user_id=batch_row.created_by, team_id=batch_row.team_id
+        )
+        for file_id in provider_file_ids:
+            model_name = decode_model_from_file_id(file_id)
+            raw_file_id = get_original_file_id(file_id)
+            await self.store_unified_file_id(
+                file_id=file_id,
+                file_object=None,
+                litellm_parent_otel_span=litellm_parent_otel_span,
+                model_mappings={model_name: raw_file_id} if model_name else {},
+                user_api_key_dict=owner_identity,
+            )
+
     async def list_user_batches(
         self,
         user_api_key_dict: UserAPIKeyAuth,
@@ -632,6 +731,10 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
                     status_code=403,
                     detail=f"User {user_api_key_dict.user_id} does not have access to the file {retrieve_file_id}",
                 )
+        if retrieve_file_id:
+            await self.enforce_provider_file_access(
+                retrieve_file_id, user_api_key_dict
+            )
         return False
 
     async def check_file_ids_access(self, file_ids: List[str], user_api_key_dict: UserAPIKeyAuth) -> None:
@@ -784,6 +887,8 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
                     )
                 data["model"] = potential_model_id
                 data[accessor_key] = get_batch_id_from_unified_batch_id(potential_llm_object_id)
+            elif retrieve_object_id and accessor_key == "batch_id":
+                await self.enforce_batch_object_access(retrieve_object_id, user_api_key_dict)
         elif call_type == CallTypes.acreate_fine_tuning_job.value:
             input_file_id = cast(Optional[str], data.get("training_file"))
             if input_file_id:
@@ -1316,7 +1421,13 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
                 user_api_key_dict=user_api_key_dict,
                 request_tags=request_tags_from_metadata(request_metadata if isinstance(request_metadata, dict) else {}),
                 persist_attribution=is_batch_create,
+                create_if_missing=is_batch_create,
             )
+            if not is_batch_create:
+                await self.store_batch_output_file_ownership(
+                    response=response,
+                    litellm_parent_otel_span=user_api_key_dict.parent_otel_span,
+                )
 
             # Only record batch creation metric on actual create (not retrieve/cancel).
             # unified_file_id in _hidden_params is only set by the create_batch endpoint.

helm/litellm/templates/_helpers.tpl

@@ -428,3 +428,16 @@ envFrom:
 {{- end }}
 {{- end }}
 {{- end -}}
+
+{{/*
+ingress-nginx's admission webhook rejects a dot in an Exact or Prefix path
+(strict-validate-path-type) and serves ImplementationSpecific as a plain
+prefix location, so a dotted path takes that type there.
+*/}}
+{{- define "litellm.ingress.pathType" -}}
+{{- if and (eq .controller "nginx") (contains "." .path) -}}
+ImplementationSpecific
+{{- else -}}
+{{- .pathType -}}
+{{- end -}}
+{{- end -}}

helm/litellm/templates/ingress.yaml

@@ -5,6 +5,10 @@
 {{- $gatewayPort := .Values.gateway.service.port -}}
 {{- $backendPort := .Values.backend.service.port -}}
 {{- $uiPort := .Values.ui.service.port -}}
+{{- $controller := .Values.ingress.controller | default "alb" -}}
+{{- if not (has $controller (list "alb" "nginx")) }}
+{{- fail (printf "ingress.controller: unknown controller %q, expected one of alb, nginx" $controller) }}
+{{- end }}
 {{/*
   Backends addressable from ingress.extraPaths, keyed by the `service` field.
 */}}
@@ -27,10 +31,11 @@
   /litellm-asset-prefix, so without /*.txt they fall to the backend catch-all
   → 404 → client-side navigation never settles and the login flow spins in an
   infinite redirect loop (/ ⇄ /ui/login). ui/nginx.conf already serves *.txt
-  from the export; the rule only routes the request to it. Needs an ingress
-  controller whose ImplementationSpecific path is a wildcard pattern
-  (AWS ALB: `*` = 0+ chars); this chart targets the AWS Load Balancer
-  Controller.
+  from the export; the rule only routes the request to it. It needs an
+  ingress controller whose ImplementationSpecific path is a wildcard pattern
+  (AWS ALB: `*` = 0+ chars), so it is rendered for ingress.controller=alb
+  only: ingress-nginx serves ImplementationSpecific as a literal prefix
+  location, where /*.txt can never match.
 */}}
 {{- $uiPaths := list
     (dict "path" "/" "pathType" "Exact")
@@ -38,8 +43,10 @@
     (dict "path" "/litellm-asset-prefix" "pathType" "Prefix")
     (dict "path" "/_next" "pathType" "Prefix")
     (dict "path" "/ui" "pathType" "Prefix")
-    (dict "path" "/*.txt" "pathType" "ImplementationSpecific")
 -}}
+{{- if eq $controller "alb" }}
+{{- $uiPaths = append $uiPaths (dict "path" "/*.txt" "pathType" "ImplementationSpecific") }}
+{{- end }}
 {{/*
   Gateway data-plane prefixes — must mirror gateway/routes/allowlist.py.
   Versioned paths are listed explicitly to avoid routing management routes
@@ -83,12 +90,6 @@
   adding to it.
 */}}
 {{- $builtinPathKeys := list "/test|Exact" "/|Prefix" -}}
-{{- range $uiPaths }}
-{{- $builtinPathKeys = append $builtinPathKeys (printf "%s|%s" .path .pathType) }}
-{{- end }}
-{{- range $gatewayPrefixes }}
-{{- $builtinPathKeys = append $builtinPathKeys (printf "%s|Prefix" .) }}
-{{- end }}
 apiVersion: networking.k8s.io/v1
 kind: Ingress
 metadata:
@@ -115,8 +116,10 @@ spec:
         paths:
           # --- UI (Next.js static export) ---
           {{- range $uiPaths }}
+          {{- $pathType := include "litellm.ingress.pathType" (dict "controller" $controller "path" .path "pathType" .pathType) }}
+          {{- $builtinPathKeys = append $builtinPathKeys (printf "%s|%s" .path $pathType) }}
           - path: {{ .path }}
-            pathType: {{ .pathType }}
+            pathType: {{ $pathType }}
             backend:
               service:
                 name: {{ $uiName }}
@@ -134,8 +137,10 @@ spec:
                 port:
                   number: {{ $gatewayPort }}
           {{- range $gatewayPrefixes }}
+          {{- $pathType := include "litellm.ingress.pathType" (dict "controller" $controller "path" . "pathType" "Prefix") }}
+          {{- $builtinPathKeys = append $builtinPathKeys (printf "%s|%s" . $pathType) }}
           - path: {{ . }}
-            pathType: Prefix
+            pathType: {{ $pathType }}
             backend:
               service:
                 name: {{ $gatewayName }}
@@ -147,10 +152,11 @@ spec:
             Rendered after every built-in path so an entry can never take
             precedence over a default, and before the backend catch-all.
             Position only decides the match on controllers that honour manifest
-            order: the AWS Load Balancer Controller this chart targets sorts
-            Exact paths first and Prefix paths longest-first, but keeps
+            order: the AWS Load Balancer Controller (ingress.controller=alb)
+            sorts Exact paths first and Prefix paths longest-first, but keeps
             ImplementationSpecific paths in manifest order, which is what the
-            /*.txt rule above already depends on.
+            /*.txt rule above already depends on. ingress-nginx ignores order
+            and serves the longest matching location.
           */}}
           {{- range $idx, $extra := .Values.ingress.extraPaths }}
           {{- if not (kindIs "map" $extra) }}
@@ -164,10 +170,11 @@ spec:
           {{- if not $target }}
           {{- fail (printf "ingress.extraPaths[%d] (path %s): unknown service %q, expected one of backend, gateway, ui" $idx $extra.path $service) }}
           {{- end }}
-          {{- $pathType := $extra.pathType | default "Prefix" }}
-          {{- if not (has $pathType (list "Prefix" "Exact" "ImplementationSpecific")) }}
-          {{- fail (printf "ingress.extraPaths[%d] (path %s): unknown pathType %q, expected one of Exact, ImplementationSpecific, Prefix" $idx $extra.path $pathType) }}
+          {{- $requestedPathType := $extra.pathType | default "Prefix" }}
+          {{- if not (has $requestedPathType (list "Prefix" "Exact" "ImplementationSpecific")) }}
+          {{- fail (printf "ingress.extraPaths[%d] (path %s): unknown pathType %q, expected one of Exact, ImplementationSpecific, Prefix" $idx $extra.path $requestedPathType) }}
           {{- end }}
+          {{- $pathType := include "litellm.ingress.pathType" (dict "controller" $controller "path" $extra.path "pathType" $requestedPathType) }}
           {{- if eq $extra.path "/" }}
           {{- fail (printf "ingress.extraPaths[%d]: path / is already routed in both directions, Exact to ui and Prefix to backend, so no pathType leaves a request for an entry here to capture" $idx) }}
           {{- end }}

helm/litellm/tests/ingress_controller_tests.yaml

@@ -0,0 +1,205 @@
+suite: test ingress.controller
+templates:
+  - ingress.yaml
+values:
+  - ./values/required.yaml
+tests:
+  - it: keeps the AWS Load Balancer Controller path types by default
+    set:
+      ingress.enabled: true
+    asserts:
+      - contains:
+          path: spec.rules[0].http.paths
+          content:
+            path: /favicon.ico
+            pathType: Exact
+            backend:
+              service:
+                name: RELEASE-NAME-litellm-ui
+                port:
+                  number: 3000
+      - contains:
+          path: spec.rules[0].http.paths
+          content:
+            path: /eu.assemblyai
+            pathType: Prefix
+            backend:
+              service:
+                name: RELEASE-NAME-litellm-gateway
+                port:
+                  number: 4000
+      - contains:
+          path: spec.rules[0].http.paths
+          content:
+            path: /*.txt
+            pathType: ImplementationSpecific
+            backend:
+              service:
+                name: RELEASE-NAME-litellm-ui
+                port:
+                  number: 3000
+
+  - it: renders no dotted Exact or Prefix path for ingress-nginx, whose admission webhook rejects them
+    set:
+      ingress.enabled: true
+      ingress.controller: nginx
+    asserts:
+      - notMatchRegexRaw:
+          pattern: 'path: /\S*\.\S*\n\s+pathType: (Exact|Prefix)\n'
+      - contains:
+          path: spec.rules[0].http.paths
+          content:
+            path: /favicon.ico
+            pathType: ImplementationSpecific
+            backend:
+              service:
+                name: RELEASE-NAME-litellm-ui
+                port:
+                  number: 3000
+      - contains:
+          path: spec.rules[0].http.paths
+          content:
+            path: /eu.assemblyai
+            pathType: ImplementationSpecific
+            backend:
+              service:
+                name: RELEASE-NAME-litellm-gateway
+                port:
+                  number: 4000
+
+  - it: drops the /*.txt wildcard for ingress-nginx and keeps every other route as is
+    set:
+      ingress.enabled: true
+      ingress.controller: nginx
+    asserts:
+      - notContains:
+          path: spec.rules[0].http.paths
+          content:
+            path: /*.txt
+          any: true
+      - contains:
+          path: spec.rules[0].http.paths
+          content:
+            path: /ui
+            pathType: Prefix
+            backend:
+              service:
+                name: RELEASE-NAME-litellm-ui
+                port:
+                  number: 3000
+      - contains:
+          path: spec.rules[0].http.paths
+          content:
+            path: /test
+            pathType: Exact
+            backend:
+              service:
+                name: RELEASE-NAME-litellm-gateway
+                port:
+                  number: 4000
+      - equal:
+          path: spec.rules[0].http.paths[-1]
+          value:
+            path: /
+            pathType: Prefix
+            backend:
+              service:
+                name: RELEASE-NAME-litellm-backend
+                port:
+                  number: 4001
+
+  - it: rejects an extraPaths entry that repeats a built-in path at the pathType ingress-nginx renders it with
+    set:
+      ingress.enabled: true
+      ingress.controller: nginx
+      ingress.extraPaths:
+        - path: /favicon.ico
+          service: ui
+          pathType: ImplementationSpecific
+    asserts:
+      - failedTemplate:
+          errorMessage: "ingress.extraPaths[0]: path /favicon.ico with pathType ImplementationSpecific is already routed by this chart, and a duplicate would take it over rather than add to it"
+
+  - it: rejects a controller it has no path types for
+    set:
+      ingress.enabled: true
+      ingress.controller: traefik
+    asserts:
+      - failedTemplate:
+          errorMessage: 'ingress.controller: unknown controller "traefik", expected one of alb, nginx'
+
+  - it: rejects an extraPaths entry that repeats a built-in path once ingress-nginx normalizes its pathType
+    set:
+      ingress.enabled: true
+      ingress.controller: nginx
+      ingress.extraPaths:
+        - path: /favicon.ico
+          service: ui
+    asserts:
+      - failedTemplate:
+          errorMessage: "ingress.extraPaths[0]: path /favicon.ico with pathType ImplementationSpecific is already routed by this chart, and a duplicate would take it over rather than add to it"
+
+  - it: renders a dotted extraPaths entry as ImplementationSpecific for ingress-nginx
+    set:
+      ingress.enabled: true
+      ingress.controller: nginx
+      ingress.extraPaths:
+        - path: /eu.assemblyai.custom
+          service: gateway
+        - path: /robots.txt
+          service: ui
+          pathType: Exact
+    asserts:
+      - notMatchRegexRaw:
+          pattern: 'path: "?/\S*\.\S*"?\n\s+pathType: (Exact|Prefix)\n'
+      - contains:
+          path: spec.rules[0].http.paths
+          content:
+            path: /eu.assemblyai.custom
+            pathType: ImplementationSpecific
+            backend:
+              service:
+                name: RELEASE-NAME-litellm-gateway
+                port:
+                  number: 4000
+      - contains:
+          path: spec.rules[0].http.paths
+          content:
+            path: /robots.txt
+            pathType: ImplementationSpecific
+            backend:
+              service:
+                name: RELEASE-NAME-litellm-ui
+                port:
+                  number: 3000
+
+  - it: keeps the requested pathType of a dotted extraPaths entry for the AWS Load Balancer Controller
+    set:
+      ingress.enabled: true
+      ingress.extraPaths:
+        - path: /eu.assemblyai.custom
+          service: gateway
+        - path: /robots.txt
+          service: ui
+          pathType: Exact
+    asserts:
+      - contains:
+          path: spec.rules[0].http.paths
+          content:
+            path: /eu.assemblyai.custom
+            pathType: Prefix
+            backend:
+              service:
+                name: RELEASE-NAME-litellm-gateway
+                port:
+                  number: 4000
+      - contains:
+          path: spec.rules[0].http.paths
+          content:
+            path: /robots.txt
+            pathType: Exact
+            backend:
+              service:
+                name: RELEASE-NAME-litellm-ui
+                port:
+                  number: 3000

helm/litellm/values.yaml

@@ -10,6 +10,18 @@ imagePullSecrets: []
 ingress:
   enabled: false
   className: ""
+  # Which ingress controller serves this Ingress. Controllers disagree on the
+  # pathTypes they accept, so this picks the pathType of the dotted paths, the
+  # built-in ones and any dotted extraPaths entry alike:
+  #   alb    AWS Load Balancer Controller (default): Exact and Prefix paths plus
+  #          the /*.txt wildcard that routes the UI's RSC payloads.
+  #   nginx  ingress-nginx: its admission webhook rejects a dot in an Exact or
+  #          Prefix path (strict-validate-path-type, on by default from v1.12.0
+  #          until v1.12.6 / v1.13.2 allowed dots again), so /favicon.ico and
+  #          /eu.assemblyai render as ImplementationSpecific, which nginx serves
+  #          as a plain prefix location. /*.txt is dropped: nginx has no
+  #          wildcard pathType, so that rule could never match there.
+  controller: alb
   annotations: {}
   host: ""        # optional; if set, becomes the rule's host
   tls: []
@@ -26,7 +38,8 @@ ingress:
   #
   #   path      required; the HTTP path to route
   #   service   which component serves it: gateway (default), backend, or ui
-  #   pathType  Prefix (default), Exact, or ImplementationSpecific
+  #   pathType  Prefix (default), Exact, or ImplementationSpecific; a dotted
+  #             path renders as ImplementationSpecific when controller is nginx
   #
   # The target component only answers paths its own route allowlist keeps, so
   # a path here still has to be one that component serves.

litellm-proxy-extras/litellm_proxy_extras/prisma_toolchain.py

@@ -18,10 +18,15 @@ recoverable one.
 constant: it grows with the number of pending migrations, so a fresh database
 that has to replay every migration this package ships overruns a per-command
 budget sized for the short bookkeeping commands, on a laptop as much as on a
-slow CI runner. The Python ``prisma`` wrapper spawns Node and the schema engine
-as separate children, so killing the wrapper on timeout leaves them running:
-the retry then contends with that orphan for Prisma's advisory lock and cannot
-finish any sooner. Migrate deploy therefore runs under its own budget.
+slow CI runner. Migrate deploy therefore runs under its own budget.
+
+The Python ``prisma`` wrapper spawns Node, which spawns the Rust schema
+engine, so killing only the wrapper on timeout leaves the engine running with
+no parent: it keeps mutating the database after the proxy has given up, holds
+Prisma's advisory lock so every retry and every later boot queues behind it,
+and dies mid-migration once its pipes close, leaving a half-applied ledger row.
+Every Prisma command therefore runs in a process group of its own, and a
+timeout kills the whole group.
 
 All three budgets are overridable so an operator can widen them without a
 release: ``LITELLM_PRISMA_BOOTSTRAP_TIMEOUT`` for the toolchain install,
@@ -35,10 +40,12 @@ the deploy override says otherwise.
 import math
 import os
 import shutil
+import signal
 import subprocess
+from collections.abc import Mapping, Sequence
 from dataclasses import dataclass
 from pathlib import Path
-from typing import Optional
+from typing import IO, Optional, Union
 
 from litellm_proxy_extras._logging import logger
 
@@ -167,6 +174,49 @@ def heal_incomplete_nodeenv_cache() -> bool:
     return True
 
 
+def _kill_process_group(process: "subprocess.Popen[str]") -> None:
+    if os.name == "nt":
+        process.kill()
+        return
+    try:
+        os.killpg(process.pid, signal.SIGKILL)
+    except ProcessLookupError:
+        return
+
+
+def run_prisma(
+    argv: Sequence[str],
+    *,
+    timeout: float,
+    env: Mapping[str, str],
+    stdout: Union[IO[str], int, None] = subprocess.PIPE,
+    stderr: Optional[int] = subprocess.PIPE,
+) -> "subprocess.CompletedProcess[str]":
+    """Run one Prisma CLI command in its own process group, bounded by ``timeout``.
+
+    Raises ``subprocess.TimeoutExpired`` once the budget is spent, after killing
+    the command together with every process it spawned, and
+    ``subprocess.CalledProcessError`` on a non-zero exit. Output is captured as
+    text unless ``stdout``/``stderr`` say otherwise.
+    """
+    with subprocess.Popen(
+        argv,
+        env=env,
+        stdout=stdout,
+        stderr=stderr,
+        text=True,
+        start_new_session=True,
+    ) as process:
+        try:
+            out, err = process.communicate(timeout=timeout)
+        except BaseException:
+            _kill_process_group(process)
+            raise
+    if process.returncode:
+        raise subprocess.CalledProcessError(process.returncode, process.args, out, err)
+    return subprocess.CompletedProcess(process.args, process.returncode, out, err)
+
+
 def ensure_prisma_toolchain(
     prisma_command: str, prisma_env: dict[str, str]
 ) -> ToolchainBootstrap:
@@ -179,14 +229,7 @@ def ensure_prisma_toolchain(
     timeout = prisma_bootstrap_timeout()
     logger.info("Preparing the Prisma CLI toolchain (timeout %ss)", timeout)
     try:
-        subprocess.run(
-            [prisma_command, BOOTSTRAP_ARG],
-            timeout=timeout,
-            check=True,
-            capture_output=True,
-            text=True,
-            env=prisma_env,
-        )
+        run_prisma([prisma_command, BOOTSTRAP_ARG], timeout=timeout, env=prisma_env)
     except subprocess.TimeoutExpired:
         logger.warning(
             "Preparing the Prisma CLI toolchain timed out after %ss. Raise %s "

litellm-proxy-extras/litellm_proxy_extras/replica_identity.py

@@ -16,7 +16,7 @@ import tempfile
 from pathlib import Path
 
 from litellm_proxy_extras._logging import logger
-from litellm_proxy_extras.prisma_toolchain import prisma_command_timeout
+from litellm_proxy_extras.prisma_toolchain import prisma_command_timeout, run_prisma
 
 REPLICA_IDENTITY_FULL_ENV_VAR = "LITELLM_SET_REPLICA_IDENTITY_FULL"
 
@@ -66,7 +66,7 @@ def apply_replica_identity_full(
         with tempfile.TemporaryDirectory(prefix="litellm_replica_identity_") as tmp_dir:
             sql_path = Path(tmp_dir) / "replica_identity_full.sql"
             sql_path.write_text(REPLICA_IDENTITY_FULL_SQL)
-            subprocess.run(
+            run_prisma(
                 [
                     prisma_command,
                     "db",
@@ -77,9 +77,6 @@ def apply_replica_identity_full(
                     schema_path,
                 ],
                 timeout=prisma_command_timeout(),
-                check=True,
-                capture_output=True,
-                text=True,
                 env=prisma_env,
             )
     except subprocess.CalledProcessError as e:

litellm-proxy-extras/litellm_proxy_extras/utils.py

@@ -10,6 +10,7 @@ from dataclasses import dataclass, replace
 from pathlib import Path
 from typing import Optional
 
+from litellm_proxy_extras import prisma_toolchain
 from litellm_proxy_extras._logging import logger
 from litellm_proxy_extras.replica_identity import (
     REPLICA_IDENTITY_FULL_ENV_VAR,
@@ -231,7 +232,7 @@ class ProxyExtrasDBManager:
             # 1. Generate migration SQL file by comparing empty state to current db state
             logger.info("Generating baseline migration...")
             migration_file = init_dir / "migration.sql"
-            subprocess.run(
+            prisma_toolchain.run_prisma(
                 [
                     _get_prisma_command(),
                     "migrate",
@@ -242,14 +243,13 @@ class ProxyExtrasDBManager:
                     "--script",
                 ],
                 stdout=open(migration_file, "w"),
-                check=True,
                 timeout=prisma_command_timeout(),
                 env=prisma_env,
             )
 
             # 3. Mark the migration as applied since it represents current state
             logger.info("Marking baseline migration as applied...")
-            subprocess.run(
+            prisma_toolchain.run_prisma(
                 [
                     _get_prisma_command(),
                     "migrate",
@@ -257,7 +257,6 @@ class ProxyExtrasDBManager:
                     "--applied",
                     "0_init",
                 ],
-                check=True,
                 timeout=prisma_command_timeout(),
                 env=prisma_env,
             )
@@ -286,7 +285,7 @@ class ProxyExtrasDBManager:
         """Mark a specific migration as rolled back"""
         # Set up environment for offline mode if configured
         prisma_env = _get_prisma_env()
-        subprocess.run(
+        prisma_toolchain.run_prisma(
             [
                 _get_prisma_command(),
                 "migrate",
@@ -295,8 +294,6 @@ class ProxyExtrasDBManager:
                 migration_name,
             ],
             timeout=prisma_command_timeout(),
-            check=True,
-            capture_output=True,
             env=prisma_env,
         )
 
@@ -348,11 +345,9 @@ class ProxyExtrasDBManager:
     def _resolve_specific_migration(migration_name: str):
         """Mark a specific migration as applied"""
         prisma_env = _get_prisma_env()
-        subprocess.run(
+        prisma_toolchain.run_prisma(
             [_get_prisma_command(), "migrate", "resolve", "--applied", migration_name],
             timeout=prisma_command_timeout(),
-            check=True,
-            capture_output=True,
             env=prisma_env,
         )
 
@@ -436,7 +431,7 @@ class ProxyExtrasDBManager:
         try:
             logger.info("Generating migration diff between DB and schema.prisma...")
             with open(diff_sql_path, "w") as f:
-                subprocess.run(
+                prisma_toolchain.run_prisma(
                     [
                         _get_prisma_command(),
                         "migrate",
@@ -447,7 +442,6 @@ class ProxyExtrasDBManager:
                         schema_path,
                         "--script",
                     ],
-                    check=True,
                     timeout=prisma_command_timeout(),
                     stdout=f,
                     env=_get_prisma_env(),
@@ -470,7 +464,7 @@ class ProxyExtrasDBManager:
             migration_files = sorted(Path(migrations_dir).glob("*/migration.sql"))
             for mig_file in migration_files:
                 try:
-                    subprocess.run(
+                    prisma_toolchain.run_prisma(
                         [
                             _get_prisma_command(),
                             "db",
@@ -481,9 +475,6 @@ class ProxyExtrasDBManager:
                             schema_path,
                         ],
                         timeout=prisma_command_timeout(),
-                        check=True,
-                        capture_output=True,
-                        text=True,
                         env=_get_prisma_env(),
                     )
                     logger.info(f"Applied migration: {mig_file.parent.name}")
@@ -516,7 +507,7 @@ class ProxyExtrasDBManager:
         applied_ok = False
         try:
             logger.info("Running prisma db execute to apply the migration diff...")
-            result = subprocess.run(
+            result = prisma_toolchain.run_prisma(
                 [
                     _get_prisma_command(),
                     "db",
@@ -527,9 +518,6 @@ class ProxyExtrasDBManager:
                     schema_path,
                 ],
                 timeout=prisma_command_timeout(),
-                check=True,
-                capture_output=True,
-                text=True,
                 env=_get_prisma_env(),
             )
             logger.info(f"prisma db execute stdout: {result.stdout}")
@@ -558,7 +546,7 @@ class ProxyExtrasDBManager:
         for migration_name in migration_names:
             try:
                 logger.info(f"Resolving migration: {migration_name}")
-                subprocess.run(
+                prisma_toolchain.run_prisma(
                     [
                         _get_prisma_command(),
                         "migrate",
@@ -567,9 +555,6 @@ class ProxyExtrasDBManager:
                         migration_name,
                     ],
                     timeout=prisma_command_timeout(),
-                    check=True,
-                    capture_output=True,
-                    text=True,
                     env=_get_prisma_env(),
                 )
                 logger.debug(f"Resolved migration: {migration_name}")
@@ -762,11 +747,12 @@ class ProxyExtrasDBManager:
             original_dir = os.getcwd()
             os.chdir(migrations_dir)
             try:
-                subprocess.run(
+                prisma_toolchain.run_prisma(
                     [_get_prisma_command(), "db", "push", "--accept-data-loss"],
                     timeout=prisma_command_timeout(),
-                    check=True,
                     env=_get_prisma_env(),
+                    stdout=None,
+                    stderr=None,
                 )
                 return True
             except (
@@ -789,12 +775,9 @@ class ProxyExtrasDBManager:
         try:
             while not budget.exhausted:
                 try:
-                    result = subprocess.run(
+                    result = prisma_toolchain.run_prisma(
                         [_get_prisma_command(), "migrate", "deploy"],
                         timeout=deploy_timeout,
-                        check=True,
-                        capture_output=True,
-                        text=True,
                         env=_get_prisma_env(),
                     )
                     logger.info(f"prisma migrate deploy stdout: {result.stdout}")
@@ -1031,12 +1014,9 @@ class ProxyExtrasDBManager:
                     logger.info("Running prisma migrate deploy")
                     try:
                         # Set migrations directory for Prisma
-                        result = subprocess.run(
+                        result = prisma_toolchain.run_prisma(
                             [_get_prisma_command(), "migrate", "deploy"],
                             timeout=prisma_migrate_deploy_timeout(),
-                            check=True,
-                            capture_output=True,
-                            text=True,
                             env=_get_prisma_env(),
                         )
                         logger.info(f"prisma migrate deploy stdout: {result.stdout}")
@@ -1108,7 +1088,7 @@ class ProxyExtrasDBManager:
                                         f"Found failed migration: {failed_migration}, marking as rolled back"
                                     )
                                     # Mark the failed migration as rolled back
-                                    subprocess.run(
+                                    prisma_toolchain.run_prisma(
                                         [
                                             _get_prisma_command(),
                                             "migrate",
@@ -1117,9 +1097,6 @@ class ProxyExtrasDBManager:
                                             failed_migration,
                                         ],
                                         timeout=prisma_command_timeout(),
-                                        check=True,
-                                        capture_output=True,
-                                        text=True,
                                         env=_get_prisma_env(),
                                     )
                                     logger.info(
@@ -1244,10 +1221,12 @@ class ProxyExtrasDBManager:
                     if ProxyExtrasDBManager.spend_logs_is_partitioned():
                         raise RuntimeError(PARTITIONED_SPEND_LOGS_PUSH_ERROR)
                     # Use prisma db push with increased timeout
-                    subprocess.run(
+                    prisma_toolchain.run_prisma(
                         [_get_prisma_command(), "db", "push", "--accept-data-loss"],
                         timeout=prisma_command_timeout(),
-                        check=True,
+                        stdout=None,
+                        stderr=None,
+                        env=_get_prisma_env(),
                     )
                     return True
             except subprocess.TimeoutExpired:

litellm-proxy-extras/tests/test_setup_database_fail_fast.py

@@ -42,7 +42,7 @@ def test_v2_p3018_permission_error_raises_runtime_error(monkeypatch, tmp_path):
         "Error: P3018\nMigration name: 20250326162113_baseline\n"
         "Database error code: 42501\npermission denied for schema public"
     )
-    with patch("subprocess.run", side_effect=_fake_migrate_deploy_failure(1, stderr)):
+    with patch("litellm_proxy_extras.prisma_toolchain.run_prisma", side_effect=_fake_migrate_deploy_failure(1, stderr)):
         with pytest.raises(RuntimeError, match="permission"):
             ProxyExtrasDBManager.setup_database(use_migrate=True, use_v2_resolver=True)
 
@@ -60,7 +60,7 @@ def test_v2_non_idempotent_p3009_raises_runtime_error(monkeypatch, tmp_path):
         "Error: P3009\nMigration `20260101000000_genuinely_broken` failed\n"
         'Reason: syntax error at or near "BRKN" LINE 42'
     )
-    with patch("subprocess.run", side_effect=_fake_migrate_deploy_failure(1, stderr)):
+    with patch("litellm_proxy_extras.prisma_toolchain.run_prisma", side_effect=_fake_migrate_deploy_failure(1, stderr)):
         with pytest.raises(RuntimeError, match="cannot be auto-recovered"):
             ProxyExtrasDBManager.setup_database(use_migrate=True, use_v2_resolver=True)
 
@@ -124,7 +124,7 @@ def test_v1_default_still_calls_resolve_all_migrations(monkeypatch, tmp_path):
     def fake_resolve(*args, **kwargs):
         resolve_called["n"] += 1
 
-    monkeypatch.setattr("subprocess.run", fake_run)
+    monkeypatch.setattr("litellm_proxy_extras.prisma_toolchain.run_prisma", fake_run)
     monkeypatch.setattr(ProxyExtrasDBManager, "_resolve_all_migrations", fake_resolve)
 
     ok = ProxyExtrasDBManager.setup_database(use_migrate=True)  # v2 flag NOT set
@@ -139,7 +139,7 @@ def test_v2_db_push_wraps_subprocess_error_as_runtime_error(monkeypatch, tmp_pat
     (tmp_path / "schema.prisma").write_text("// stub")
 
     stderr = "db push error"
-    with patch("subprocess.run", side_effect=_fake_migrate_deploy_failure(1, stderr)):
+    with patch("litellm_proxy_extras.prisma_toolchain.run_prisma", side_effect=_fake_migrate_deploy_failure(1, stderr)):
         with pytest.raises(RuntimeError, match="prisma db push failed"):
             ProxyExtrasDBManager.setup_database(use_migrate=False, use_v2_resolver=True)
 
@@ -209,7 +209,7 @@ def test_v2_resolve_specific_migration_failure_raises_runtime_error(
         "Error: P3009\nMigration `20260101000000_some_migration` failed\n"
         "relation already exists"
     )
-    with patch("subprocess.run", side_effect=_fake_migrate_deploy_failure(1, stderr)):
+    with patch("litellm_proxy_extras.prisma_toolchain.run_prisma", side_effect=_fake_migrate_deploy_failure(1, stderr)):
         with pytest.raises(
             RuntimeError, match="Failed to mark migration .* as applied"
         ):
@@ -228,7 +228,7 @@ def test_v2_does_not_call_resolve_all_migrations(monkeypatch, tmp_path):
         stdout = "Applied migration.\n"
         stderr = ""
 
-    monkeypatch.setattr("subprocess.run", lambda *a, **kw: FakeResult())
+    monkeypatch.setattr("litellm_proxy_extras.prisma_toolchain.run_prisma", lambda *a, **kw: FakeResult())
 
     resolve_called = {"n": 0}
     monkeypatch.setattr(
@@ -296,7 +296,7 @@ def test_v2_p3018_deadlock_rolls_back_and_retries(monkeypatch, tmp_path):
         "_resolve_specific_migration",
         lambda name: pytest.fail("a deadlocked migration must never be marked applied"),
     )
-    monkeypatch.setattr("subprocess.run", _succeed_after(1, _DEADLOCK_P3018_STDERR))
+    monkeypatch.setattr("litellm_proxy_extras.prisma_toolchain.run_prisma", _succeed_after(1, _DEADLOCK_P3018_STDERR))
 
     ok = ProxyExtrasDBManager.setup_database(use_migrate=True, use_v2_resolver=True)
     assert ok is True
@@ -309,7 +309,7 @@ def test_v2_p3018_persistent_deadlock_exhausts_attempts(monkeypatch, tmp_path):
     monkeypatch.setattr(ProxyExtrasDBManager, "_roll_back_migration", lambda name: None)
 
     with patch(
-        "subprocess.run",
+        "litellm_proxy_extras.prisma_toolchain.run_prisma",
         side_effect=_fake_migrate_deploy_failure(1, _DEADLOCK_P3018_STDERR),
     ):
         with pytest.raises(RuntimeError, match="after 4 attempts"):
@@ -343,7 +343,7 @@ def test_v2_p3009_deadlocked_ledger_row_rolls_back_and_retries(monkeypatch, tmp_
         "_resolve_specific_migration",
         lambda name: pytest.fail("a deadlocked migration must never be marked applied"),
     )
-    monkeypatch.setattr("subprocess.run", _succeed_after(1, stderr))
+    monkeypatch.setattr("litellm_proxy_extras.prisma_toolchain.run_prisma", _succeed_after(1, stderr))
 
     ok = ProxyExtrasDBManager.setup_database(use_migrate=True, use_v2_resolver=True)
     assert ok is True
@@ -372,7 +372,7 @@ def test_v2_p3009_empty_ledger_logs_rolls_back_and_retries(monkeypatch, tmp_path
         "_resolve_specific_migration",
         lambda name: pytest.fail("a deadlocked migration must never be marked applied"),
     )
-    monkeypatch.setattr("subprocess.run", _succeed_after(1, stderr))
+    monkeypatch.setattr("litellm_proxy_extras.prisma_toolchain.run_prisma", _succeed_after(1, stderr))
 
     ok = ProxyExtrasDBManager.setup_database(use_migrate=True, use_v2_resolver=True)
     assert ok is True
@@ -395,7 +395,7 @@ def test_v2_p3009_unreadable_ledger_still_raises(monkeypatch, tmp_path):
         "_roll_back_migration",
         lambda name: pytest.fail("an unreadable ledger must not trigger a retry"),
     )
-    monkeypatch.setattr("subprocess.run", _succeed_after(1, stderr))
+    monkeypatch.setattr("litellm_proxy_extras.prisma_toolchain.run_prisma", _succeed_after(1, stderr))
 
     with pytest.raises(RuntimeError, match="cannot be auto-recovered"):
         ProxyExtrasDBManager.setup_database(use_migrate=True, use_v2_resolver=True)
@@ -417,7 +417,7 @@ def test_v2_p3009_non_deadlock_ledger_row_still_raises(monkeypatch, tmp_path):
         lambda name: 'ERROR: syntax error at or near "BRKN"',
     )
 
-    with patch("subprocess.run", side_effect=_fake_migrate_deploy_failure(1, stderr)):
+    with patch("litellm_proxy_extras.prisma_toolchain.run_prisma", side_effect=_fake_migrate_deploy_failure(1, stderr)):
         with pytest.raises(RuntimeError, match="cannot be auto-recovered"):
             ProxyExtrasDBManager.setup_database(use_migrate=True, use_v2_resolver=True)
 
@@ -427,7 +427,7 @@ def test_v2_bare_deadlock_stderr_retries(monkeypatch, tmp_path):
     waiter as victim) is retried, not fatal."""
     _stub_v2_env(monkeypatch, tmp_path)
     monkeypatch.setattr(
-        "subprocess.run", _succeed_after(1, "Database error: deadlock detected")
+        "litellm_proxy_extras.prisma_toolchain.run_prisma", _succeed_after(1, "Database error: deadlock detected")
     )
 
     ok = ProxyExtrasDBManager.setup_database(use_migrate=True, use_v2_resolver=True)
@@ -446,7 +446,10 @@ def test_v2_advisory_lock_timeout_retries(monkeypatch, tmp_path):
     """v2: the advisory-lock waiter that times out while a peer's retry holds
     the lock retries instead of dying."""
     _stub_v2_env(monkeypatch, tmp_path)
-    monkeypatch.setattr("subprocess.run", _succeed_after(2, _P1002_ADVISORY_LOCK_STDERR))
+    monkeypatch.setattr(
+        "litellm_proxy_extras.prisma_toolchain.run_prisma",
+        _succeed_after(2, _P1002_ADVISORY_LOCK_STDERR),
+    )
 
     ok = ProxyExtrasDBManager.setup_database(use_migrate=True, use_v2_resolver=True)
     assert ok is True
@@ -456,7 +459,7 @@ def test_v2_p1002_without_advisory_lock_context_still_raises(monkeypatch, tmp_pa
     """v2: a plain P1002 (database unreachable) stays fatal."""
     _stub_v2_env(monkeypatch, tmp_path)
     stderr = "Error: P1002\n\nThe database server at `db`:`5432` was reached but timed out."
-    monkeypatch.setattr("subprocess.run", _succeed_after(1, stderr))
+    monkeypatch.setattr("litellm_proxy_extras.prisma_toolchain.run_prisma", _succeed_after(1, stderr))
 
     with pytest.raises(RuntimeError, match="cannot be auto-recovered"):
         ProxyExtrasDBManager.setup_database(use_migrate=True, use_v2_resolver=True)

litellm-rust/ADDING_A_PROVIDER.md

@@ -26,4 +26,4 @@ variants of it. The test for a good abstraction is that adding the next provider
 is a few declarative lines, not a new file of duplicated flow. Only diverge from
 the base when behavior is genuinely different, and say so explicitly in the PR.
 
-**Calling:** hosts invoke the core entrypoint — the Python bridge and the `ai-gateway` route service both call `litellm_core::messages::messages`. Never add a provider handler to `ai-gateway`. Register new modules in `lib.rs` / `mod.rs`, then run `cargo fmt && cargo clippy --workspace -- -D warnings && cargo test --workspace`.
+**Calling:** hosts invoke the core entrypoint — the Python bridge and the `ai-gateway` route service both call `litellm_core::messages::messages`. Never add a provider handler to `ai-gateway`. Register new modules in `lib.rs` / `mod.rs`, then run the commands under "Checks" in [CLAUDE.md](CLAUDE.md).

litellm-rust/CLAUDE.md

@@ -174,10 +174,14 @@ for changes under `litellm-rust/`.
 ```bash
 cd litellm-rust
 cargo fmt --check
+cargo clippy --workspace --all-targets -- -D warnings
+cargo clippy -p litellm-core --all-targets --features bedrock-auth -- -D warnings
 # the ai-gateway binary + server code is behind the `server` feature
-cargo clippy -p litellm-ai-gateway --all-targets --features server -- -D warnings
-cargo clippy -p litellm-core -p litellm-python-interop -p litellm-python-bridge --all-targets -- -D warnings
+cargo clippy -p litellm-ai-gateway --all-targets --all-features -- -D warnings
 cargo test --workspace
+cargo test -p litellm-core --features bedrock-auth
+# the `auth`, `routes`, `state` and `realtime` tests only exist under `server`
+cargo test -p litellm-ai-gateway --features server
 ```
 
 When a Rust path is exposed through Python, add Python parity tests that compare

litellm-rust/README.md

@@ -49,11 +49,6 @@ function per top-level route, mirroring the core entrypoints.
 
 ## Checks
 
-Run these before pushing Rust changes. GitHub Actions runs the same checks for
-changes under `litellm-rust/`.
-
-```bash
-cargo fmt --check
-cargo clippy --workspace --all-targets -- -D warnings
-cargo test --workspace
-```
+Run the commands under "Checks" in [CLAUDE.md](CLAUDE.md) before pushing Rust
+changes. That list is the single source of truth and matches what GitHub Actions
+runs for changes under `litellm-rust/`.

litellm-rust/crates/CODING_STANDARDS/PROVIDER_CODING_STANDARDS.md

@@ -49,11 +49,5 @@ Rules for adding or changing an LLM provider/route in `litellm-rust`. `messages`
 
 ## Checks before push
 
-25. Run, and keep green:
-    ```bash
-    cd litellm-rust
-    cargo fmt --check
-    cargo clippy -p litellm-ai-gateway --all-targets --features server -- -D warnings
-    cargo clippy -p litellm-core -p litellm-python-interop -p litellm-python-bridge --all-targets -- -D warnings
-    cargo test --workspace
-    ```
+25. Run, and keep green, the commands under "Checks" in `litellm-rust/CLAUDE.md`.
+    That list is the single source of truth and matches what GitHub Actions runs.

litellm-rust/crates/ai-gateway/Dockerfile

@@ -36,12 +36,12 @@ FROM chef AS builder
 # whenever only gateway source changes.
 COPY --from=planner /build/litellm-rust/recipe.json recipe.json
 RUN cargo chef cook --locked --release \
-        -p litellm-ai-gateway --features python-config \
+        -p litellm-ai-gateway --features server,python-config \
         --recipe-path recipe.json
 # Now copy the real sources and build the gateway binary. Deps are already cooked
 # above, so this step only recompiles the gateway crate.
 COPY litellm-rust/ .
-RUN cargo build --locked --release -p litellm-ai-gateway --features python-config
+RUN cargo build --locked --release -p litellm-ai-gateway --bin litellm-ai-gateway --features server,python-config
 
 # ---- Runtime ----------------------------------------------------------------
 # python:3.11-slim-bookworm ships libpython3.11, matching the builder's PyO3

litellm-rust/crates/ai-gateway/README.md

@@ -100,7 +100,7 @@ Worker tuning, rarely needed: `LITELLM_LOG_CHANNEL_CAPACITY` (4096),
 
 ## Build & run with Docker
 
-The image is built `--features python-config` and installs litellm **from this
+The image is built `--features server,python-config` and installs litellm **from this
 repo's source** (the config reader is newer than any PyPI release), so the build
 **context is the repo root**:
 
@@ -135,10 +135,10 @@ docker run --rm -p 4001:4001 \
 ```bash
 # config.yaml mode — needs litellm importable in the active python env
 LITELLM_CONFIG_PATH=./crates/ai-gateway/config.yaml \
-  cargo run --release -p litellm-ai-gateway --features python-config
+  cargo run --release -p litellm-ai-gateway --features server,python-config
 
 # env stand-in mode — no python, no config
-cargo run --release -p litellm-ai-gateway
+cargo run --release -p litellm-ai-gateway --features server
 ```
 
 ## Deploy on Render

litellm-rust/crates/ai-gateway/src/realtime/streaming.rs

@@ -106,16 +106,16 @@ impl RealTimeStreaming {
     /// `litellm_call_id`, replacing the gateway-generated fallback.
     fn on_session(&mut self, event: &RealtimeEvent) {
         let session = event.data.get("session").and_then(Value::as_object);
-        if let Some(id) = session.and_then(|s| s.get("id")).and_then(Value::as_str) {
-            if !id.is_empty() {
-                self.id = id.to_string();
-                self.litellm_call_id = id.to_string();
-            }
+        if let Some(id) = session.and_then(|s| s.get("id")).and_then(Value::as_str)
+            && !id.is_empty()
+        {
+            self.id = id.to_string();
+            self.litellm_call_id = id.to_string();
         }
-        if let Some(model) = session.and_then(|s| s.get("model")).and_then(Value::as_str) {
-            if !model.is_empty() {
-                self.model = model.to_string();
-            }
+        if let Some(model) = session.and_then(|s| s.get("model")).and_then(Value::as_str)
+            && !model.is_empty()
+        {
+            self.model = model.to_string();
         }
     }
 
@@ -323,6 +323,32 @@ mod tests {
         assert_eq!(streaming.dropped(), 0);
     }
 
+    #[test]
+    fn blank_session_id_and_model_keep_the_gateway_fallbacks() {
+        let mut streaming = RealTimeStreaming::new(
+            Vec::new(),
+            "call_fallback".to_string(),
+            "gpt-realtime".to_string(),
+            RequestMetadata::default(),
+        );
+
+        streaming.observe(&event(
+            r#"{"type":"session.created","session":{"id":"","model":""}}"#,
+        ));
+        let payload = streaming.build_payload();
+        assert_eq!(payload.id, "call_fallback");
+        assert_eq!(payload.litellm_call_id, "call_fallback");
+        assert_eq!(payload.model, "gpt-realtime");
+
+        streaming.observe(&event(
+            r#"{"type":"session.updated","session":{"id":"sess_002","model":""}}"#,
+        ));
+        let payload = streaming.build_payload();
+        assert_eq!(payload.id, "sess_002");
+        assert_eq!(payload.litellm_call_id, "sess_002");
+        assert_eq!(payload.model, "gpt-realtime");
+    }
+
     #[test]
     fn payload_serializes_with_camelcase_times_and_realtime_call_type() {
         let mut streaming = RealTimeStreaming::new(

litellm-rust/crates/ai-gateway/src/routes/realtime/service.rs

@@ -50,18 +50,17 @@ where
         provider_model,
         params.api_key.as_deref(),
         params.api_base.as_deref(),
-    ) {
-        if let Some(handoff) = pool.take(&key) {
-            return crate::io::realtime::realtime_warm(
-                provider_model,
-                handoff,
-                idle_timeout,
-                observe,
-                client_in,
-                client_out,
-            )
-            .await;
-        }
+    ) && let Some(handoff) = pool.take(&key)
+    {
+        return crate::io::realtime::realtime_warm(
+            provider_model,
+            handoff,
+            idle_timeout,
+            observe,
+            client_in,
+            client_out,
+        )
+        .await;
     }
 
     // Cold path: fresh dial (the original behavior).

litellm/_service_logger.py

@@ -1,4 +1,5 @@
 import asyncio
+from collections.abc import Callable, Coroutine
 from datetime import datetime, timedelta
 from typing import TYPE_CHECKING, Any, Final
 
@@ -83,6 +84,47 @@ class ServiceLogging(CustomLogger):
                 return open_telemetry_logger
         return None
 
+    @staticmethod
+    def _sync_dispatch_loop() -> asyncio.AbstractEventLoop | None:
+        """The event loop a blocking caller can dispatch on, or ``None`` if it has none."""
+        try:
+            loop: Final = asyncio.get_event_loop()
+        except RuntimeError:
+            return None
+        return None if loop.is_closed() else loop
+
+    @staticmethod
+    async def _emit_guarded(hook: Callable[[], Coroutine[object, object, None]]) -> None:
+        """Emit one service event, absorbing anything the callbacks raise.
+
+        Monitoring must not break the call it monitors. Sync callers are the ones that
+        swallow their own service failures (a Redis batch read returns an empty dict),
+        so an exception from a misconfigured callback would replace a Redis outage with
+        a callback error and skip the caller's fallback handling.
+        """
+        try:
+            await hook()
+        except Exception as e:
+            verbose_logger.exception("Error emitting service event - %s", e)
+
+    @staticmethod
+    def _dispatch_from_sync(hook: Callable[[], Coroutine[object, object, None]]) -> None:
+        """Run an async service hook from a blocking caller, whatever event loop it holds.
+
+        Takes a factory rather than a coroutine so the hook is built on the path that
+        runs it, and only ever once.
+        """
+        loop: Final = ServiceLogging._sync_dispatch_loop()
+        try:
+            if loop is None:
+                asyncio.run(ServiceLogging._emit_guarded(hook))
+            elif loop.is_running():
+                loop.create_task(ServiceLogging._emit_guarded(hook))
+            else:
+                loop.run_until_complete(ServiceLogging._emit_guarded(hook))
+        except Exception as e:
+            verbose_logger.exception("Error dispatching service event - %s", e)
+
     def service_success_hook(
         self,
         service: ServiceTypes,
@@ -99,54 +141,45 @@ class ServiceLogging(CustomLogger):
         if self.mock_testing:
             self.mock_testing_sync_success_hook += 1
 
-        try:
-            # Try to get the current event loop
-            loop: Final = asyncio.get_event_loop()
-            # Check if the loop is running
-            if loop.is_running():
-                # If we're in a running loop, create a task
-                loop.create_task(
-                    self.async_service_success_hook(
-                        service=service,
-                        duration=duration,
-                        call_type=call_type,
-                        parent_otel_span=parent_otel_span,
-                        start_time=start_time,
-                        end_time=end_time,
-                    )
-                )
-            else:
-                # Loop exists but not running, we can use run_until_complete
-                loop.run_until_complete(
-                    self.async_service_success_hook(
-                        service=service,
-                        duration=duration,
-                        call_type=call_type,
-                        parent_otel_span=parent_otel_span,
-                        start_time=start_time,
-                        end_time=end_time,
-                    )
-                )
-        except RuntimeError:
-            # No event loop exists, create a new one and run
-            asyncio.run(
-                self.async_service_success_hook(
-                    service=service,
-                    duration=duration,
-                    call_type=call_type,
-                    parent_otel_span=parent_otel_span,
-                    start_time=start_time,
-                    end_time=end_time,
-                )
+        self._dispatch_from_sync(
+            lambda: self.async_service_success_hook(
+                service=service,
+                duration=duration,
+                call_type=call_type,
+                parent_otel_span=parent_otel_span,
+                start_time=start_time,
+                end_time=end_time,
             )
+        )
 
-    def service_failure_hook(self, service: ServiceTypes, duration: float, error: Exception, call_type: str):
+    def service_failure_hook(
+        self,
+        service: ServiceTypes,
+        duration: float,
+        error: Exception,
+        call_type: str,
+        parent_otel_span: Span | None = None,
+        start_time: datetime | float | None = None,
+        end_time: float | datetime | None = None,
+    ):
         """
-        [TODO] Not implemented for sync calls yet. V0 is focused on async monitoring (used by proxy).
+        Handles both sync and async monitoring by checking for existing event loop.
         """
         if self.mock_testing:
             self.mock_testing_sync_failure_hook += 1
 
+        self._dispatch_from_sync(
+            lambda: self.async_service_failure_hook(
+                service=service,
+                duration=duration,
+                error=error,
+                call_type=call_type,
+                parent_otel_span=parent_otel_span,
+                start_time=start_time,
+                end_time=end_time,
+            )
+        )
+
     async def async_service_success_hook(
         self,
         service: ServiceTypes,

litellm/caching/dual_cache.py

@@ -8,10 +8,9 @@ Has 4 primary methods:
     - async_get_cache
 """
 
-import asyncio
 import time
 import traceback
-from concurrent.futures import ThreadPoolExecutor
+from collections.abc import Sequence
 from threading import Lock
 from typing import TYPE_CHECKING, Any, Final
 
@@ -188,31 +187,38 @@ class DualCache(BaseCache):
         local_only: bool = False,
         **kwargs,
     ):
-        received_args: Final = locals()
-        received_args.pop("self")
-
-        def run_in_new_loop():
-            """Run the coroutine in a new event loop within this thread."""
-            new_loop: Final = asyncio.new_event_loop()
-            try:
-                asyncio.set_event_loop(new_loop)
-                return new_loop.run_until_complete(self.async_batch_get_cache(**received_args))
-            finally:
-                new_loop.close()
-                asyncio.set_event_loop(None)
-
         try:
-            # First, try to get the current event loop
-            _ = asyncio.get_running_loop()
-            # If we're already in an event loop, run in a separate thread
-            # to avoid nested event loop issues
-            with ThreadPoolExecutor(max_workers=1) as executor:
-                future: Final = executor.submit(run_in_new_loop)
-                return future.result()
+            in_memory_result: Final = (
+                self.in_memory_cache.batch_get_cache(keys, **kwargs) if self.in_memory_cache is not None else None
+            )
+            result: Final = in_memory_result if in_memory_result is not None else tuple(None for _ in keys)
 
-        except RuntimeError:
-            # No running event loop, we can safely run in this thread
-            return run_in_new_loop()
+            if None not in result or self.redis_cache is None or local_only:
+                return result
+
+            sublist_keys, previous_access_times = self._reserve_redis_batch_keys(time.time(), keys, result)
+            if len(sublist_keys) == 0:
+                return result
+
+            try:
+                redis_result: Final = self.redis_cache.batch_get_cache(
+                    key_list=sublist_keys, parent_otel_span=parent_otel_span
+                )
+            except Exception:
+                # Do not throttle subsequent callers if the Redis read fails.
+                self._rollback_redis_batch_key_reservations(previous_access_times)
+                raise
+
+            if self.in_memory_cache is not None:
+                for key, value in redis_result.items():
+                    if value is not None:
+                        self.in_memory_cache.set_cache(key, value, **self._backfill_kwargs(kwargs))
+
+            return list(  # mutable-ok: public list contract
+                redis_result.get(key) if value is None else value for key, value in zip(keys, result)
+            )
+        except Exception:
+            verbose_logger.error(traceback.format_exc())
 
     async def async_get_cache(
         self,
@@ -251,7 +257,7 @@ class DualCache(BaseCache):
         self,
         current_time: float,
         keys: list[str],
-        result: list[Any],
+        result: Sequence[Any],
     ) -> tuple[list[str], dict[str, float | None]]:
         """
         Atomically choose keys to fetch from Redis and reserve their access time.

litellm/caching/redis_cache.py

@@ -27,6 +27,7 @@ from litellm.constants import (
     REDIS_CIRCUIT_BREAKER_ENABLED,
     REDIS_CIRCUIT_BREAKER_FAILURE_THRESHOLD,
     REDIS_CIRCUIT_BREAKER_RECOVERY_TIMEOUT,
+    REDIS_CIRCUIT_BREAKER_TIMEOUT_MIN_DURATION,
 )
 from litellm.litellm_core_utils.core_helpers import _get_parent_otel_span_from_kwargs
 from litellm.litellm_core_utils.coroutine_checker import coroutine_checker
@@ -41,6 +42,8 @@ from .base_cache import BaseCache
 
 if TYPE_CHECKING:
     from opentelemetry.trace import Span as _Span
+    from prometheus_client import Counter as _PromCounter
+    from prometheus_client import Gauge as _PromGauge
     from redis.asyncio import Redis, RedisCluster
     from redis.asyncio.client import Pipeline
     from redis.asyncio.cluster import ClusterPipeline
@@ -78,10 +81,18 @@ class _AsyncRedisCommands(Protocol):
     def pipeline(self, transaction: bool = True) -> "Pipeline[bytes]": ...
 
 
+_BREAKER_GUARD_FRAME_NAMES: Final = frozenset(
+    {"<lambda>", "wrapper", "_run_under_circuit_breaker", "_run_under_circuit_breaker_sync"}
+)
+
+
 def _get_call_stack_info(num_frames: int = 2) -> str:
     """
     Get the function names from the previous 1-2 functions in the call stack.
 
+    Frames belonging to this module's circuit-breaker guards are skipped so the
+    reported callers stay the real ones even on guarded methods.
+
     Args:
         num_frames: Number of previous frames to include (default: 2)
 
@@ -102,11 +113,11 @@ def _get_call_stack_info(num_frames: int = 2) -> str:
             return "unknown"
         function_names: Final = []
 
-        for _ in range(num_frames):
-            if frame is None:
-                break
-            func_name = frame.f_code.co_name
-            function_names.append(func_name)
+        while frame is not None and len(function_names) < num_frames:
+            if frame.f_code.co_name in _BREAKER_GUARD_FRAME_NAMES and frame.f_globals.get("__name__") == __name__:
+                frame = frame.f_back
+                continue
+            function_names.append(frame.f_code.co_name)
             frame = frame.f_back
 
         if not function_names:
@@ -127,10 +138,20 @@ class RedisCircuitBreaker:
       HALF_OPEN - recovery probe: allow one request through
 
     Transitions:
-      CLOSED    -> OPEN      after failure_threshold consecutive failures
+      CLOSED    -> OPEN      after failure_threshold consecutive hard connectivity
+                             failures, or after an unbroken run of timeout failures
+                             (no success or hard failure in between) that reaches
+                             failure_threshold and spans timeout_min_duration seconds
       OPEN      -> HALF_OPEN after recovery_timeout seconds
       HALF_OPEN -> CLOSED    on success
       HALF_OPEN -> OPEN      on failure (resets timer)
+
+    Timeouts are accounted separately from hard connectivity failures because the async
+    Redis timeout includes time waiting for the worker event loop to resume: one loop
+    stall makes every in-flight operation time out together, which satisfies a purely
+    consecutive threshold instantly even though Redis is healthy. Requiring a
+    timeout-only streak to also span timeout_min_duration filters such bursts while a
+    real outage that surfaces as timeouts still opens the breaker after that duration.
     """
 
     CLOSED = "closed"
@@ -142,13 +163,19 @@ class RedisCircuitBreaker:
         failure_threshold: int,
         recovery_timeout: int,
         enabled: bool = True,
+        timeout_min_duration: float = REDIS_CIRCUIT_BREAKER_TIMEOUT_MIN_DURATION,
     ) -> None:
         self.failure_threshold = failure_threshold
         self.recovery_timeout = recovery_timeout
         self.enabled = enabled
+        self.timeout_min_duration = timeout_min_duration
         self._failure_count = 0
+        self._hard_failure_count = 0
+        self._timeout_count = 0
+        self._timeout_streak_started_at: float | None = None
         self._opened_at: float | None = None
         self._state = self.CLOSED
+        _breaker_metrics().record_state_change(None, self._state)
 
     def is_open(self) -> bool:
         """Returns True if Redis calls should be skipped."""
@@ -161,24 +188,45 @@ class RedisCircuitBreaker:
             return True
         if self._state == self.OPEN:
             if time.time() - (self._opened_at or 0) > self.recovery_timeout:
-                self._state = self.HALF_OPEN
+                self._set_state(self.HALF_OPEN)
                 return False  # this caller is the designated probe
             return True
         return False
 
-    def record_failure(self) -> None:
+    def _should_open(self, now: float) -> bool:
+        if self._state == self.HALF_OPEN:
+            return True
+        if self._hard_failure_count >= self.failure_threshold:
+            return True
+        if self._timeout_count < self.failure_threshold:
+            return False
+        return now - (self._timeout_streak_started_at or now) >= self.timeout_min_duration
+
+    def record_failure(self, is_timeout: bool = False) -> None:
         if not self.enabled:
             return
+        now: Final = time.time()
         self._failure_count += 1
-        self._opened_at = time.time()
-        if self._failure_count >= self.failure_threshold:
+        if is_timeout:
+            self._timeout_count += 1
+            if self._timeout_streak_started_at is None:
+                self._timeout_streak_started_at = now
+        else:
+            self._hard_failure_count += 1
+            self._timeout_count = 0
+            self._timeout_streak_started_at = None
+        self._opened_at = now
+        _breaker_metrics().record_failure("timeout" if is_timeout else "connectivity")
+        if self._should_open(now):
             if self._state != self.OPEN:
                 verbose_logger.warning(
-                    "Redis circuit breaker OPENED after %d consecutive failures — fast-failing Redis calls for %ds",
+                    "Redis circuit breaker OPENED after %d consecutive failures"
+                    " (%d hard connectivity) — fast-failing Redis calls for %ds",
                     self._failure_count,
+                    self._hard_failure_count,
                     self.recovery_timeout,
                 )
-            self._state = self.OPEN
+            self._set_state(self.OPEN)
 
     def record_success(self) -> None:
         if not self.enabled:
@@ -186,7 +234,17 @@ class RedisCircuitBreaker:
         if self._state == self.HALF_OPEN:
             verbose_logger.info("Redis circuit breaker CLOSED — Redis recovered")
         self._failure_count = 0
-        self._state = self.CLOSED
+        self._hard_failure_count = 0
+        self._timeout_count = 0
+        self._timeout_streak_started_at = None
+        self._set_state(self.CLOSED)
+
+    def _set_state(self, state: str) -> None:
+        if state == self._state:
+            return
+        _breaker_metrics().record_transition(state)
+        _breaker_metrics().record_state_change(self._state, state)
+        self._state = state
 
 
 _RedisCallResult = TypeVar("_RedisCallResult")
@@ -226,6 +284,78 @@ def _is_redis_health_failure(exc: BaseException) -> bool:
         return True
 
 
+@functools.lru_cache(maxsize=1)
+def _redis_timeout_error_types() -> tuple[type, ...]:
+    """Health failures that are timeouts rather than unambiguous connectivity errors.
+
+    ``builtins.TimeoutError`` covers ``asyncio.TimeoutError`` and ``socket.timeout``
+    (aliases since py3.11 / py3.10). ``redis.exceptions.TimeoutError`` does not subclass
+    either, so it is listed explicitly.
+    """
+    try:
+        from redis.exceptions import TimeoutError as RedisTimeoutError
+    except ImportError:
+        return (TimeoutError,)
+    return (RedisTimeoutError, TimeoutError)
+
+
+def _is_redis_timeout_failure(exc: BaseException) -> bool:
+    return isinstance(exc, _redis_timeout_error_types())
+
+
+class _BreakerMetrics:
+    """Prometheus metrics for the Redis circuit breaker; no-ops when the client is absent.
+
+    Registered lazily on the default registry (which /metrics serves) via the module-level
+    ``_breaker_metrics`` singleton so repeated RedisCache construction never re-registers.
+    """
+
+    def __init__(self) -> None:
+        self._state_gauge: _PromGauge | None = None
+        self._transitions: _PromCounter | None = None
+        self._failures: _PromCounter | None = None
+        try:
+            from prometheus_client import Counter as PromCounter
+            from prometheus_client import Gauge
+        except ImportError:
+            return
+        self._state_gauge = Gauge(
+            "litellm_redis_circuit_breaker_state",
+            "Number of Redis circuit breakers currently in each state",
+            labelnames=("state",),
+        )
+        self._transitions = PromCounter(
+            "litellm_redis_circuit_breaker_transitions",
+            "Redis circuit breaker state transitions",
+            labelnames=("state",),
+        )
+        self._failures = PromCounter(
+            "litellm_redis_circuit_breaker_failures",
+            "Redis health failures counted by the circuit breaker",
+            labelnames=("failure_class",),
+        )
+
+    def record_state_change(self, old_state: str | None, new_state: str) -> None:
+        if self._state_gauge is None:
+            return
+        if old_state is not None:
+            self._state_gauge.labels(old_state).dec()
+        self._state_gauge.labels(new_state).inc()
+
+    def record_transition(self, state: str) -> None:
+        if self._transitions is not None:
+            self._transitions.labels(state).inc()
+
+    def record_failure(self, failure_class: str) -> None:
+        if self._failures is not None:
+            self._failures.labels(failure_class).inc()
+
+
+@functools.lru_cache(maxsize=1)
+def _breaker_metrics() -> _BreakerMetrics:
+    return _BreakerMetrics()
+
+
 def _record_swallowed_redis_failure(breaker: RedisCircuitBreaker, exc: BaseException) -> None:
     """Record a Redis failure that the calling method is about to swallow.
 
@@ -237,10 +367,27 @@ def _record_swallowed_redis_failure(breaker: RedisCircuitBreaker, exc: BaseExcep
     """
     if not _is_redis_health_failure(exc):
         return
-    breaker.record_failure()
+    breaker.record_failure(is_timeout=_is_redis_timeout_failure(exc))
     _swallowed_redis_failures.set(_swallowed_redis_failures.get() + 1)
 
 
+def _enter_circuit_breaker(breaker: RedisCircuitBreaker, name: str) -> int:
+    """Reject the call if the breaker is open, else return the swallowed-failure count to compare against."""
+    if breaker.is_open():
+        raise Exception(f"Redis circuit breaker is open — skipping {name}")
+    return _swallowed_redis_failures.get()
+
+
+def _exit_circuit_breaker(breaker: RedisCircuitBreaker, swallowed_before: int) -> None:
+    """Record success only when nothing failed while the call ran.
+
+    Several Redis methods catch their own connection errors and return a default, so a
+    method that returned is not on its own proof of a healthy Redis.
+    """
+    if _swallowed_redis_failures.get() == swallowed_before:
+        breaker.record_success()
+
+
 async def _run_under_circuit_breaker(
     breaker: RedisCircuitBreaker,
     name: str,
@@ -249,20 +396,33 @@ async def _run_under_circuit_breaker(
     """Run one Redis coroutine under a circuit breaker.
 
     Shared by the method decorator and the Lua script executor so both feed the same
-    health signal. Success is recorded only when nothing failed while ``call`` ran,
-    because several Redis methods catch their own connection errors and return a default.
+    health signal.
     """
-    if breaker.is_open():
-        raise Exception(f"Redis circuit breaker is open — skipping {name}")
-    swallowed_before: Final = _swallowed_redis_failures.get()
+    swallowed_before: Final = _enter_circuit_breaker(breaker, name)
     try:
         result: Final = await call()
+    except Exception as e:
+        if _is_redis_health_failure(e):
+            breaker.record_failure(is_timeout=_is_redis_timeout_failure(e))
+        raise
+    _exit_circuit_breaker(breaker, swallowed_before)
+    return result
+
+
+def _run_under_circuit_breaker_sync(
+    breaker: RedisCircuitBreaker,
+    name: str,
+    call: Callable[[], _RedisCallResult],
+) -> _RedisCallResult:
+    """Run one blocking Redis call under a circuit breaker, feeding the same health signal as the async path."""
+    swallowed_before: Final = _enter_circuit_breaker(breaker, name)
+    try:
+        result: Final = call()
     except Exception as e:
         if _is_redis_health_failure(e):
             breaker.record_failure()
         raise
-    if _swallowed_redis_failures.get() == swallowed_before:
-        breaker.record_success()
+    _exit_circuit_breaker(breaker, swallowed_before)
     return result
 
 
@@ -288,6 +448,14 @@ def _redis_circuit_breaker_guard(method):
     return wrapper
 
 
+def _redis_circuit_breaker_guard_sync(method: Callable[..., _RedisCallResult]) -> Callable[..., _RedisCallResult]:
+    return functools.wraps(method)(
+        lambda self, *args, **kwargs: _run_under_circuit_breaker_sync(
+            self._circuit_breaker, method.__name__, lambda: method(self, *args, **kwargs)
+        )
+    )
+
+
 class RedisCache(BaseCache):
     # if users don't provider one, use the default litellm cache
 
@@ -1146,14 +1314,13 @@ class RedisCache(BaseCache):
         """
         key_value_dict = {}
         _key_list: Final = [key for key in key_list if key is not None]
+        start_time: Final = time.time()
 
         try:
-            _keys: Final = []
-            for cache_key in _key_list:
-                cache_key = self.check_and_fix_namespace(key=cache_key or "")
-                _keys.append(cache_key)
-            start_time: Final = time.time()
+            swallowed_before: Final = _enter_circuit_breaker(self._circuit_breaker, "batch_get_cache")
+            _keys: Final = [self.check_and_fix_namespace(key=cache_key or "") for cache_key in _key_list]
             results: Final = self._run_redis_mget_operation(keys=_keys)
+            _exit_circuit_breaker(self._circuit_breaker, swallowed_before)
             end_time: Final = time.time()
             _duration: Final = end_time - start_time
             self.service_logger_obj.service_success_hook(
@@ -1178,7 +1345,18 @@ class RedisCache(BaseCache):
 
             return decoded_results
         except Exception as e:
+            failed_at: Final = time.time()
+            self.service_logger_obj.service_failure_hook(
+                service=ServiceTypes.REDIS,
+                duration=failed_at - start_time,
+                error=e,
+                call_type=f"batch_get_cache <- {_get_call_stack_info()}",
+                start_time=start_time,
+                end_time=failed_at,
+                parent_otel_span=parent_otel_span,
+            )
             verbose_logger.error("Error occurred in batch get cache - %s", e)
+            _record_swallowed_redis_failure(self._circuit_breaker, e)
             return key_value_dict
 
     @_redis_circuit_breaker_guard

litellm/completion_extras/litellm_responses_transformation/transformation.py

@@ -932,7 +932,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
         return OpenAiResponsesToChatCompletionStreamIterator(streaming_response, sync_stream, json_mode)
 
     def _convert_content_str_to_input_text(self, content: str, role: str) -> dict[str, object]:
-        if role == "user" or role == "system" or role == "tool":
+        if role in ("user", "system", "developer", "tool"):
             return {"type": "input_text", "text": content}
         else:
             return {"type": "output_text", "text": content}

litellm/constants.py

@@ -7,6 +7,7 @@ from litellm.litellm_core_utils.env_utils import get_env_int, get_env_int_in_ran
 
 DEFAULT_HEALTH_CHECK_PROMPT: Final = str(os.getenv("DEFAULT_HEALTH_CHECK_PROMPT", "test from litellm"))
 AZURE_DEFAULT_RESPONSES_API_VERSION: Final = str(os.getenv("AZURE_DEFAULT_RESPONSES_API_VERSION", "preview"))
+AZURE_OPENAI_AUDIO_PROVIDERS: Final = frozenset({"azure", "azure_ai"})
 ROUTER_MAX_FALLBACKS: Final = int(os.getenv("ROUTER_MAX_FALLBACKS", 5))
 ROUTER_FALLBACK_ERROR_DETAIL_MAX_CHARS: Final = 2000
 RUNTIME_UPDATABLE_ROUTER_SETTINGS: Final[frozenset[str]] = frozenset(
@@ -39,6 +40,7 @@ ROUTER_SETTINGS_MANAGED_OUTSIDE_CONFIG: Final[frozenset[str]] = frozenset(
         "router_general_settings",
         "ignore_invalid_deployments",
         "fallback_access_check",
+        "heuristic_v2_router_limit",
     }
 )
 DEFAULT_BATCH_SIZE: Final = int(os.getenv("DEFAULT_BATCH_SIZE", 512))
@@ -430,6 +432,9 @@ REDIS_CONNECTION_POOL_TIMEOUT: Final = int(os.getenv("REDIS_CONNECTION_POOL_TIME
 REDIS_CIRCUIT_BREAKER_FAILURE_THRESHOLD: Final = int(os.getenv("REDIS_CIRCUIT_BREAKER_FAILURE_THRESHOLD", 5))
 REDIS_CIRCUIT_BREAKER_RECOVERY_TIMEOUT: Final = int(os.getenv("REDIS_CIRCUIT_BREAKER_RECOVERY_TIMEOUT", 60))
 REDIS_CIRCUIT_BREAKER_ENABLED: Final = os.getenv("REDIS_CIRCUIT_BREAKER_ENABLED", "true").lower() == "true"
+# minimum seconds a timeout-only failure streak must span before it can open the breaker,
+# so one event-loop stall timing out many queued calls at once does not trip it
+REDIS_CIRCUIT_BREAKER_TIMEOUT_MIN_DURATION: Final = float(os.getenv("REDIS_CIRCUIT_BREAKER_TIMEOUT_MIN_DURATION", 5.0))
 # Seconds of idle before a Redis cluster connection is validated with a PING and
 # reconnected if dead, so a connection silently dropped by a cluster restart
 # (e.g. ElastiCache Serverless maintenance) is not reused while broken

litellm/experimental_mcp_client/client.py

@@ -70,7 +70,7 @@ def to_basic_auth(auth_value: str) -> str:
 
 
 def strip_auth_scheme(auth_value: str, scheme: str) -> str:
-    """Return ``auth_value`` with a leading ``<scheme> `` removed, or unchanged when absent.
+    """Return ``auth_value`` with a leading ``<scheme>`` and separator removed, or unchanged when absent.
 
     Callers supply both a bare credential and a complete header value, so prefixing
     unconditionally yields ``Bearer Bearer <jwt>``. Scheme names are case-insensitive per
@@ -78,10 +78,9 @@ def strip_auth_scheme(auth_value: str, scheme: str) -> str:
     with the scheme text and a scheme with nothing behind it are returned untouched.
     Surrounding whitespace is left to ``_strip_header_whitespace`` at header-build time.
     """
-    scheme_name, _, remainder = auth_value.lstrip().partition(" ")
-    credential: Final = remainder.lstrip()
-    if credential and scheme_name.lower() == scheme.lower():
-        return credential
+    parts: Final = auth_value.split(None, 1)
+    if len(parts) == 2 and parts[0].lower() == scheme.lower():
+        return parts[1]
     return auth_value
 
 

litellm/integrations/custom_guardrail.py

@@ -831,10 +831,10 @@ class CustomGuardrail(CustomLogger):
         # should run guardrail
         litellm_guardrails: Final = request_data.get("guardrails")
         if litellm_guardrails is None or not isinstance(litellm_guardrails, list):
-            return response
+            return None
 
         if self.should_run_guardrail(data=request_data, event_type=GuardrailEventHooks.post_call) is not True:
-            return response
+            return None
 
         # CHECK IF GUARDRAIL REJECTS THE REQUEST
         result: Final = await self.async_post_call_success_hook(
@@ -850,7 +850,7 @@ class CustomGuardrail(CustomLogger):
         )
 
         if not self._is_valid_response_type(result):
-            return response
+            return None
 
         return result
 

litellm/integrations/vector_store_integrations/vector_store_pre_call_hook.py

@@ -6,7 +6,8 @@ It searches the vector store for relevant context and appends it to the messages
 """
 
 from collections.abc import Awaitable, Callable
-from typing import TYPE_CHECKING, Any, Final, cast
+from dataclasses import dataclass
+from typing import TYPE_CHECKING, Any, Final, Protocol, cast
 
 import litellm
 import litellm.vector_stores
@@ -24,10 +25,35 @@ from litellm.types.vector_stores import (
 
 if TYPE_CHECKING:
     from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
+    from litellm.proxy.utils import PrismaClient
+    from litellm.router import Router
 else:
     LiteLLMLoggingObj = Any
 
 
+class ProxyRuntime(Protocol):
+    def llm_router(self) -> "Router | None": ...
+
+    def prisma_client(self) -> "PrismaClient | None": ...
+
+
+@dataclass(frozen=True, slots=True)
+class ProxyServerRuntime:
+    def llm_router(self) -> "Router | None":
+        try:
+            from litellm.proxy.proxy_server import llm_router
+        except ImportError:
+            return None
+        return llm_router
+
+    def prisma_client(self) -> "PrismaClient | None":
+        try:
+            from litellm.proxy.proxy_server import prisma_client
+        except ImportError:
+            return None
+        return prisma_client
+
+
 class VectorStorePreCallHook(CustomLogger):
     CONTENT_PREFIX_STRING = "Context:\n\n"
     """
@@ -39,8 +65,9 @@ class VectorStorePreCallHook(CustomLogger):
     3. Appends the search results as context to the messages
     """
 
-    def __init__(self):
+    def __init__(self, proxy_runtime: ProxyRuntime | None = None):
         super().__init__()
+        self.proxy_runtime: Final[ProxyRuntime] = proxy_runtime or ProxyServerRuntime()
 
     async def async_get_chat_completion_prompt(
         self,
@@ -79,21 +106,8 @@ class VectorStorePreCallHook(CustomLogger):
             if litellm.vector_store_registry is None:
                 return model, messages, non_default_params
 
-            # Get prisma_client for database fallback
-            prisma_client = None
-            llm_router = None
-            try:
-                from litellm.proxy.proxy_server import (
-                    llm_router as _llm_router,
-                )
-                from litellm.proxy.proxy_server import (
-                    prisma_client as _prisma_client,
-                )
-
-                prisma_client = _prisma_client
-                llm_router = _llm_router
-            except ImportError:
-                pass
+            prisma_client: Final = self.proxy_runtime.prisma_client()
+            llm_router: Final = self.proxy_runtime.llm_router()
 
             # Use database fallback to ensure synchronization across instances
             vector_stores_to_run: list[
@@ -136,15 +150,23 @@ class VectorStorePreCallHook(CustomLogger):
                         Callable[..., Awaitable[VectorStoreSearchResponse]],
                         litellm.vector_stores.asearch,
                     )
-                search_response = await search_function(
-                    **{
-                        "vector_store_id": vector_store_id,
-                        "query": query,
-                        "custom_llm_provider": custom_llm_provider,
-                        "metadata": request_metadata,
-                        **litellm_params_for_vector_store,
-                    },
-                )
+                try:
+                    search_response = await search_function(
+                        **{
+                            "vector_store_id": vector_store_id,
+                            "query": query,
+                            "custom_llm_provider": custom_llm_provider,
+                            "metadata": request_metadata,
+                            **litellm_params_for_vector_store,
+                        },
+                    )
+                except Exception as search_error:
+                    verbose_logger.warning(
+                        "Vector store search failed for vector_store_id=%s, continuing without its context: %s",
+                        vector_store_id,
+                        search_error,
+                    )
+                    continue
 
                 verbose_logger.debug("search_response: %s", search_response)
 
@@ -153,7 +175,7 @@ class VectorStorePreCallHook(CustomLogger):
 
                 # Process search results and append as context
                 modified_messages = self._append_search_results_to_messages(
-                    messages=messages, search_response=search_response
+                    messages=modified_messages, search_response=search_response
                 )
 
                 # Get the number of results for logging

litellm/litellm_core_utils/internal_call_metadata.py

@@ -18,9 +18,11 @@ caller's identity metadata, minus two things that must never be forwarded as-is:
 from __future__ import annotations
 
 from collections.abc import Mapping
+from types import MappingProxyType
 from typing import Final
 
 from litellm.constants import INTERNAL_CALL_ORIGIN_METADATA_KEY, NON_INFERENCE_CALL_TYPES
+from litellm.litellm_core_utils.initialize_dynamic_callback_params import initialize_standard_callback_dynamic_params
 from litellm.types.utils import BACKGROUND_RESPONSE_COST_POLL_CALL_ORIGIN, InternalCallOrigin
 
 BUDGET_RESERVATION_METADATA_KEYS: Final = frozenset({"user_api_key_budget_reservation"})
@@ -142,6 +144,19 @@ def forwarded_internal_call_metadata(
     }
 
 
+def parent_session_kwargs(request_kwargs: Mapping[str, object] | None) -> Mapping[str, str]:
+    kwargs: Final = request_kwargs or MappingProxyType({})
+    return MappingProxyType(
+        {k: v for k in ("litellm_session_id", "litellm_trace_id") if isinstance(v := kwargs.get(k), str)}
+    )
+
+
+def effective_turn_off_message_logging(request_kwargs: Mapping[str, object] | None) -> bool | None:
+    return initialize_standard_callback_dynamic_params(dict(request_kwargs) if request_kwargs else None).get(
+        "turn_off_message_logging"
+    )
+
+
 def sanitized_forwardable_call_metadata(
     parent_metadata: Mapping[str, object],
     call_origin: InternalCallOrigin,

litellm/litellm_core_utils/litellm_logging.py

@@ -414,6 +414,11 @@ def _resolve_vertex_location_for_cost(
     return VertexBase.get_vertex_region(configured_location, model)
 
 
+def _provider_response_id(source: object) -> str | None:
+    candidate: Final = source.get("id") if isinstance(source, dict) else getattr(source, "id", None)
+    return candidate if isinstance(candidate, str) and candidate else None
+
+
 class Logging(LiteLLMLoggingBaseClass):
     global \
         supabaseClient, \
@@ -429,6 +434,7 @@ class Logging(LiteLLMLoggingBaseClass):
     custom_pricing: bool = False
     stream_options = None
     litellm_request_debug: bool = False
+    streamed_anthropic_message_id: str | None = None
 
     def __init__(
         self,
@@ -2136,7 +2142,7 @@ class Logging(LiteLLMLoggingBaseClass):
             self.model_call_details["cache_hit"] = cache_hit
 
             if self.call_type == CallTypes.anthropic_messages.value:
-                result = self._handle_anthropic_messages_response_logging(result=result)
+                result = self._anthropic_messages_logged_response(result=result)
             elif (
                 self.call_type == CallTypes.generate_content.value
                 or self.call_type == CallTypes.agenerate_content.value
@@ -3825,6 +3831,23 @@ class Logging(LiteLLMLoggingBaseClass):
             )
         return None
 
+    def record_streamed_anthropic_message_id(self, message_id: str) -> None:
+        self.streamed_anthropic_message_id = message_id
+
+    def _anthropic_messages_logged_response(self, result: Any) -> ModelResponse:
+        """
+        The ModelResponse a /v1/messages spend_logs row is built from.
+
+        A streaming call bridged onto the Responses API is the one case where the `msg_` id the
+        caller was served is minted locally rather than issued upstream, so it is absent from the
+        response the row would otherwise be keyed on and has to be carried over here.
+        """
+        logged: Final = self._handle_anthropic_messages_response_logging(result=result)
+        streamed_message_id: Final = self.streamed_anthropic_message_id
+        if streamed_message_id is None:
+            return logged
+        return logged.model_copy(update={"id": streamed_message_id})
+
     def _handle_anthropic_messages_response_logging(self, result: Any) -> ModelResponse:
         """
         Handles logging for Anthropic messages responses.
@@ -3851,11 +3874,12 @@ class Logging(LiteLLMLoggingBaseClass):
         if isinstance(result, ResponsesAPIResponse):
             return self._translate_responses_api_response_to_model_response(result)
 
+        provider_response_id: Final = _provider_response_id(result)
         httpx_response: Final = self.model_call_details.get("httpx_response", None)
         if httpx_response and isinstance(httpx_response, httpx.Response):
             result = litellm.AnthropicConfig().transform_response(
                 raw_response=httpx_response,
-                model_response=litellm.ModelResponse(),
+                model_response=litellm.ModelResponse(id=provider_response_id),
                 model=self.model,
                 messages=[],
                 logging_obj=self,
@@ -3878,7 +3902,7 @@ class Logging(LiteLLMLoggingBaseClass):
                     status_code=200,
                     headers={},
                 ),
-                model_response=litellm.ModelResponse(),
+                model_response=litellm.ModelResponse(id=provider_response_id),
                 json_mode=None,
                 speed=self.optional_params.get("speed") if self.optional_params else None,
             )
@@ -3901,7 +3925,7 @@ class Logging(LiteLLMLoggingBaseClass):
             return LiteLLMResponsesTransformationHandler().transform_response(
                 model=self.model,
                 raw_response=result,
-                model_response=litellm.ModelResponse(),
+                model_response=litellm.ModelResponse(id=_provider_response_id(result)),
                 logging_obj=self,
                 request_data={},
                 messages=[],
@@ -3916,7 +3940,7 @@ class Logging(LiteLLMLoggingBaseClass):
                 "usage-only ModelResponse to keep the spend_logs row.",
                 str(e),
             )
-            model_response: Final = litellm.ModelResponse()
+            model_response: Final = litellm.ModelResponse(id=_provider_response_id(result))
             model_response.model = self.model
             usage: Final = getattr(result, "usage", None)
             if usage is not None and ResponseAPILoggingUtils._is_response_api_usage(usage):

litellm/litellm_core_utils/llm_cost_calc/tool_call_cost_tracking.py

@@ -5,6 +5,8 @@ Helper utilities for tracking the cost of built-in tools.
 from collections.abc import Mapping
 from typing import Final, Literal
 
+from pydantic import ValidationError
+
 import litellm
 from litellm.constants import OPENAI_FILE_SEARCH_COST_PER_1K_CALLS
 from litellm.litellm_core_utils.llm_cost_calc.utils import (
@@ -13,6 +15,7 @@ from litellm.litellm_core_utils.llm_cost_calc.utils import (
 from litellm.types.llms.openai import (
     FileSearchTool,
     ResponsesAPIResponse,
+    ResponsesToolUsage,
     WebSearchOptions,
 )
 from litellm.types.utils import (
@@ -32,6 +35,17 @@ def _output_item_type(output_item: object) -> str | None:
     return item_type if isinstance(item_type, str) else None
 
 
+def _reported_web_search_requests(response_object: ResponsesAPIResponse) -> int | None:
+    tool_usage: Final = getattr(response_object, "tool_usage", None)
+    if tool_usage is None:
+        return None
+    try:
+        web_search: Final = ResponsesToolUsage.model_validate(tool_usage).web_search
+    except ValidationError:
+        return None
+    return None if web_search is None else web_search.num_requests
+
+
 def _usage_reports_server_side_web_search_calls(usage: Usage) -> bool:
     details: Final = getattr(usage, "server_side_tool_usage_details", None)
     if not isinstance(details, Mapping):
@@ -182,15 +196,19 @@ class StandardBuiltInToolCostTracking:
 
         Providers that report a request count in usage (gemini, anthropic, xai, vertex) are handled by
         get_cost_for_web_search_request and never reach here. This path prices per call, so it must count
-        the web_search_call items. Chat-completions responses only expose url_citation annotations with no
-        count, so they floor to a single billable search.
+        the web_search_call items, unless the response reports the billable count itself
+        (Bedrock's tool_usage.web_search.num_requests, which excludes open_page fetches). Chat-completions
+        responses only expose url_citation annotations with no count, so they floor to a single billable search.
         """
-        if isinstance(response_object, ResponsesAPIResponse):
-            count = sum(
-                1 for output_item in response_object.output if _output_item_type(output_item) == "web_search_call"
-            )
-            return max(count, 1)
-        return 1
+        if not isinstance(response_object, ResponsesAPIResponse):
+            return 1
+        reported: Final = _reported_web_search_requests(response_object)
+        if reported is not None:
+            return reported
+        count: Final = sum(
+            1 for output_item in response_object.output if _output_item_type(output_item) == "web_search_call"
+        )
+        return max(count, 1)
 
     @staticmethod
     def _handle_file_search_cost(

litellm/litellm_core_utils/llm_cost_calc/utils.py

@@ -415,40 +415,64 @@ def _is_off_peak(off_peak: Mapping[str, object], current_time: datetime | None =
     return False
 
 
-def _coerce_off_peak_rate(value: object, default: float) -> float:
+@dataclass(frozen=True, slots=True)
+class TokenRates:
+    input_rate: float
+    output_rate: float
+    cache_read_rate: float
+    cache_creation_rate: float
+    reasoning_rate: float | None
+
+    @property
+    def billed_reasoning_rate(self) -> float:
+        return self.output_rate if self.reasoning_rate is None else self.reasoning_rate
+
+
+def _parse_off_peak_rate(value: object) -> float | None:
     if isinstance(value, bool):
-        return default
+        return None
     if isinstance(value, (int, float)):
         return float(value)
     if isinstance(value, str):
         try:
             return float(value)
         except ValueError:
-            return default
-    return default
+            return None
+    return None
 
 
-def _apply_off_peak_pricing(
-    model_info: ModelInfo,
-    current_time: datetime | None,
-    prompt_base_cost: float,
-    completion_base_cost: float,
-    cache_read_cost: float,
-) -> tuple[float, float, float]:
+def _off_peak_rate(off_peak: Mapping[str, object], key: str, standard_rate: float) -> float:
+    parsed: Final = _parse_off_peak_rate(off_peak.get(key))
+    return standard_rate if parsed is None else parsed
+
+
+def _open_off_peak_block(model_info: ModelInfo, current_time: datetime | None) -> Mapping[str, object] | None:
+    off_peak: Final = model_info.get("off_peak_pricing")
+    if not isinstance(off_peak, Mapping) or not _is_off_peak(off_peak, current_time):
+        return None
+    return off_peak
+
+
+def apply_off_peak_pricing(model_info: ModelInfo, current_time: datetime | None, rates: TokenRates) -> TokenRates:
     """Swap in off-peak per-token rates when the current UTC time is inside one of the model's
     off_peak_pricing rules, the every-day hours_utc windows or a day-of-week-qualified entry in
     windows. An off-peak rate replaces the rate that would otherwise apply rather than
     discounting it, so a model that also has tiered or above-threshold pricing bills the flat
     off-peak rate for the whole request while the window is open. Any rate left unset in
-    off_peak_pricing falls back to the standard rate.
+    off_peak_pricing falls back to the standard rate, so a block without
+    output_cost_per_reasoning_token keeps the model's own reasoning rate, or its off-peak output
+    rate when reasoning has no dedicated rate at all.
     """
-    off_peak: Final = model_info.get("off_peak_pricing")
-    if not isinstance(off_peak, Mapping) or not _is_off_peak(off_peak, current_time):
-        return prompt_base_cost, completion_base_cost, cache_read_cost
-    return (
-        _coerce_off_peak_rate(off_peak.get("input_cost_per_token"), prompt_base_cost),
-        _coerce_off_peak_rate(off_peak.get("output_cost_per_token"), completion_base_cost),
-        _coerce_off_peak_rate(off_peak.get("cache_read_input_token_cost"), cache_read_cost),
+    off_peak: Final = _open_off_peak_block(model_info, current_time)
+    if off_peak is None:
+        return rates
+    off_peak_reasoning_rate: Final = _parse_off_peak_rate(off_peak.get("output_cost_per_reasoning_token"))
+    return TokenRates(
+        input_rate=_off_peak_rate(off_peak, "input_cost_per_token", rates.input_rate),
+        output_rate=_off_peak_rate(off_peak, "output_cost_per_token", rates.output_rate),
+        cache_read_rate=_off_peak_rate(off_peak, "cache_read_input_token_cost", rates.cache_read_rate),
+        cache_creation_rate=_off_peak_rate(off_peak, "cache_creation_input_token_cost", rates.cache_creation_rate),
+        reasoning_rate=rates.reasoning_rate if off_peak_reasoning_rate is None else off_peak_reasoning_rate,
     )
 
 
@@ -458,14 +482,28 @@ def _apply_off_peak_to_base_costs(
     base_costs: tuple[float, float, float, float, float],
 ) -> tuple[float, float, float, float, float]:
     """Apply off-peak rates to an already-resolved set of base costs, whichever pricing path
-    produced them. Cache-creation rates are passed through untouched, since off_peak_pricing
-    has no field for them.
+    produced them. The one-hour cache-creation rate passes through untouched, since
+    off_peak_pricing has no field for it, and reasoning is left to _resolve_billed_reasoning_rate.
     """
     prompt, completion, cache_creation, cache_creation_above_1hr, cache_read = base_costs
-    off_peak_prompt, off_peak_completion, off_peak_cache_read = _apply_off_peak_pricing(
-        model_info, current_time, prompt, completion, cache_read
+    rates: Final = apply_off_peak_pricing(
+        model_info,
+        current_time,
+        TokenRates(
+            input_rate=prompt,
+            output_rate=completion,
+            cache_read_rate=cache_read,
+            cache_creation_rate=cache_creation,
+            reasoning_rate=None,
+        ),
+    )
+    return (
+        rates.input_rate,
+        rates.output_rate,
+        rates.cache_creation_rate,
+        cache_creation_above_1hr,
+        rates.cache_read_rate,
     )
-    return (off_peak_prompt, off_peak_completion, cache_creation, cache_creation_above_1hr, off_peak_cache_read)
 
 
 def _get_token_base_cost(
@@ -1029,6 +1067,29 @@ def _resolve_reasoning_token_cost(
     return standard_reasoning_cost if standard_reasoning_cost is not None else completion_base_cost
 
 
+def _resolve_billed_reasoning_rate(
+    model_info: ModelInfo,
+    usage: Usage,
+    service_tier: str | None,
+    completion_base_cost: float,
+    current_time: datetime | None,
+) -> float:
+    off_peak: Final = _open_off_peak_block(model_info, current_time)
+    off_peak_reasoning_rate: Final = (
+        None if off_peak is None else _parse_off_peak_rate(off_peak.get("output_cost_per_reasoning_token"))
+    )
+    if off_peak_reasoning_rate is not None:
+        return off_peak_reasoning_rate
+    tiered_reasoning_rate: Final = _get_tiered_reasoning_rate(model_info=model_info, usage=usage)
+    if tiered_reasoning_rate is not None:
+        return tiered_reasoning_rate
+    return _resolve_reasoning_token_cost(
+        model_info=model_info,
+        service_tier=service_tier,
+        completion_base_cost=completion_base_cost,
+    )
+
+
 def generic_cost_per_token(
     model: str,
     usage: Usage,
@@ -1037,6 +1098,7 @@ def generic_cost_per_token(
     data_residency: str | None = None,
     model_info: ModelInfo | None = None,
     vertex_location: str | None = None,
+    current_time: datetime | None = None,
 ) -> tuple[float, float]:
     """
     Calculates the cost per token for a given model, prompt tokens, and completion tokens.
@@ -1051,6 +1113,7 @@ def generic_cost_per_token(
         - vertex_location: optional Vertex AI location the request was served from
           (e.g. "us-east5", "global"), used to apply the per-model
           regional-endpoint uplift multiplier when non-global.
+        - current_time: the moment the request is billed at, for off_peak_pricing; defaults to now, UTC
 
     Returns:
         Tuple[float, float] - prompt_cost_in_usd, completion_cost_in_usd
@@ -1117,6 +1180,7 @@ def generic_cost_per_token(
             usage.prompt_tokens - cache_hit - audio_tokens - cache_creation - image_tokens - video_tokens, 0
         )
 
+    billing_time: Final = current_time if current_time is not None else datetime.now(timezone.utc)
     (
         prompt_base_cost,
         completion_base_cost,
@@ -1127,6 +1191,7 @@ def generic_cost_per_token(
         model_info=model_info,
         usage=usage,
         service_tier=service_tier,
+        current_time=billing_time,
         threshold_is_inclusive=_uses_inclusive_token_thresholds(custom_llm_provider),
     )
 
@@ -1185,17 +1250,13 @@ def generic_cost_per_token(
 
     ## REASONING COST
     if not is_text_tokens_total and reasoning_tokens and reasoning_tokens > 0:
-        tiered_reasoning_rate: Final = _get_tiered_reasoning_rate(model_info=model_info, usage=usage)
-        _output_cost_per_reasoning_token = (
-            tiered_reasoning_rate
-            if tiered_reasoning_rate is not None
-            else _resolve_reasoning_token_cost(
-                model_info=model_info,
-                service_tier=service_tier,
-                completion_base_cost=completion_base_cost,
-            )
+        completion_cost += float(reasoning_tokens) * _resolve_billed_reasoning_rate(
+            model_info=model_info,
+            usage=usage,
+            service_tier=service_tier,
+            completion_base_cost=completion_base_cost,
+            current_time=billing_time,
         )
-        completion_cost += float(reasoning_tokens) * _output_cost_per_reasoning_token
 
     ## IMAGE COST
     if not is_text_tokens_total and image_tokens and image_tokens > 0:
@@ -1247,6 +1308,7 @@ def get_token_type_cost_breakdown(
     service_tier: str | None = None,
     data_residency: str | None = None,
     vertex_location: str | None = None,
+    current_time: datetime | None = None,
 ) -> TokenTypeCostBreakdown:
     """
     Provider-agnostic cost of reasoning and cache tokens, derived from the usage
@@ -1265,6 +1327,7 @@ def get_token_type_cost_breakdown(
     except Exception:
         return TokenTypeCostBreakdown(0.0, 0.0, 0.0)
 
+    billing_time: Final = current_time if current_time is not None else datetime.now(timezone.utc)
     (
         _prompt_base_cost,
         completion_base_cost,
@@ -1275,6 +1338,7 @@ def get_token_type_cost_breakdown(
         model_info=model_info,
         usage=usage,
         service_tier=service_tier,
+        current_time=billing_time,
         threshold_is_inclusive=_uses_inclusive_token_thresholds(custom_llm_provider),
     )
 
@@ -1284,18 +1348,12 @@ def get_token_type_cost_breakdown(
     if not reasoning_tokens:
         reasoning_tokens = _coerce_token_count(getattr(usage, "reasoning_tokens", 0))
 
-    # Reasoning is billed at the selected tier's reasoning rate for tiered models,
-    # else at the service-tier-aware per-reasoning-token rate - this mirrors how the
-    # total completion cost is computed, so the breakdown can never diverge from it.
-    tiered_reasoning_rate: Final = _get_tiered_reasoning_rate(model_info=model_info, usage=usage)
-    reasoning_rate: Final = (
-        tiered_reasoning_rate
-        if tiered_reasoning_rate is not None
-        else _resolve_reasoning_token_cost(
-            model_info=model_info,
-            service_tier=service_tier,
-            completion_base_cost=completion_base_cost,
-        )
+    reasoning_rate: Final = _resolve_billed_reasoning_rate(
+        model_info=model_info,
+        usage=usage,
+        service_tier=service_tier,
+        completion_base_cost=completion_base_cost,
+        current_time=billing_time,
     )
     reasoning_cost = float(reasoning_tokens) * reasoning_rate
 

litellm/litellm_core_utils/secret_redaction.py

@@ -11,7 +11,7 @@ from typing import Final
 
 from litellm.constants import MINIMUM_CUSTOM_KEY_LENGTH
 
-_REDACTED: Final = "REDACTED"
+REDACTED: Final = "REDACTED"
 
 
 def _build_secret_patterns() -> "re.Pattern[str]":
@@ -89,7 +89,7 @@ _SECRET_RE: Final = _build_secret_patterns()
 
 def redact_string(value: str) -> str:
     """Scrub known secret/credential patterns from *value* and return the result."""
-    return _SECRET_RE.sub(_REDACTED, value)
+    return _SECRET_RE.sub(REDACTED, value)
 
 
 _UNIX_SYSTEM_PATH: Final = r"/(?:etc|var|opt|usr|home|root|private|Users|tmp|mnt|srv)/[^\s'\"\)\]}>,]+"
@@ -110,7 +110,7 @@ def redact_internal_details(value: str) -> str:
     on top of redact_string(). For client-facing messages only: server logs keep this detail."""
     marker_index: Final = value.find(_TRACEBACK_MARKER)
     without_traceback: Final = value[:marker_index].rstrip() if marker_index != -1 else value
-    return _INTERNAL_DETAIL_RE.sub(_REDACTED, redact_string(without_traceback))
+    return _INTERNAL_DETAIL_RE.sub(REDACTED, redact_string(without_traceback))
 
 
 def redact_structured_value(key: str | None, value: str) -> str:
@@ -126,4 +126,4 @@ def redact_structured_value(key: str | None, value: str) -> str:
     if scrubbed != value or key is None:
         return scrubbed
     rendered: Final = f"'{key}': '{value}'"
-    return _REDACTED if redact_string(rendered) != rendered else value
+    return REDACTED if redact_string(rendered) != rendered else value

litellm/litellm_core_utils/sensitive_data_masker.py

@@ -1,9 +1,10 @@
-from collections.abc import Mapping
+from collections.abc import Mapping, Sequence
 from typing import Any, Final
 
 from pydantic import BaseModel
 
-from litellm.constants import DEFAULT_MAX_RECURSE_DEPTH_SENSITIVE_DATA_MASKER
+from litellm.constants import DEFAULT_MAX_RECURSE_DEPTH, DEFAULT_MAX_RECURSE_DEPTH_SENSITIVE_DATA_MASKER
+from litellm.litellm_core_utils.secret_redaction import REDACTED
 
 
 class SensitiveDataMasker:
@@ -214,6 +215,46 @@ def mask_sensitive_keys(data: dict[str, Any], sensitive_fields: set[str]) -> dic
     return masked
 
 
+def redact_credentials_in_payload(data: Mapping[str, object]) -> Mapping[str, object]:
+    """Return a copy of ``data`` where every value under a credential-named key is
+    replaced by the shared ``REDACTED`` marker, nested mappings are recursed into,
+    and every other value is preserved by identity.
+
+    Sensitive-key detection is delegated to the shared :class:`SensitiveDataMasker`,
+    so the credential names stay in one place. Unlike
+    :func:`mask_credentials_in_payload`, no prefix or suffix of the secret survives
+    and non-string secrets are covered too, which is what a payload rendered
+    straight to stdout needs. ``None`` is preserved so an unset credential still
+    reads as unset, and lists and tuples are rebuilt element by element so a
+    credential nested inside one is caught as well. The walk is bounded only to stop
+    runaway recursion, and a container sitting at that bound is replaced wholesale
+    rather than passed through, so burying a credential deeper than the walk goes
+    hides it instead of exposing it.
+    """
+    return _redact_mapping(data, 0)
+
+
+def _redact_mapping(data: Mapping[str, object], depth: int) -> Mapping[str, object]:
+    return {key: _redact_entry(key, value, depth) for key, value in data.items()}
+
+
+def _redact_entry(key: str, value: object, depth: int) -> object:
+    if value is not None and _default_masker.is_sensitive_key(key):
+        return REDACTED
+    if not isinstance(value, (Mapping, list, tuple)):
+        return value
+    if depth >= DEFAULT_MAX_RECURSE_DEPTH:
+        return REDACTED
+    if isinstance(value, Mapping):
+        return _redact_mapping(value, depth + 1)
+    return _redact_sequence(value, depth + 1)
+
+
+def _redact_sequence(values: Sequence[object], depth: int) -> Sequence[object]:
+    redacted: Final = tuple(_redact_entry("", item, depth) for item in values)
+    return redacted if isinstance(values, tuple) else list(redacted)
+
+
 # Usage example:
 """
 masker = SensitiveDataMasker()

litellm/litellm_core_utils/streaming_handler.py

@@ -2337,6 +2337,9 @@ class CustomStreamWrapper:
         else:
             self.sent_last_chunk = True
             processed_chunk: Final = self.finish_reason_handler()
+            if self.stream_options is None:
+                usage: Final = calculate_total_usage(chunks=self.chunks)
+                processed_chunk._hidden_params["usage"] = usage  # pyright: ignore[reportPrivateUsage]  # sync parity
             # see sync __next__'s sibling branch: deliberately do NOT restore
             # here - this chunk is still this call's own data, and restoring
             # before returning it would corrupt the caller's own log

litellm/llms/anthropic/experimental_pass_through/adapters/handler.py

@@ -21,6 +21,7 @@ from litellm.llms.anthropic.experimental_pass_through.context_management import
 )
 from litellm.llms.anthropic.experimental_pass_through.utils import (
     is_reasoning_auto_summary_enabled,
+    litellm_logging_obj_from_kwargs,
     local_model_name,
 )
 from litellm.types.llms.anthropic_messages.anthropic_response import (
@@ -621,6 +622,7 @@ class LiteLLMMessagesToCompletionTransformationHandler:
                 tool_name_mapping=tool_name_mapping,
                 polyfill_result=polyfill_result,
                 is_async=True,
+                litellm_logging_obj=litellm_logging_obj_from_kwargs(kwargs),
             )
             if transformed_stream is not None:
                 return transformed_stream
@@ -755,6 +757,7 @@ class LiteLLMMessagesToCompletionTransformationHandler:
                 tool_name_mapping=tool_name_mapping,
                 polyfill_result=polyfill_result,
                 is_async=False,
+                litellm_logging_obj=litellm_logging_obj_from_kwargs(kwargs),
             )
             if transformed_stream is not None:
                 return transformed_stream

litellm/llms/anthropic/experimental_pass_through/adapters/streaming_iterator.py

@@ -31,6 +31,7 @@ from litellm.types.llms.anthropic import (
 from litellm.types.utils import AdapterCompletionStreamWrapper, Delta
 
 if TYPE_CHECKING:
+    from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObject
     from litellm.types.utils import ModelResponseStream
 
 
@@ -287,12 +288,16 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper):
         applied_edits: list[AppliedEdit] | None = None,
         compaction_block: CompactionBlock | None = None,
         iterations_usage: list[UsageIteration] | None = None,
+        litellm_logging_obj: "LiteLLMLoggingObject | None" = None,
     ):
         # Wrap the upstream stream so chunks that carry both content and a
         # finish_reason (fake-streamed providers) are split into two — see
         # _CombinedChunkSplitter.
         super().__init__(_CombinedChunkSplitter(completion_stream))
         self.model = model
+        self._message_id: str = f"msg_{uuid.uuid4()}"
+        if litellm_logging_obj is not None:
+            litellm_logging_obj.record_streamed_anthropic_message_id(self._message_id)
         # Mapping of truncated tool names to original names (for OpenAI's 64-char limit)
         self.tool_name_mapping = tool_name_mapping or {}
         # Polyfill applied_edits on final message_delta.
@@ -507,7 +512,7 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper):
                     {
                         "type": "message_start",
                         "message": {
-                            "id": f"msg_{uuid.uuid4()}",
+                            "id": self._message_id,
                             "type": "message",
                             "role": "assistant",
                             "content": [],
@@ -741,7 +746,7 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper):
                     {
                         "type": "message_start",
                         "message": {
-                            "id": f"msg_{uuid.uuid4()}",
+                            "id": self._message_id,
                             "type": "message",
                             "role": "assistant",
                             "content": [],

litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py

@@ -174,6 +174,7 @@ from litellm.types.utils import Choices, ModelResponse, StreamingChoices, Usage
 from .streaming_iterator import AnthropicStreamWrapper
 
 if TYPE_CHECKING:
+    from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObject
     from litellm.types.llms.anthropic import ContentBlockContentBlockDict
 
 ToolResultContent: TypeAlias = str | list[ToolMessageContentPart]
@@ -264,6 +265,7 @@ class AnthropicAdapter:
         tool_name_mapping: dict[str, str] | None = None,
         polyfill_result: PolyfillResult | None = None,
         is_async: bool = True,
+        litellm_logging_obj: "LiteLLMLoggingObject | None" = None,
     ) -> AsyncIterator[bytes] | Iterator[bytes] | None:
         """
         Translate OpenAI streaming response to Anthropic format.
@@ -290,6 +292,7 @@ class AnthropicAdapter:
             applied_edits=applied_edits,
             compaction_block=compaction_block,
             iterations_usage=iterations_usage,
+            litellm_logging_obj=litellm_logging_obj,
         )
         # Return the SSE-wrapped version for proper event formatting.
         if is_async:

litellm/llms/anthropic/experimental_pass_through/messages/utils.py

@@ -6,6 +6,7 @@ from litellm.types.llms.anthropic import AnthropicMessagesRequestOptionalParams
 from litellm.types.llms.anthropic_messages.anthropic_response import (
     AnthropicMessagesResponse,
 )
+from litellm.types.llms.openai import ChatCompletionSystemMessage
 
 if TYPE_CHECKING:
     from litellm.exceptions import ContentPolicyViolationError
@@ -36,6 +37,16 @@ def safeguard_refusal_error(model: str, stop_details: Mapping[str, object]) -> "
     )
 
 
+def anthropic_system_to_openai_message(system: object) -> ChatCompletionSystemMessage | None:
+    """
+    Return the Anthropic Messages top-level ``system`` (a string or a list of text
+    blocks) as an OpenAI-style system message, or None when the request has none.
+    """
+    if not isinstance(system, (str, list)) or not system:
+        return None
+    return ChatCompletionSystemMessage(role="system", content=system)
+
+
 @lru_cache(maxsize=1)
 def _anthropic_messages_optional_param_keys() -> frozenset[str]:
     """

litellm/llms/anthropic/experimental_pass_through/responses_adapters/handler.py

@@ -20,7 +20,7 @@ from litellm.types.llms.anthropic_messages.anthropic_response import (
 )
 from litellm.types.llms.openai import ResponsesAPIResponse
 
-from ..utils import local_model_name
+from ..utils import litellm_logging_obj_from_kwargs, local_model_name
 from .streaming_iterator import AnthropicResponsesStreamWrapper
 from .transformation import LiteLLMAnthropicToResponsesAPIAdapter
 
@@ -186,7 +186,9 @@ class LiteLLMMessagesToResponsesAPIHandler:
 
         if stream:
             wrapper: Final = AnthropicResponsesStreamWrapper(
-                responses_stream=result, model=local_model_name(model, kwargs.get("custom_llm_provider"))
+                responses_stream=result,
+                model=local_model_name(model, kwargs.get("custom_llm_provider")),
+                litellm_logging_obj=litellm_logging_obj_from_kwargs(responses_kwargs),
             )
             return wrapper.async_anthropic_sse_wrapper()
 
@@ -266,7 +268,9 @@ class LiteLLMMessagesToResponsesAPIHandler:
 
         if stream:
             wrapper: Final = AnthropicResponsesStreamWrapper(
-                responses_stream=result, model=local_model_name(model, kwargs.get("custom_llm_provider"))
+                responses_stream=result,
+                model=local_model_name(model, kwargs.get("custom_llm_provider")),
+                litellm_logging_obj=litellm_logging_obj_from_kwargs(responses_kwargs),
             )
             return wrapper.async_anthropic_sse_wrapper()
 

litellm/llms/anthropic/experimental_pass_through/responses_adapters/streaming_iterator.py

@@ -4,7 +4,7 @@ import json
 import traceback
 from collections import deque
 from collections.abc import AsyncIterator, Mapping
-from typing import Any, Final
+from typing import TYPE_CHECKING, Any, Final
 
 from litellm import verbose_logger
 from litellm._uuid import uuid
@@ -12,6 +12,9 @@ from litellm.types.llms.anthropic_messages.anthropic_response import AnthropicUs
 
 from .transformation import LiteLLMAnthropicToResponsesAPIAdapter
 
+if TYPE_CHECKING:
+    from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObject
+
 
 class AnthropicResponsesStreamWrapper:
     """
@@ -31,10 +34,13 @@ class AnthropicResponsesStreamWrapper:
         self,
         responses_stream: Any,
         model: str,
+        litellm_logging_obj: "LiteLLMLoggingObject | None" = None,
     ) -> None:
         self.responses_stream = responses_stream
         self.model = model
         self._message_id: str = f"msg_{uuid.uuid4()}"
+        if litellm_logging_obj is not None:
+            litellm_logging_obj.record_streamed_anthropic_message_id(self._message_id)
         self._current_block_index: int = -1
         # Map item_id -> content_block_index so we can stop the right block later
         self._item_id_to_block_index: dict[str, int] = {}

litellm/llms/anthropic/experimental_pass_through/utils.py

@@ -1,11 +1,14 @@
 import os
 from collections.abc import Mapping
 from types import MappingProxyType
-from typing import Final
+from typing import TYPE_CHECKING, Final
 
 import litellm
 from litellm.types.utils import ModelInfo
 
+if TYPE_CHECKING:
+    from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObject
+
 OPENAI_MAX_PROMPT_CACHE_KEY_LENGTH: Final = 64
 
 _EFFORT_DEGRADATION_CHAIN: Final[Mapping[str, tuple[str, ...]]] = MappingProxyType(
@@ -24,6 +27,14 @@ def prompt_cache_key_from_user_id(user_id: object) -> str | None:
     return str(user_id)[:OPENAI_MAX_PROMPT_CACHE_KEY_LENGTH] or None
 
 
+def litellm_logging_obj_from_kwargs(kwargs: Mapping[str, object]) -> "LiteLLMLoggingObject | None":
+    """The logging object the bridged call logs through, when the caller supplied one."""
+    from litellm.litellm_core_utils.litellm_logging import Logging
+
+    candidate: Final = kwargs.get("litellm_logging_obj")
+    return candidate if isinstance(candidate, Logging) else None
+
+
 def local_model_name(model: str, custom_llm_provider: object) -> str:
     """The id the provider itself knows, for reporting back to the caller in ``message_start``."""
     return model.removeprefix(f"{custom_llm_provider}/") if isinstance(custom_llm_provider, str) else model

litellm/llms/azure/chat/gpt_5_transformation.py

@@ -7,6 +7,7 @@ from litellm.exceptions import UnsupportedParamsError
 from litellm.llms.openai.chat.gpt_5_transformation import (
     OpenAIGPT5Config,
     _get_effort_level,
+    is_gpt_reasoning_series_name,
 )
 from litellm.types.llms.openai import AllMessageValues
 
@@ -35,26 +36,7 @@ class AzureOpenAIGPT5Config(AzureOpenAIConfig, OpenAIGPT5Config):
 
     @classmethod
     def is_model_gpt_5_model(cls, model: str) -> bool:
-        """Check if the Azure model string refers to a gpt-5 variant.
-
-        Accepts both explicit gpt-5 model names and the ``gpt5_series/`` prefix
-        used for manual routing.
-        """
-        # The gpt-5-chat* family (gpt-5-chat, gpt-5-chat-latest, gpt-5-chat-2025-08-07,
-        # …) are regular chat models: they support temperature and tool_choice but NOT
-        # reasoning_effort.  They must NOT be routed through the GPT-5 reasoning path.
-        #
-        # Versioned chat models such as gpt-5.3-chat and gpt-5.1-chat ARE reasoning
-        # models and must stay on the GPT-5 path.  The distinguishing feature is that
-        # the gpt-5-chat family has a literal "-chat" immediately after "gpt-5"
-        # (i.e. "gpt-5-chat…"), while versioned chat models interpose a minor version
-        # number (i.e. "gpt-5.<digit>-chat").
-        #
-        # Using a startswith("gpt-5-chat") prefix check on the normalized name (rather
-        # than a substring check) makes this boundary explicit and avoids any ambiguity
-        # if future model names coincidentally contain "gpt-5-chat" as an interior run.
-        _normalized: Final = model.split("/")[-1]  # strip provider prefix, e.g. "azure/"
-        return ("gpt-5" in model and not _normalized.startswith("gpt-5-chat")) or "gpt5_series" in model
+        return is_gpt_reasoning_series_name(model) or "gpt5_series" in model
 
     def get_supported_openai_params(self, model: str) -> list[str]:
         """Get supported parameters for Azure OpenAI GPT-5 models.

litellm/llms/azure/chat/gpt_transformation.py

@@ -14,6 +14,7 @@ from litellm.litellm_core_utils.prompt_templates.factory import (
     convert_to_azure_openai_messages,
 )
 from litellm.llms.base_llm.chat.transformation import BaseLLMException
+from litellm.llms.openai.chat.gpt_5_transformation import GPT_REASONING_SERIES_MARKERS
 from litellm.types.llms.azure import (
     API_VERSION_MONTH_SUPPORTED_RESPONSE_FORMAT,
     API_VERSION_YEAR_SUPPORTED_RESPONSE_FORMAT,
@@ -139,7 +140,7 @@ class AzureOpenAIConfig(BaseConfig):
         name family needs the rename, including the ``gpt-5-chat*`` models that are excluded from
         the reasoning path by https://github.com/BerriAI/litellm/issues/13781.
         """
-        return "gpt-5" in model or "gpt5_series" in model
+        return any(marker in model for marker in GPT_REASONING_SERIES_MARKERS) or "gpt5_series" in model
 
     def _is_response_format_supported_model(self, model: str) -> bool:
         """

litellm/llms/azure_ai/chat/transformation.py

@@ -15,6 +15,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import (
     filter_value_from_dict,
 )
 from litellm.llms.azure.common_utils import BaseAzureLLM
+from litellm.llms.azure_ai.common_utils import is_foundry_model_inference_base
 from litellm.llms.base_llm.chat.transformation import LiteLLMLoggingObj
 from litellm.llms.openai.common_utils import drop_params_from_unprocessable_entity_error
 from litellm.llms.openai.openai import OpenAIConfig
@@ -207,20 +208,18 @@ class AzureAIStudioConfig(OpenAIConfig):
                 message["content"] = texts
         return stripped_messages
 
-    def _is_azure_openai_model(self, model: str, api_base: str | None) -> bool:
-        try:
-            if "/" in model:
-                model = model.split("/", 1)[1]
-            if (
-                model in litellm.open_ai_chat_completion_models
-                or model in litellm.open_ai_text_completion_models
-                or model in litellm.open_ai_embedding_models
-            ):
-                return True
+    def _is_foundry_model_inference_base(self, api_base: str) -> bool:
+        return is_foundry_model_inference_base(api_base)
 
-        except Exception:
+    def _is_azure_openai_model(self, model: str, api_base: str | None) -> bool:
+        if api_base is None or self._is_foundry_model_inference_base(api_base):
             return False
-        return False
+        stripped_model: Final = model.split("/", 1)[1] if "/" in model else model
+        return (
+            stripped_model in litellm.open_ai_chat_completion_models
+            or stripped_model in litellm.open_ai_text_completion_models
+            or stripped_model in litellm.open_ai_embedding_models
+        )
 
     def _get_openai_compatible_provider_info(
         self,

litellm/llms/azure_ai/common_utils.py

@@ -1,5 +1,6 @@
 from collections.abc import Mapping
 from typing import Final, Literal
+from urllib.parse import urlparse
 
 import litellm
 from litellm.llms.base_llm.base_utils import BaseLLMModelInfo, BaseTokenCounter
@@ -10,6 +11,14 @@ from litellm.types.router import GenericLiteLLMParams
 AzureAIApiKeyHeader = Literal["Authorization", "api-key", "Api-Key", "Ocp-Apim-Subscription-Key"]
 
 
+def is_foundry_model_inference_base(api_base: str) -> bool:
+    parsed: Final = urlparse(api_base)
+    host: Final = parsed.hostname
+    if host is None or not host.endswith(".services.ai.azure.com"):
+        return False
+    return "/openai/deployments" not in parsed.path
+
+
 def get_azure_ai_entra_token(litellm_params: Mapping[str, object] | None = None) -> str | None:
     """
     Resolve an Entra ID / OAuth access token for an Azure AI Foundry deployment.

litellm/llms/azure_ai/embed/handler.py

@@ -1,8 +1,10 @@
 from typing import Final
+from urllib.parse import urlsplit, urlunsplit
 
 from openai import OpenAI
 
 import litellm
+from litellm.llms.azure_ai.common_utils import is_foundry_model_inference_base
 from litellm.llms.custom_httpx.http_handler import (
     AsyncHTTPHandler,
     HTTPHandler,
@@ -16,6 +18,16 @@ from litellm.utils import convert_to_model_response_object
 from .cohere_transformation import AzureAICohereConfig
 
 
+def _foundry_models_route_base(api_base: str | None) -> str | None:
+    if api_base is None or not is_foundry_model_inference_base(api_base):
+        return api_base
+    parts: Final = urlsplit(api_base)
+    path: Final = parts.path.rstrip("/")
+    if path.endswith("/models"):
+        return api_base
+    return urlunsplit((parts.scheme, parts.netloc, f"{path}/models", parts.query, parts.fragment))
+
+
 class AzureAIEmbedding(OpenAIChatCompletion):
     def _process_response(
         self,
@@ -214,6 +226,7 @@ class AzureAIEmbedding(OpenAIChatCompletion):
 
         assemble result in-order, and return
         """
+        resolved_api_base: Final = _foundry_models_route_base(api_base)
         if aembedding is True:
             return self.async_embedding(
                 model,
@@ -223,7 +236,7 @@ class AzureAIEmbedding(OpenAIChatCompletion):
                 model_response,
                 optional_params,
                 api_key,
-                api_base,
+                resolved_api_base,
                 client,
             )
 
@@ -245,7 +258,7 @@ class AzureAIEmbedding(OpenAIChatCompletion):
                 model_response=model_response,
                 optional_params=optional_params,
                 api_key=api_key,
-                api_base=api_base,
+                api_base=resolved_api_base,
                 client=client,
             )
 
@@ -262,7 +275,7 @@ class AzureAIEmbedding(OpenAIChatCompletion):
                 model_response,
                 optional_params,
                 api_key,
-                api_base,
+                resolved_api_base,
                 client=(client if client is not None and isinstance(client, OpenAI) else None),
                 aembedding=aembedding,
                 shared_session=shared_session,

litellm/llms/bedrock/base_aws_llm.py

@@ -7,7 +7,7 @@ import urllib.parse
 from collections.abc import Callable
 from datetime import datetime
 from threading import Lock
-from typing import TYPE_CHECKING, Any, ClassVar, Final, Literal, cast, get_args
+from typing import TYPE_CHECKING, Any, ClassVar, Final, Literal, cast, get_args, overload
 
 import httpx
 from pydantic import BaseModel, ValidationError
@@ -48,12 +48,24 @@ _STS_REGION_FROM_ENDPOINT_PATTERN: Final = re.compile(
 SIGV4_COMPUTED_HEADERS: Final = frozenset({"authorization", "x-amz-date", "x-amz-security-token", "date"})
 
 
-class Boto3CredentialsInfo(BaseModel):
-    credentials: Credentials
+class BedrockRequestTarget(BaseModel):
     aws_region_name: str
     aws_bedrock_runtime_endpoint: str | None
 
 
+class Boto3CredentialsInfo(BedrockRequestTarget):
+    credentials: Credentials
+
+
+class BearerRequestTarget(BedrockRequestTarget):
+    credentials: None = None
+
+
+def bedrock_bearer_token(api_key: str | None) -> str | None:
+    token: Final = api_key if api_key is not None else get_secret_str("AWS_BEARER_TOKEN_BEDROCK")
+    return token or None
+
+
 class _WebIdentityTokenClaims(BaseModel):
     aud: str | list[str] | None = None
     iss: str | None = None
@@ -1387,9 +1399,26 @@ class BaseAWSLLM:
         else:
             return f"https://bedrock-runtime.{aws_region_name}.{dns_suffix}"
 
+    @overload
     def _get_boto_credentials_from_optional_params(
-        self, optional_params: dict, model: str | None = None
-    ) -> Boto3CredentialsInfo:
+        self,
+        optional_params: dict,  # mutable-ok: the implementation pops the aws_* keys out of the caller's dict in place
+        model: str | None = None,
+        bearer_token: None = None,
+    ) -> Boto3CredentialsInfo: ...
+
+    @overload
+    def _get_boto_credentials_from_optional_params(
+        self,
+        optional_params: dict,  # mutable-ok: the implementation pops the aws_* keys out of the caller's dict in place
+        model: str | None = None,
+        *,
+        bearer_token: str,
+    ) -> BearerRequestTarget: ...
+
+    def _get_boto_credentials_from_optional_params(
+        self, optional_params: dict, model: str | None = None, bearer_token: str | None = None
+    ) -> Boto3CredentialsInfo | BearerRequestTarget:
         """
         Get boto3 credentials from optional params
 
@@ -1420,6 +1449,12 @@ class BaseAWSLLM:
         )  # https://bedrock-runtime.{region_name}.amazonaws.com
         aws_external_id: Final = optional_params.pop("aws_external_id", None)
 
+        if bearer_token is not None:
+            return BearerRequestTarget(
+                aws_region_name=aws_region_name,
+                aws_bedrock_runtime_endpoint=aws_bedrock_runtime_endpoint,
+            )
+
         credentials: Final[Credentials] = self.get_credentials(
             aws_access_key_id=aws_access_key_id,
             aws_secret_access_key=aws_secret_access_key,
@@ -1432,7 +1467,6 @@ class BaseAWSLLM:
             aws_sts_endpoint=aws_sts_endpoint,
             aws_external_id=aws_external_id,
         )
-
         return Boto3CredentialsInfo(
             credentials=credentials,
             aws_region_name=aws_region_name,
@@ -1451,14 +1485,9 @@ class BaseAWSLLM:
         api_key: str | None = None,
         supports_bearer_token: bool = True,
     ) -> AWSPreparedRequest:
-        if not supports_bearer_token:
-            aws_bearer_token: str | None = None
-        elif api_key is not None:
-            aws_bearer_token = api_key
-        else:
-            aws_bearer_token = get_secret_str("AWS_BEARER_TOKEN_BEDROCK")
+        aws_bearer_token: Final = bedrock_bearer_token(api_key) if supports_bearer_token else None
 
-        if aws_bearer_token:
+        if aws_bearer_token is not None:
             try:
                 from botocore.awsrequest import AWSRequest
             except ImportError:
@@ -1555,13 +1584,9 @@ class BaseAWSLLM:
         Returns:
             Tuple[dict, Optional[str]]: A tuple containing the headers and the json str body of the request
         """
-        if api_key is not None:
-            aws_bearer_token: str | None = api_key
-        else:
-            aws_bearer_token = get_secret_str("AWS_BEARER_TOKEN_BEDROCK")
+        aws_bearer_token: Final = bedrock_bearer_token(api_key)
 
-        # If aws bearer token is set, use it directly in the header
-        if aws_bearer_token:
+        if aws_bearer_token is not None:
             headers = headers or {}
             headers["Content-Type"] = "application/json"
             headers["Authorization"] = f"Bearer {aws_bearer_token}"

litellm/llms/bedrock/chat/converse_handler.py

@@ -21,7 +21,7 @@ from litellm.rust_bridge.chat_completions import rust_chat_completions_accepts
 from litellm.types.utils import ModelResponse
 from litellm.utils import CustomStreamWrapper
 
-from ..base_aws_llm import BaseAWSLLM, Credentials
+from ..base_aws_llm import BaseAWSLLM, Credentials, bedrock_bearer_token
 from ..common_utils import BedrockError, _get_all_bedrock_regions
 from .invoke_handler import AWSEventStreamDecoder, MockResponseIterator, make_call
 
@@ -349,17 +349,21 @@ class BedrockConverseLLM(BaseAWSLLM):
 
         litellm_params["aws_region_name"] = aws_region_name  # [DO NOT DELETE] important for async calls
 
-        credentials: Final[Credentials | None] = self.get_credentials(
-            aws_access_key_id=aws_access_key_id,
-            aws_secret_access_key=aws_secret_access_key,
-            aws_session_token=aws_session_token,
-            aws_region_name=aws_region_name,
-            aws_session_name=aws_session_name,
-            aws_profile_name=aws_profile_name,
-            aws_role_name=aws_role_name,
-            aws_web_identity_token=aws_web_identity_token,
-            aws_sts_endpoint=aws_sts_endpoint,
-            aws_external_id=aws_external_id,
+        credentials: Final[Credentials | None] = (
+            None
+            if bedrock_bearer_token(api_key) is not None
+            else self.get_credentials(
+                aws_access_key_id=aws_access_key_id,
+                aws_secret_access_key=aws_secret_access_key,
+                aws_session_token=aws_session_token,
+                aws_region_name=aws_region_name,
+                aws_session_name=aws_session_name,
+                aws_profile_name=aws_profile_name,
+                aws_role_name=aws_role_name,
+                aws_web_identity_token=aws_web_identity_token,
+                aws_sts_endpoint=aws_sts_endpoint,
+                aws_external_id=aws_external_id,
+            )
         )
 
         ### SET RUNTIME ENDPOINT ###

litellm/llms/bedrock/chat/invoke_transformations/amazon_moonshot_transformation.py

@@ -149,19 +149,15 @@ class AmazonMoonshotConfig(AmazonInvokeConfig, MoonshotChatConfig):
         - Temperature and parameter validation
 
         """
-        # Filter out AWS credentials using the existing method from BaseAWSLLM
-        self._get_boto_credentials_from_optional_params(optional_params, model)
+        inference_params: Final = {k: v for k, v in optional_params.items() if k not in self.aws_authentication_params}
 
-        # Strip routing prefixes to get the actual model ID
         clean_model_id: Final = self._get_model_id(model)
 
-        # Use Moonshot's transform_request which handles message transformation
-        # and tool_choice="required" workaround
         return MoonshotChatConfig.transform_request(
             self,
             model=clean_model_id,
             messages=messages,
-            optional_params=optional_params,
+            optional_params=inference_params,
             litellm_params=litellm_params,
             headers=headers,
         )

litellm/llms/bedrock/embed/embedding.py

@@ -6,7 +6,7 @@ import copy
 import json
 import urllib.parse
 from collections.abc import Callable
-from typing import TYPE_CHECKING, Any, Final, get_args
+from typing import TYPE_CHECKING, Final, get_args, overload
 
 import httpx
 
@@ -26,7 +26,7 @@ from litellm.types.llms.bedrock import (
 )
 from litellm.types.utils import EmbeddingResponse, LlmProviders
 
-from ..base_aws_llm import BaseAWSLLM
+from ..base_aws_llm import BaseAWSLLM, Credentials, bedrock_bearer_token
 from ..common_utils import BedrockError
 from .amazon_nova_transformation import AmazonNovaEmbeddingConfig
 from .amazon_titan_g1_transformation import AmazonTitanG1Config
@@ -42,14 +42,25 @@ if TYPE_CHECKING:
 
 
 class BedrockEmbedding(BaseAWSLLM):
+    @overload
+    def _load_credentials(
+        self,
+        optional_params: dict,  # mutable-ok: the implementation pops the aws_* keys out of the caller's dict in place
+        bearer_token: None = None,
+    ) -> tuple[Credentials, str]: ...
+
+    @overload
+    def _load_credentials(
+        self,
+        optional_params: dict,  # mutable-ok: the implementation pops the aws_* keys out of the caller's dict in place
+        bearer_token: str,
+    ) -> tuple[None, str]: ...
+
     def _load_credentials(
         self,
         optional_params: dict,
-    ) -> tuple[Any, str]:
-        try:
-            from botocore.credentials import Credentials
-        except ImportError:
-            raise ImportError("Missing boto3 to call bedrock. Run 'pip install boto3'.")
+        bearer_token: str | None = None,
+    ) -> tuple[Credentials | None, str]:
         ## CREDENTIALS ##
         # pop aws_secret_access_key, aws_access_key_id, aws_session_token, aws_region_name from kwargs, since completion calls fail with them
         aws_secret_access_key: Final = optional_params.pop("aws_secret_access_key", None)
@@ -78,17 +89,21 @@ class BedrockEmbedding(BaseAWSLLM):
             if aws_region_name is None:
                 aws_region_name = "us-west-2"
 
-        credentials: Final[Credentials] = self.get_credentials(
-            aws_access_key_id=aws_access_key_id,
-            aws_secret_access_key=aws_secret_access_key,
-            aws_session_token=aws_session_token,
-            aws_region_name=aws_region_name,
-            aws_session_name=aws_session_name,
-            aws_profile_name=aws_profile_name,
-            aws_role_name=aws_role_name,
-            aws_web_identity_token=aws_web_identity_token,
-            aws_sts_endpoint=aws_sts_endpoint,
-            aws_external_id=aws_external_id,
+        credentials: Final[Credentials | None] = (
+            None
+            if bearer_token is not None
+            else self.get_credentials(
+                aws_access_key_id=aws_access_key_id,
+                aws_secret_access_key=aws_secret_access_key,
+                aws_session_token=aws_session_token,
+                aws_region_name=aws_region_name,
+                aws_session_name=aws_session_name,
+                aws_profile_name=aws_profile_name,
+                aws_role_name=aws_role_name,
+                aws_web_identity_token=aws_web_identity_token,
+                aws_sts_endpoint=aws_sts_endpoint,
+                aws_external_id=aws_external_id,
+            )
         )
         return credentials, aws_region_name
 
@@ -233,7 +248,7 @@ class BedrockEmbedding(BaseAWSLLM):
         client: HTTPHandler | None,
         timeout: float | httpx.Timeout | None,
         batch_data: list[dict],
-        credentials: Any,
+        credentials: Credentials | None,
         extra_headers: dict | None,
         endpoint_url: str,
         aws_region_name: str,
@@ -301,7 +316,7 @@ class BedrockEmbedding(BaseAWSLLM):
         client: AsyncHTTPHandler | None,
         timeout: float | httpx.Timeout | None,
         batch_data: list[dict],
-        credentials: Any,
+        credentials: Credentials | None,
         extra_headers: dict | None,
         endpoint_url: str,
         aws_region_name: str,
@@ -383,7 +398,9 @@ class BedrockEmbedding(BaseAWSLLM):
         litellm_params: dict,
         api_key: str | None = None,
     ) -> EmbeddingResponse:
-        credentials, aws_region_name = self._load_credentials(optional_params)
+        credentials, aws_region_name = self._load_credentials(
+            optional_params, bearer_token=bedrock_bearer_token(api_key)
+        )
 
         ### TRANSFORMATION ###
         unencoded_model_id: Final = optional_params.pop("model_id", None) or model  # default to model if not passed

litellm/llms/bedrock/image_edit/handler.py

@@ -29,7 +29,7 @@ from litellm.llms.custom_httpx.http_handler import (
 )
 from litellm.types.utils import ImageResponse
 
-from ..base_aws_llm import BaseAWSLLM
+from ..base_aws_llm import BaseAWSLLM, bedrock_bearer_token
 from ..common_utils import BedrockError
 
 if TYPE_CHECKING:
@@ -198,7 +198,9 @@ class BedrockImageEdit(BaseAWSLLM):
         Returns:
             BedrockImageEditPreparedRequest: The prepared request object
         """
-        boto3_credentials_info: Final = self._get_boto_credentials_from_optional_params(optional_params, model)
+        boto3_credentials_info: Final = self._get_boto_credentials_from_optional_params(
+            optional_params, model, bearer_token=bedrock_bearer_token(api_key)
+        )
 
         # Use the existing ARN-aware provider detection method
         bedrock_provider: Final = self.get_bedrock_invoke_provider(model)

litellm/llms/bedrock/image_generation/image_handler.py

@@ -29,7 +29,7 @@ from litellm.llms.custom_httpx.http_handler import (
 )
 from litellm.types.utils import ImageResponse
 
-from ..base_aws_llm import BaseAWSLLM
+from ..base_aws_llm import BaseAWSLLM, bedrock_bearer_token
 from ..common_utils import BedrockError
 
 if TYPE_CHECKING:
@@ -220,7 +220,9 @@ class BedrockImageGeneration(BaseAWSLLM):
             prepped (httpx.Request): The prepared request object
             body (bytes): The request body
         """
-        boto3_credentials_info: Final = self._get_boto_credentials_from_optional_params(optional_params, model)
+        boto3_credentials_info: Final = self._get_boto_credentials_from_optional_params(
+            optional_params, model, bearer_token=bedrock_bearer_token(api_key)
+        )
 
         # Use the existing ARN-aware provider detection method
         bedrock_provider: Final = self.get_bedrock_invoke_provider(model)

litellm/llms/custom_httpx/container_handler.py

@@ -139,7 +139,7 @@ def _build_query_params(
     return {name: value if isinstance(value, str) else str(value) for name, value in supplied if value is not None}
 
 
-def _error_message_from_response(response: httpx.Response) -> str:
+def error_message_from_response(response: httpx.Response) -> str:
     try:
         body: Final = response.json()
     except ValueError:
@@ -153,6 +153,16 @@ def _error_message_from_response(response: httpx.Response) -> str:
     return response.text
 
 
+def raise_for_error_status(response: httpx.Response, container_provider_config: "BaseContainerConfig") -> None:
+    if not httpx.codes.is_error(response.status_code):
+        return
+    raise container_provider_config.get_error_class(
+        error_message=error_message_from_response(response),
+        status_code=response.status_code,
+        headers=response.headers,
+    )
+
+
 def _transform_response(
     response: httpx.Response,
     returns_binary: bool,
@@ -163,7 +173,7 @@ def _transform_response(
     if httpx.codes.is_error(response.status_code):
         raise BaseLLMException(
             status_code=response.status_code,
-            message=_error_message_from_response(response),
+            message=error_message_from_response(response),
             headers=dict(response.headers),
         )
 

litellm/llms/custom_httpx/llm_http_handler.py

@@ -77,6 +77,7 @@ from litellm.llms.base_llm.vector_store_files.transformation import (
     BaseVectorStoreFilesConfig,
 )
 from litellm.llms.base_llm.videos.transformation import BaseVideoConfig
+from litellm.llms.custom_httpx.container_handler import raise_for_error_status
 from litellm.llms.custom_httpx.http_handler import (
     AsyncHTTPHandler,
     HTTPHandler,
@@ -99,9 +100,12 @@ from litellm.types.containers.main import (
 )
 from litellm.types.files import StreamingMediaUploadConfig, TwoStepFileUploadConfig
 from litellm.types.integrations.custom_logger import (
+    NON_CODE_INTERPRETER_INTERCEPTION_INTERNAL_PREFIXES,
     AgenticLoopPlan,
     AgenticLoopRequestPatch,
     AgenticLoopSafetyError,
+    converted_stream_requested,
+    is_interception_internal_key,
 )
 from litellm.types.llms.anthropic_messages.anthropic_response import (
     AnthropicMessagesResponse,
@@ -2760,6 +2764,7 @@ class BaseLLMHTTPHandler:
         )
 
         if self._has_agentic_completion_hook(logging_obj):
+            agentic_kwargs: Final = dict(litellm_params)  # mutable-ok: agentic hooks mutate kwargs in place
             final_response: Final = run_async_function(
                 self._call_agentic_completion_hooks,
                 response=initial_response,
@@ -2770,10 +2775,19 @@ class BaseLLMHTTPHandler:
                 logging_obj=logging_obj,
                 stream=False,
                 custom_llm_provider=custom_llm_provider,
-                kwargs=dict(litellm_params),
+                kwargs=agentic_kwargs,
                 api_surface="responses",
             )
-            return final_response if final_response is not None else initial_response
+            result: Final = final_response if final_response is not None else initial_response
+            if converted_stream_requested(agentic_kwargs) and not agentic_kwargs.get("_agentic_loop_depth"):
+                return self._wrap_responses_response_as_fake_stream(
+                    result=result,
+                    model=model,
+                    responses_api_provider_config=responses_api_provider_config,
+                    logging_obj=logging_obj,
+                    custom_llm_provider=custom_llm_provider,
+                )
+            return result
 
         return initial_response
 
@@ -2939,6 +2953,7 @@ class BaseLLMHTTPHandler:
             logging_obj=logging_obj,
         )
 
+        agentic_kwargs: Final = dict(litellm_params)  # mutable-ok: agentic hooks mutate kwargs in place
         final_response: Final = await self._call_agentic_completion_hooks(
             response=initial_response,
             model=model,
@@ -2948,15 +2963,12 @@ class BaseLLMHTTPHandler:
             logging_obj=logging_obj,
             stream=False,
             custom_llm_provider=custom_llm_provider,
-            kwargs=dict(litellm_params),
+            kwargs=agentic_kwargs,
             api_surface="responses",
         )
 
         result: Final = final_response if final_response is not None else initial_response
-        interception_converted_stream: Final = litellm_params.get(
-            "_code_interpreter_interception_converted_stream"
-        ) or litellm_params.get("_websearch_interception_converted_stream")
-        if interception_converted_stream and not litellm_params.get("_agentic_loop_depth"):
+        if converted_stream_requested(agentic_kwargs) and not agentic_kwargs.get("_agentic_loop_depth"):
             return self._wrap_responses_response_as_fake_stream(
                 result=result,
                 model=model,
@@ -5420,8 +5432,7 @@ class BaseLLMHTTPHandler:
         kwargs_for_followup: Final = {
             k: v
             for k, v in kwargs.items()
-            if not k.startswith("_websearch_interception")
-            and not k.startswith("_compression_interception")
+            if not is_interception_internal_key(k, prefixes=NON_CODE_INTERPRETER_INTERCEPTION_INTERNAL_PREFIXES)
             and k != "_code_interpreter_interception_converted_stream"
             and k not in internal_keys
             and k not in optional_params
@@ -8753,17 +8764,19 @@ class BaseLLMHTTPHandler:
                 json=data,
                 timeout=timeout,
             )
-
-            return container_provider_config.transform_container_create_response(
-                raw_response=response,
-                logging_obj=logging_obj,
-            )
-
         except Exception as e:
             raise self._handle_error(
                 e=e,
                 provider_config=container_provider_config,
             )
+        raise_for_error_status(
+            response=response,
+            container_provider_config=container_provider_config,
+        )
+        return container_provider_config.transform_container_create_response(
+            raw_response=response,
+            logging_obj=logging_obj,
+        )
 
     async def async_container_create_handler(
         self,
@@ -8829,17 +8842,19 @@ class BaseLLMHTTPHandler:
                 json=data,
                 timeout=timeout,
             )
-
-            return container_provider_config.transform_container_create_response(
-                raw_response=response,
-                logging_obj=logging_obj,
-            )
-
         except Exception as e:
             raise self._handle_error(
                 e=e,
                 provider_config=container_provider_config,
             )
+        raise_for_error_status(
+            response=response,
+            container_provider_config=container_provider_config,
+        )
+        return container_provider_config.transform_container_create_response(
+            raw_response=response,
+            logging_obj=logging_obj,
+        )
 
     def container_list_handler(
         self,
@@ -8919,17 +8934,19 @@ class BaseLLMHTTPHandler:
                 headers=headers,
                 params=params or None,
             )
-
-            return container_provider_config.transform_container_list_response(
-                raw_response=response,
-                logging_obj=logging_obj,
-            )
-
         except Exception as e:
             raise self._handle_error(
                 e=e,
                 provider_config=container_provider_config,
             )
+        raise_for_error_status(
+            response=response,
+            container_provider_config=container_provider_config,
+        )
+        return container_provider_config.transform_container_list_response(
+            raw_response=response,
+            logging_obj=logging_obj,
+        )
 
     async def async_container_list_handler(
         self,
@@ -8996,17 +9013,19 @@ class BaseLLMHTTPHandler:
                 headers=headers,
                 params=params or None,
             )
-
-            return container_provider_config.transform_container_list_response(
-                raw_response=response,
-                logging_obj=logging_obj,
-            )
-
         except Exception as e:
             raise self._handle_error(
                 e=e,
                 provider_config=container_provider_config,
             )
+        raise_for_error_status(
+            response=response,
+            container_provider_config=container_provider_config,
+        )
+        return container_provider_config.transform_container_list_response(
+            raw_response=response,
+            logging_obj=logging_obj,
+        )
 
     def container_retrieve_handler(
         self,
@@ -9084,17 +9103,19 @@ class BaseLLMHTTPHandler:
                 headers=headers,
                 params=params or None,
             )
-
-            return container_provider_config.transform_container_retrieve_response(
-                raw_response=response,
-                logging_obj=logging_obj,
-            )
-
         except Exception as e:
             raise self._handle_error(
                 e=e,
                 provider_config=container_provider_config,
             )
+        raise_for_error_status(
+            response=response,
+            container_provider_config=container_provider_config,
+        )
+        return container_provider_config.transform_container_retrieve_response(
+            raw_response=response,
+            logging_obj=logging_obj,
+        )
 
     async def async_container_retrieve_handler(
         self,
@@ -9161,17 +9182,19 @@ class BaseLLMHTTPHandler:
                 headers=headers,
                 params=params or None,
             )
-
-            return container_provider_config.transform_container_retrieve_response(
-                raw_response=response,
-                logging_obj=logging_obj,
-            )
-
         except Exception as e:
             raise self._handle_error(
                 e=e,
                 provider_config=container_provider_config,
             )
+        raise_for_error_status(
+            response=response,
+            container_provider_config=container_provider_config,
+        )
+        return container_provider_config.transform_container_retrieve_response(
+            raw_response=response,
+            logging_obj=logging_obj,
+        )
 
     def container_delete_handler(
         self,
@@ -9249,17 +9272,19 @@ class BaseLLMHTTPHandler:
                 headers=headers,
                 params=params or None,
             )
-
-            return container_provider_config.transform_container_delete_response(
-                raw_response=response,
-                logging_obj=logging_obj,
-            )
-
         except Exception as e:
             raise self._handle_error(
                 e=e,
                 provider_config=container_provider_config,
             )
+        raise_for_error_status(
+            response=response,
+            container_provider_config=container_provider_config,
+        )
+        return container_provider_config.transform_container_delete_response(
+            raw_response=response,
+            logging_obj=logging_obj,
+        )
 
     async def async_container_delete_handler(
         self,
@@ -9326,17 +9351,19 @@ class BaseLLMHTTPHandler:
                 headers=headers,
                 params=params or None,
             )
-
-            return container_provider_config.transform_container_delete_response(
-                raw_response=response,
-                logging_obj=logging_obj,
-            )
-
         except Exception as e:
             raise self._handle_error(
                 e=e,
                 provider_config=container_provider_config,
             )
+        raise_for_error_status(
+            response=response,
+            container_provider_config=container_provider_config,
+        )
+        return container_provider_config.transform_container_delete_response(
+            raw_response=response,
+            logging_obj=logging_obj,
+        )
 
     def container_file_list_handler(
         self,
@@ -9418,17 +9445,19 @@ class BaseLLMHTTPHandler:
                 headers=headers,
                 params=params or None,
             )
-
-            return container_provider_config.transform_container_file_list_response(
-                raw_response=response,
-                logging_obj=logging_obj,
-            )
-
         except Exception as e:
             raise self._handle_error(
                 e=e,
                 provider_config=container_provider_config,
             )
+        raise_for_error_status(
+            response=response,
+            container_provider_config=container_provider_config,
+        )
+        return container_provider_config.transform_container_file_list_response(
+            raw_response=response,
+            logging_obj=logging_obj,
+        )
 
     async def async_container_file_list_handler(
         self,
@@ -9497,17 +9526,19 @@ class BaseLLMHTTPHandler:
                 headers=headers,
                 params=params or None,
             )
-
-            return container_provider_config.transform_container_file_list_response(
-                raw_response=response,
-                logging_obj=logging_obj,
-            )
-
         except Exception as e:
             raise self._handle_error(
                 e=e,
                 provider_config=container_provider_config,
             )
+        raise_for_error_status(
+            response=response,
+            container_provider_config=container_provider_config,
+        )
+        return container_provider_config.transform_container_file_list_response(
+            raw_response=response,
+            logging_obj=logging_obj,
+        )
 
     def container_file_content_handler(
         self,
@@ -9583,17 +9614,19 @@ class BaseLLMHTTPHandler:
                 headers=headers,
                 params=params or None,
             )
-
-            return container_provider_config.transform_container_file_content_response(
-                raw_response=response,
-                logging_obj=logging_obj,
-            )
-
         except Exception as e:
             raise self._handle_error(
                 e=e,
                 provider_config=container_provider_config,
             )
+        raise_for_error_status(
+            response=response,
+            container_provider_config=container_provider_config,
+        )
+        return container_provider_config.transform_container_file_content_response(
+            raw_response=response,
+            logging_obj=logging_obj,
+        )
 
     async def async_container_file_content_handler(
         self,
@@ -9659,17 +9692,19 @@ class BaseLLMHTTPHandler:
                 headers=headers,
                 params=params or None,
             )
-
-            return container_provider_config.transform_container_file_content_response(
-                raw_response=response,
-                logging_obj=logging_obj,
-            )
-
         except Exception as e:
             raise self._handle_error(
                 e=e,
                 provider_config=container_provider_config,
             )
+        raise_for_error_status(
+            response=response,
+            container_provider_config=container_provider_config,
+        )
+        return container_provider_config.transform_container_file_content_response(
+            raw_response=response,
+            logging_obj=logging_obj,
+        )
 
     ###### VECTOR STORE HANDLER ######
     @staticmethod

litellm/llms/dashscope/cost_calculator.py

@@ -8,10 +8,13 @@ See https://help.aliyun.com/zh/model-studio/billing-for-model-studio
 """
 
 from dataclasses import dataclass
+from datetime import datetime
 from typing import Final
 
 from litellm.litellm_core_utils.llm_cost_calc.tiered_pricing import select_tier_for_input, tier_rate
 from litellm.litellm_core_utils.llm_cost_calc.utils import (
+    TokenRates,
+    apply_off_peak_pricing,
     parse_completion_tokens_details,
     parse_prompt_tokens_details,
 )
@@ -57,69 +60,68 @@ def _flat_rate(model_info: ModelInfo, cost_key: str, fallback_cost_key: str) ->
     return float(value)
 
 
-def _calculate_prompt_cost(
-    breakdown: TokenBreakdown,
-    model_info: ModelInfo,
-    tier: dict | None,
-) -> float:
-    if tier is not None:
-        return (
-            (breakdown.text_tokens * tier_rate(tier, "input_cost_per_token"))
-            + (breakdown.cached_tokens * tier_rate(tier, "cache_read_input_token_cost", "input_cost_per_token"))
-            + (
-                breakdown.cache_creation_tokens
-                * tier_rate(tier, "cache_creation_input_token_cost", "input_cost_per_token")
-            )
-        )
-
-    input_cost: Final = float(model_info.get("input_cost_per_token") or 0.0)
-    cache_read_cost: Final = _flat_rate(model_info, "cache_read_input_token_cost", "input_cost_per_token")
-    cache_creation_cost: Final = _flat_rate(model_info, "cache_creation_input_token_cost", "input_cost_per_token")
-
-    return (
-        (breakdown.text_tokens * input_cost)
-        + (breakdown.cached_tokens * cache_read_cost)
-        + (breakdown.cache_creation_tokens * cache_creation_cost)
+def _flat_rates(model_info: ModelInfo) -> TokenRates:
+    reasoning_rate: Final = model_info.get("output_cost_per_reasoning_token")
+    return TokenRates(
+        input_rate=float(model_info.get("input_cost_per_token") or 0.0),
+        cache_read_rate=_flat_rate(model_info, "cache_read_input_token_cost", "input_cost_per_token"),
+        cache_creation_rate=_flat_rate(model_info, "cache_creation_input_token_cost", "input_cost_per_token"),
+        output_rate=float(model_info.get("output_cost_per_token") or 0.0),
+        reasoning_rate=None if reasoning_rate is None else float(reasoning_rate),
     )
 
 
-def _calculate_completion_cost(
-    breakdown: TokenBreakdown,
-    model_info: ModelInfo,
-    tier: dict | None,
-) -> float:
+def _tier_rates(model_info: ModelInfo, tier: dict) -> TokenRates:
     # A tier that declares output rates keeps the request on them, all-or-nothing. A tier table
     # spelling out only input rates would serve every completion for free, so there the model's
     # own output rates stand in
-    tier_declares_output: Final = tier is not None and "output_cost_per_token" in tier
-    output_cost: Final = (
-        tier_rate(tier, "output_cost_per_token")
-        if tier_declares_output
-        else float(model_info.get("output_cost_per_token") or 0.0)
-    )
-    tier_declares_reasoning: Final = tier is not None and "output_cost_per_reasoning_token" in tier
-    model_reasoning_rate: Final = None if tier_declares_output else model_info.get("output_cost_per_reasoning_token")
-    reasoning_cost: Final = (
-        tier_rate(tier, "output_cost_per_reasoning_token", "output_cost_per_token")
-        if tier_declares_reasoning
-        else float(model_reasoning_rate)
-        if model_reasoning_rate is not None
-        else output_cost
+    flat_rates: Final = _flat_rates(model_info)
+    tier_declares_output: Final = "output_cost_per_token" in tier
+    tier_declares_reasoning: Final = "output_cost_per_reasoning_token" in tier
+    return TokenRates(
+        input_rate=tier_rate(tier, "input_cost_per_token"),
+        cache_read_rate=tier_rate(tier, "cache_read_input_token_cost", "input_cost_per_token"),
+        cache_creation_rate=tier_rate(tier, "cache_creation_input_token_cost", "input_cost_per_token"),
+        output_rate=tier_rate(tier, "output_cost_per_token") if tier_declares_output else flat_rates.output_rate,
+        reasoning_rate=(
+            tier_rate(tier, "output_cost_per_reasoning_token")
+            if tier_declares_reasoning
+            else None
+            if tier_declares_output
+            else flat_rates.reasoning_rate
+        ),
     )
 
-    return (breakdown.completion_tokens * output_cost) + (breakdown.reasoning_tokens * reasoning_cost)
+
+def _bill(breakdown: TokenBreakdown, rates: TokenRates) -> tuple[float, float]:
+    prompt_cost: Final = (
+        (breakdown.text_tokens * rates.input_rate)
+        + (breakdown.cached_tokens * rates.cache_read_rate)
+        + (breakdown.cache_creation_tokens * rates.cache_creation_rate)
+    )
+    completion_cost: Final = (breakdown.completion_tokens * rates.output_rate) + (
+        breakdown.reasoning_tokens * rates.billed_reasoning_rate
+    )
+    return prompt_cost, completion_cost
 
 
-def cost_per_token(model: str, usage: Usage, custom_llm_provider: str = "dashscope") -> tuple[float, float]:
+def cost_per_token(
+    model: str,
+    usage: Usage,
+    custom_llm_provider: str = "dashscope",
+    current_time: datetime | None = None,
+) -> tuple[float, float]:
     """
     Calculate cost per token for Dashscope models.
 
-    Supports both tiered and flat pricing with cached and reasoning tokens.
+    Supports both tiered and flat pricing with cached and reasoning tokens, and swaps in the
+    model's off_peak_pricing rates while one of its windows is open.
 
     Args:
         model: Model name without provider prefix
         usage: LiteLLM Usage block
         custom_llm_provider: The provider id the request resolved to; dashscope or one of its brand aliases
+        current_time: The moment the request is billed at; defaults to now, UTC
 
     Returns:
         Tuple[float, float] - (prompt_cost_in_usd, completion_cost_in_usd)
@@ -133,8 +135,7 @@ def cost_per_token(model: str, usage: Usage, custom_llm_provider: str = "dashsco
         if tiered_pricing
         else None
     )
+    standard_rates: Final = _flat_rates(model_info) if tier is None else _tier_rates(model_info, tier)
+    rates: Final = apply_off_peak_pricing(model_info, current_time, standard_rates)
 
-    prompt_cost: Final = _calculate_prompt_cost(breakdown=breakdown, model_info=model_info, tier=tier)
-    completion_cost: Final = _calculate_completion_cost(breakdown=breakdown, model_info=model_info, tier=tier)
-
-    return prompt_cost, completion_cost
+    return _bill(breakdown, rates)

litellm/llms/openai/chat/gpt_5_transformation.py

@@ -61,6 +61,14 @@ def _get_effort_level(value: str | dict | None) -> str | None:
     return None
 
 
+GPT_REASONING_SERIES_MARKERS: Final = ("gpt-5", "gpt-6")
+
+
+def is_gpt_reasoning_series_name(model: str) -> bool:
+    normalized: Final = model.split("/")[-1]
+    return any(marker in model for marker in GPT_REASONING_SERIES_MARKERS) and not normalized.startswith("gpt-5-chat")
+
+
 class OpenAIGPT5Config(OpenAIGPTConfig):
     """Configuration for gpt-5 models including GPT-5-Codex variants.
 
@@ -73,21 +81,7 @@ class OpenAIGPT5Config(OpenAIGPTConfig):
 
     @classmethod
     def is_model_gpt_5_model(cls, model: str) -> bool:
-        # The gpt-5-chat* family (gpt-5-chat, gpt-5-chat-latest, gpt-5-chat-2025-08-07,
-        # …) are regular chat models: they support temperature and tool_choice but NOT
-        # reasoning_effort.  They must NOT be routed through the GPT-5 reasoning path.
-        #
-        # Versioned chat models such as gpt-5.3-chat and gpt-5.1-chat ARE reasoning
-        # models and must stay on the GPT-5 path.  The distinguishing feature is that
-        # the gpt-5-chat family has a literal "-chat" immediately after "gpt-5"
-        # (i.e. "gpt-5-chat…"), while versioned chat models interpose a minor version
-        # number (i.e. "gpt-5.<digit>-chat").
-        #
-        # Using a startswith("gpt-5-chat") prefix check on the normalized name (rather
-        # than a substring check) makes this boundary explicit and avoids any ambiguity
-        # if future model names coincidentally contain "gpt-5-chat" as an interior run.
-        _normalized: Final = model.split("/")[-1]  # strip provider prefix, e.g. "openai/"
-        return "gpt-5" in model and not _normalized.startswith("gpt-5-chat")
+        return is_gpt_reasoning_series_name(model)
 
     @classmethod
     def is_model_gpt_5_search_model(cls, model: str) -> bool:
@@ -122,6 +116,8 @@ class OpenAIGPT5Config(OpenAIGPTConfig):
     def is_model_gpt_5_4_plus_model(cls, model: str) -> bool:
         """Check if the model is gpt-5.4 or newer (5.4, 5.5, 5.6, etc., including pro)."""
         model_name: Final = model.split("/")[-1]
+        if model_name.startswith("gpt-6"):
+            return True
         if not model_name.startswith("gpt-5."):
             return False
         try:

litellm/llms/openai/common_utils.py

@@ -11,6 +11,7 @@ import time
 import uuid
 from collections.abc import AsyncIterator, Iterator, Mapping
 from typing import TYPE_CHECKING, Any, Final, Literal, NamedTuple, Optional
+from urllib.parse import urlsplit
 
 import httpx
 import openai
@@ -43,6 +44,14 @@ _OPENAI_INIT_PARAMS: Final[tuple[str, ...]] = _get_client_init_params(OpenAI)
 _AZURE_OPENAI_INIT_PARAMS: Final[tuple[str, ...]] = _get_client_init_params(AzureOpenAI)
 
 
+_OPENAI_API_HOST: Final[str] = "api.openai.com"
+
+
+def is_openai_backed_api_base(api_base: str) -> bool:
+    hostname: Final = urlsplit(api_base).hostname
+    return hostname is not None and (hostname == _OPENAI_API_HOST or hostname.endswith(f".{_OPENAI_API_HOST}"))
+
+
 class OpenAIError(BaseLLMException):
     def __init__(
         self,

litellm/llms/openai/openai.py

@@ -2,7 +2,6 @@ import time
 import types
 from collections.abc import AsyncIterator, Callable, Coroutine, Iterable, Iterator, Mapping
 from typing import TYPE_CHECKING, Any, Final, Literal, Optional, cast
-from urllib.parse import urlparse
 
 import httpx
 
@@ -55,6 +54,7 @@ from .common_utils import (
     OpenAIError,
     build_output_token_limit_response,
     drop_params_from_unprocessable_entity_error,
+    is_openai_backed_api_base,
     is_output_token_limit_error,
 )
 from .workload_identity import resolve_openai_workload_identity_config
@@ -1190,10 +1190,8 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM):
         """
         if stream_options is not None:
             return {"stream_options": stream_options}
-        else:
-            # by default litellm will include usage for openai endpoints
-            if api_base is None or urlparse(api_base).hostname == "api.openai.com":
-                return {"stream_options": {"include_usage": True}}
+        if api_base is None or is_openai_backed_api_base(api_base):
+            return {"stream_options": {"include_usage": True}}
         return {}
 
     # Embedding

litellm/llms/openai/responses/guardrail_translation/handler.py

@@ -33,8 +33,9 @@ import time
 import uuid
 from collections.abc import Mapping, Sequence
 from dataclasses import dataclass
+from itertools import accumulate
 from types import MappingProxyType
-from typing import TYPE_CHECKING, Any, Final, Union, cast
+from typing import TYPE_CHECKING, Any, Final, NamedTuple, Union, cast
 
 from openai.types.responses.response_function_tool_call import ResponseFunctionToolCall
 from pydantic import BaseModel, TypeAdapter
@@ -42,6 +43,7 @@ from typing_extensions import ReadOnly, TypedDict
 
 from litellm._logging import verbose_proxy_logger
 from litellm.completion_extras.litellm_responses_transformation.transformation import (
+    LiteLLMResponsesTransformationHandler,
     OpenAiResponsesToChatCompletionStreamIterator,
 )
 from litellm.llms.base_llm.guardrail_translation.base_translation import (
@@ -74,6 +76,7 @@ from litellm.types.llms.openai import (
     OutputTextDoneEvent,
     ResponseAPIUsage,
     ResponseCompletedEvent,
+    ResponsesAPIOptionalRequestParams,
     ResponsesAPIResponse,
     ResponsesAPIStreamEvents,
     ResponsesAPIStreamingResponse,
@@ -115,6 +118,199 @@ class ResponsesStreamChunk(TypedDict, total=False):
     content_index: ReadOnly[int]
 
 
+_PATCHABLE_ITEM_FIELDS: Final[Mapping[str, str]] = MappingProxyType(
+    {"function_call_output": "output", "message": "content"}
+)
+
+_EMPTY_RESPONSES_REQUEST: Final[ResponsesAPIOptionalRequestParams] = {}
+
+
+def _item_rewrite_field(item: Mapping[str, object]) -> str | None:
+    item_type: Final = item.get("type")
+    if item_type is None:
+        return "content" if "content" in item else None
+    if not isinstance(item_type, str):
+        return None
+    return _PATCHABLE_ITEM_FIELDS.get(item_type)
+
+
+def _rewritten_input_item(item: Mapping[str, object], rewritten: object) -> Mapping[str, object] | None:
+    field: Final = _item_rewrite_field(item)
+    if field is None or not isinstance(rewritten, Mapping):
+        return None
+    rewritten_content: Final = rewritten.get("content")
+    if isinstance(item.get(field), str) and isinstance(rewritten_content, str):
+        return {**item, field: rewritten_content}  # mutable-ok: request input items must stay JSON-plain dicts
+    rewritten_row: Final = cast("AllMessageValues", rewritten)  # cast-ok: guardrails hand back chat-shaped rows
+    converted_items, _ = LiteLLMResponsesTransformationHandler().convert_chat_completion_messages_to_responses_api(
+        [rewritten_row]  # mutable-ok: converter signature takes a list
+    )
+    if len(converted_items) != 1 or not isinstance(converted_items[0], Mapping):
+        return None
+    first_converted: Final = cast("Mapping[str, object]", converted_items[0])  # cast-ok: isinstance-checked above
+    converted_value: Final = first_converted.get(field)
+    if converted_value is None:
+        return None
+    return {**item, field: converted_value}  # mutable-ok: request input items must stay JSON-plain dicts
+
+
+def _is_function_call_item(item: object) -> bool:
+    return isinstance(item, Mapping) and item.get("type") in ("function_call", "custom_tool_call")
+
+
+def _last_message_role(messages: Sequence[object]) -> str | None:
+    if not messages:
+        return None
+    last: Final = messages[-1]
+    role: Final = last.get("role") if isinstance(last, Mapping) else getattr(last, "role", None)
+    return role if isinstance(role, str) else None
+
+
+def _provenance_unit_bounds(
+    raw_input: Sequence[object],
+    solo_conversions: Sequence[Sequence[object]],
+) -> tuple[tuple[int, int], ...]:
+    trailing_roles: Final = tuple(
+        accumulate(
+            (_last_message_role(messages) for messages in solo_conversions),
+            lambda previous, current: current if current is not None else previous,
+        )
+    )
+    start_indexes: Final = tuple(
+        index
+        for index in range(len(raw_input))
+        if index == 0 or not (_is_function_call_item(raw_input[index]) and trailing_roles[index - 1] == "assistant")
+    )
+    return tuple(zip(start_indexes, (*start_indexes[1:], len(raw_input))))
+
+
+def _input_item_provenance(
+    raw_input: Sequence[object],
+    expected_messages: Sequence[object],
+) -> tuple[Mapping[int, int], frozenset[int]] | None:
+    if not all(isinstance(item, Mapping) for item in raw_input):
+        return None
+    solo_conversions: Final = tuple(
+        LiteLLMCompletionResponsesConfig.transform_responses_api_input_to_messages(
+            input=cast("ResponseInputParam", [item]),  # cast-ok: items checked as Mappings above
+            responses_api_request=_EMPTY_RESPONSES_REQUEST,
+        )
+        for item in raw_input
+    )
+    full_conversion: Final = tuple(
+        LiteLLMCompletionResponsesConfig.transform_responses_api_input_to_messages(
+            input=cast("ResponseInputParam", list(raw_input)),  # cast-ok: items checked as Mappings above
+            responses_api_request=_EMPTY_RESPONSES_REQUEST,
+        )
+    )
+    if full_conversion != tuple(expected_messages):
+        return None
+    units: Final = _provenance_unit_bounds(raw_input, solo_conversions)
+    unit_messages: Final = tuple(
+        tuple(solo_conversions[start])
+        if end - start == 1
+        else tuple(
+            LiteLLMCompletionResponsesConfig.transform_responses_api_input_to_messages(
+                input=cast("ResponseInputParam", list(raw_input[start:end])),  # cast-ok: checked as Mappings above
+                responses_api_request=_EMPTY_RESPONSES_REQUEST,
+            )
+        )
+        for start, end in units
+    )
+    if tuple(message for messages in unit_messages for message in messages) != full_conversion:
+        return None
+    boundaries: Final = tuple(accumulate((len(messages) for messages in unit_messages), initial=0))
+    item_for_message: Final = MappingProxyType(
+        {
+            message_index: start
+            for unit_index, (start, end) in enumerate(units)
+            if end - start == 1
+            for message_index in range(boundaries[unit_index], boundaries[unit_index + 1])
+        }
+    )
+    tainted: Final = frozenset(
+        message_index
+        for unit_index, (start, end) in enumerate(units)
+        if end - start > 1
+        for message_index in range(boundaries[unit_index], boundaries[unit_index + 1])
+    )
+    return item_for_message, tainted
+
+
+class _RequestFields(NamedTuple):
+    input: tuple[object, ...]
+    instructions: str | None
+
+
+class _ExtractedInputs(NamedTuple):
+    inputs: GenericGuardrailAPIInputs
+    task_mappings: tuple[tuple[int, int | None], ...]
+
+
+def _patched_request_fields(
+    raw_input: object,
+    instructions: object,
+    original_messages: Sequence[object],
+    structured_messages: Sequence[object],
+) -> _RequestFields | None:
+    if not isinstance(raw_input, list) or len(original_messages) != len(structured_messages):
+        return None
+    offset: Final = 1 if instructions else 0
+    provenance: Final = _input_item_provenance(raw_input, tuple(original_messages)[offset:])
+    if provenance is None:
+        return None
+    item_for_message, tainted = provenance
+    changed: Final = tuple(
+        (index, rewritten)
+        for index, (original, rewritten) in enumerate(zip(original_messages, structured_messages))
+        if original != rewritten
+    )
+    instruction_rewrites: Final = tuple(rewritten for index, rewritten in changed if index < offset)
+    rewritten_instructions: Final = (
+        instruction_rewrites[0].get("content")
+        if instruction_rewrites and isinstance(instruction_rewrites[0], Mapping)
+        else instructions
+    )
+    instructions_value: Final = rewritten_instructions if isinstance(rewritten_instructions, str) else None
+    if rewritten_instructions is not None and instructions_value is None:
+        return None
+    body_changes: Final = tuple((index - offset, rewritten) for index, rewritten in changed if index >= offset)
+    if any(message_index in tainted or message_index not in item_for_message for message_index, _ in body_changes):
+        return None
+    replacements: Final = MappingProxyType(
+        {
+            item_for_message[message_index]: _rewritten_input_item(
+                cast("Mapping[str, object]", raw_input[item_for_message[message_index]]),  # cast-ok: checked Mappings
+                rewritten,
+            )
+            for message_index, rewritten in body_changes
+        }
+    )
+    if len(replacements) != len(body_changes) or any(item is None for item in replacements.values()):
+        return None
+    return _RequestFields(
+        input=tuple(replacements.get(index, item) for index, item in enumerate(raw_input)),
+        instructions=instructions_value,
+    )
+
+
+def _patch_or_convert_request_fields(
+    raw_input: object,
+    instructions: object,
+    original_messages: Sequence[object],
+    structured_messages: Sequence[AllMessageValues],
+) -> _RequestFields | None:
+    if not isinstance(structured_messages, list):
+        return None
+    patched: Final = _patched_request_fields(raw_input, instructions, original_messages, structured_messages)
+    if patched is not None:
+        return patched
+    input_items, converted_instructions = (
+        LiteLLMResponsesTransformationHandler().convert_chat_completion_messages_to_responses_api(structured_messages)
+    )
+    return _RequestFields(input=tuple(input_items), instructions=converted_instructions)
+
+
 def _next_stream_sequence_number(responses_so_far: Sequence[Any] | None) -> int:
     sequence_numbers: Final = (
         item.get("sequence_number") if isinstance(item, dict) else getattr(item, "sequence_number", None)
@@ -162,9 +358,8 @@ class OpenAIResponsesHandler(BaseTranslation):
         Handles both string input and list of message objects.
         """
         input_data: Final[str | ResponseInputParam | None] = data.get("input")
-        if input_data is None:
+        if not isinstance(input_data, (str, list)):
             return data
-
         structured_messages: Final = self.get_structured_messages(data)
         raw_tools: Final = data.get("tools")
         original_tools: Final[tuple[Mapping[str, object], ...]] = (
@@ -173,94 +368,93 @@ class OpenAIResponsesHandler(BaseTranslation):
         flattened_tool_groups: Final = tuple(
             form.chat_tools for form in LiteLLMCompletionResponsesConfig.responses_tools_to_chat_forms(original_tools)
         )
-        flattened_tools: Final = tuple(
-            cast(ChatCompletionToolParam, tool)  # cast-ok: mcp tools ride along in the guardrail's tool list
-            for group in flattened_tool_groups
-            for tool in group
-        )
-        tools_to_check: Final[list[ChatCompletionToolParam]] = list(  # mutable-ok: guardrail inputs want a list
-            copy.deepcopy(flattened_tools)
-        )
-
-        # Handle simple string input
-        if isinstance(input_data, str):
-            inputs = GenericGuardrailAPIInputs(texts=[input_data])
-            if tools_to_check:
-                inputs["tools"] = tools_to_check
-            if structured_messages:
-                inputs["structured_messages"] = structured_messages
-            # Include model information if available
-            model = data.get("model")
-            if model:
-                inputs["model"] = model
-
-            guardrailed_inputs = await guardrail_to_apply.apply_guardrail(
-                inputs=inputs,
-                request_data=data,
-                input_type="request",
-                logging_obj=litellm_logging_obj,
-            )
-            guardrailed_texts = guardrailed_inputs.get("texts", [])
-            data["input"] = guardrailed_texts[0] if guardrailed_texts else input_data
-            self._apply_guardrailed_tools_to_data(
-                data, original_tools, flattened_tool_groups, guardrailed_inputs.get("tools")
-            )
-            verbose_proxy_logger.debug("OpenAI Responses API: Processed string input")
-            return data
-
-        # Handle list input (ResponseInputParam)
-        if not isinstance(input_data, list):
+        extracted: Final = self._extract_guardrail_inputs(data, input_data, flattened_tool_groups)
+        if not extracted.inputs.get("texts"):
             return data
+        if structured_messages:
+            extracted.inputs["structured_messages"] = structured_messages
+        guardrailed_inputs: Final = await guardrail_to_apply.apply_guardrail(
+            inputs=extracted.inputs,
+            request_data=data,
+            input_type="request",
+            logging_obj=litellm_logging_obj,
+        )
+        self._apply_guardrailed_tools_to_data(
+            data, original_tools, flattened_tool_groups, guardrailed_inputs.get("tools")
+        )
+        written_back: Final = self._written_back_request_fields(data, structured_messages, guardrailed_inputs)
+        if written_back is not None:
+            data["input"] = list(written_back.input)  # mutable-ok: JSON body
+            if written_back.instructions is None:
+                data.pop("instructions", None)
+            else:
+                data["instructions"] = written_back.instructions  # rebind-ok: data is an out-param
+        elif isinstance(input_data, str):
+            guardrailed_texts: Final = guardrailed_inputs.get("texts") or ()
+            data["input"] = guardrailed_texts[0] if guardrailed_texts else input_data  # rebind-ok: data is an out-param
+        else:
+            await self._apply_guardrail_responses_to_input(
+                messages=input_data,
+                responses=guardrailed_inputs.get("texts") or (),
+                task_mappings=extracted.task_mappings,
+            )
+        verbose_proxy_logger.debug("OpenAI Responses API: Processed input messages: %s", data.get("input"))
+        return data
 
+    def _extract_guardrail_inputs(
+        self,
+        data: Mapping[str, object],
+        input_data: "str | ResponseInputParam",
+        flattened_tool_groups: Sequence[Sequence[Mapping[str, object]]],
+    ) -> _ExtractedInputs:
         texts_to_check: Final[list[str]] = []
         images_to_check: Final[list[str]] = []
         task_mappings: Final[list[tuple[int, int | None]]] = []
-
-        # Step 1: Extract all text content, images, and tools
-        for msg_idx, message in enumerate(input_data):
-            self._extract_input_text_and_images(
-                message=message,
-                msg_idx=msg_idx,
-                texts_to_check=texts_to_check,
-                images_to_check=images_to_check,
-                task_mappings=task_mappings,
+        tools_to_check: Final[list[ChatCompletionToolParam]] = list(  # mutable-ok: guardrail inputs want a list
+            copy.deepcopy(
+                tuple(
+                    cast(ChatCompletionToolParam, tool)  # cast-ok: mcp tools ride along in the guardrail's tool list
+                    for group in flattened_tool_groups
+                    for tool in group
+                )
             )
+        )
+        if isinstance(input_data, str):
+            texts_to_check.append(input_data)
+        else:
+            for msg_idx, message in enumerate(input_data):
+                self._extract_input_text_and_images(
+                    message=message,
+                    msg_idx=msg_idx,
+                    texts_to_check=texts_to_check,
+                    images_to_check=images_to_check,
+                    task_mappings=task_mappings,
+                )
+        inputs: Final = GenericGuardrailAPIInputs(texts=texts_to_check)
+        if images_to_check:
+            inputs["images"] = images_to_check
+        if tools_to_check:
+            inputs["tools"] = tools_to_check
+        model: Final = data.get("model")
+        if isinstance(model, str):
+            inputs["model"] = model
+        return _ExtractedInputs(inputs=inputs, task_mappings=tuple(task_mappings))
 
-        # Step 2: Apply guardrail to all texts in batch
-        if texts_to_check:
-            inputs = GenericGuardrailAPIInputs(texts=texts_to_check)
-            if images_to_check:
-                inputs["images"] = images_to_check
-            if tools_to_check:
-                inputs["tools"] = tools_to_check
-            if structured_messages:
-                inputs["structured_messages"] = structured_messages
-            # Include model information if available
-            model = data.get("model")
-            if model:
-                inputs["model"] = model
-            guardrailed_inputs = await guardrail_to_apply.apply_guardrail(
-                inputs=inputs,
-                request_data=data,
-                input_type="request",
-                logging_obj=litellm_logging_obj,
-            )
-
-            guardrailed_texts = guardrailed_inputs.get("texts", [])
-            self._apply_guardrailed_tools_to_data(
-                data, original_tools, flattened_tool_groups, guardrailed_inputs.get("tools")
-            )
-
-            # Step 3: Map guardrail responses back to original input structure
-            await self._apply_guardrail_responses_to_input(
-                messages=input_data,
-                responses=guardrailed_texts,
-                task_mappings=task_mappings,
-            )
-
-        verbose_proxy_logger.debug("OpenAI Responses API: Processed input messages: %s", input_data)
-
-        return data
+    @staticmethod
+    def _written_back_request_fields(
+        data: Mapping[str, object],
+        structured_messages: Sequence[AllMessageValues] | None,
+        guardrailed_inputs: GenericGuardrailAPIInputs,
+    ) -> _RequestFields | None:
+        guardrailed: Final = guardrailed_inputs.get("structured_messages")
+        if guardrailed is None or guardrailed is structured_messages:
+            return None
+        return _patch_or_convert_request_fields(
+            data.get("input"),
+            data.get("instructions"),
+            structured_messages or (),
+            guardrailed,
+        )
 
     def extract_request_tool_names(self, data: dict) -> list[str]:
         """Extract tool names from Responses API request (tools[].name for function
@@ -331,8 +525,8 @@ class OpenAIResponsesHandler(BaseTranslation):
     async def _apply_guardrail_responses_to_input(
         self,
         messages: Any,  # Can be List[Dict[str, Any]] or ResponseInputParam
-        responses: list[str],
-        task_mappings: list[tuple[int, int | None]],
+        responses: Sequence[str],
+        task_mappings: Sequence[tuple[int, int | None]],
     ) -> None:
         """
         Apply guardrail responses back to input messages.

litellm/llms/openai/responses/transformation.py

@@ -15,6 +15,7 @@ from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response impo
 )
 from litellm.litellm_core_utils.url_utils import encode_url_path_segment
 from litellm.llms.base_llm.responses.transformation import BaseResponsesAPIConfig
+from litellm.llms.openai.chat.gpt_5_transformation import is_gpt_reasoning_series_name
 from litellm.responses.litellm_completion_transformation.custom_tools import TOOL_CALL_ITEM_ID_PREFIX_BY_TYPE
 from litellm.secret_managers.main import get_secret_str
 from litellm.types.llms.openai import *
@@ -88,7 +89,7 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig):
         parts: Final = model.split("/")
         if len(parts) > 1 and parts[0] not in ("openai",):
             return False
-        return "gpt-5" in model and "gpt-5-chat" not in model
+        return is_gpt_reasoning_series_name(model)
 
     @staticmethod
     def _supports_reasoning_effort_none(model: str) -> bool:

litellm/llms/openai/workload_identity.py

@@ -8,7 +8,7 @@ from urllib.parse import urlparse
 import litellm
 from litellm.secret_managers.main import get_secret_str, normalize_nonempty_secret_str
 
-from .common_utils import OpenAIError
+from .common_utils import OpenAIError, is_openai_backed_api_base
 
 if TYPE_CHECKING:
     from collections.abc import Callable
@@ -16,7 +16,6 @@ if TYPE_CHECKING:
     from openai.auth import SubjectTokenProvider, WorkloadIdentity, WorkloadIdentityAuth
 
 OPENAI_WIF_CLIENT_ID: Final = "litellm"
-_OPENAI_API_HOST: Final = "api.openai.com"
 _SDK_UPGRADE_MESSAGE: Final = (
     "OpenAI workload identity federation requires openai>=2.32.0. "
     "Upgrade the installed openai package to use OPENAI_IDENTITY_PROVIDER_ID / "
@@ -75,7 +74,7 @@ def _targets_openai_api(api_base: str | None) -> bool:
     if api_base is None:
         return True
     parsed: Final = urlparse(api_base)
-    return parsed.scheme == "https" and parsed.hostname == _OPENAI_API_HOST
+    return parsed.scheme == "https" and is_openai_backed_api_base(api_base)
 
 
 @lru_cache(maxsize=16)

litellm/llms/vertex_ai/common_utils.py

@@ -998,6 +998,16 @@ def replace_project_and_location_in_route(requested_route: str, vertex_project:
     return modified_route
 
 
+def _api_version_for_route(requested_route: str) -> Literal["v1", "v1beta1"]:
+    return "v1beta1" if "cachedContent" in requested_route else "v1"
+
+
+def _with_api_version(requested_route: str) -> str:
+    if not requested_route.startswith("/projects/"):
+        return requested_route
+    return f"/{_api_version_for_route(requested_route)}{requested_route}"
+
+
 def construct_target_url(
     base_url: str,
     requested_route: str,
@@ -1017,18 +1027,19 @@ def construct_target_url(
 
     new_base_url: Final = httpx.URL(base_url)
     if "locations" in requested_route:  # contains the target project id + location
-        if vertex_project and vertex_location:
-            requested_route = replace_project_and_location_in_route(requested_route, vertex_project, vertex_location)
-        return new_base_url.copy_with(path=requested_route)
+        targeted_route: Final = (
+            replace_project_and_location_in_route(requested_route, vertex_project, vertex_location)
+            if vertex_project and vertex_location
+            else requested_route
+        )
+        return new_base_url.copy_with(path=_with_api_version(targeted_route))
 
     """
     - Add endpoint version (e.g. v1beta for cachedContent, v1 for rest)
     - Add default project id
     - Add default location
     """
-    vertex_version: Literal["v1", "v1beta1"] = "v1"
-    if "cachedContent" in requested_route:
-        vertex_version = "v1beta1"
+    vertex_version: Literal["v1", "v1beta1"] = _api_version_for_route(requested_route)
 
     # Check if the requested route starts with a version
     # e.g. /v1beta1/publishers/google/models/gemini-3-pro-preview:streamGenerateContent

litellm/main.py

@@ -26,6 +26,7 @@ from copy import deepcopy
 from functools import partial
 from types import MappingProxyType
 from typing import TYPE_CHECKING, Any, Final, Literal, Optional, Protocol, Union, cast, get_args
+from urllib.parse import urlsplit
 
 from litellm._logging import _redact_string
 from litellm._uuid import uuid
@@ -60,6 +61,7 @@ if TYPE_CHECKING:
     from litellm.types.utils import TokenCountResponse
 
 from litellm.constants import (
+    AZURE_OPENAI_AUDIO_PROVIDERS,
     DEFAULT_MOCK_RESPONSE_COMPLETION_TOKEN_COUNT,
     DEFAULT_MOCK_RESPONSE_PROMPT_TOKEN_COUNT,
 )
@@ -984,6 +986,12 @@ def mock_completion(
 
 
 _OPENAI_DEFAULT_API_BASE: Final = "https://api.openai.com/v1"
+_OPENAI_API_HOST: Final = "api.openai.com"
+
+
+def _is_openai_backed_api_base(api_base: str) -> bool:
+    hostname: Final = urlsplit(api_base).hostname
+    return hostname is not None and (hostname == _OPENAI_API_HOST or hostname.endswith(f".{_OPENAI_API_HOST}"))
 
 
 def _resolve_openai_api_base(api_base: str | None) -> str:
@@ -1053,7 +1061,7 @@ def responses_api_bridge_check(
     #   natively by Chat Completions with reasoning on, so custom-only requests stay on
     #   chat and keep their native custom tool_call response shape.
     # - The UNSET-effort arm only fires against endpoints known to enforce that
-    #   constraint (the default OpenAI endpoint, or Azure OpenAI where api_base is
+    #   constraint (any api.openai.com host, or Azure OpenAI where api_base is
     #   always set): chat-only OpenAI-compatible backends registered under the openai
     #   provider with a custom api_base and gpt-5.4+ model names serve tools without
     #   reasoning fine and have no /responses route, so they keep pre-existing
@@ -1068,14 +1076,15 @@ def responses_api_bridge_check(
         reasoning_active = reasoning_effort.get("effort") != "none" or reasoning_effort.get("summary") is not None
     else:
         reasoning_active = reasoning_effort != "none"
-    # The reasoning+tools constraint is enforced only by the real OpenAI endpoint (and Azure OpenAI).
-    # Resolve the effective base arg>global>env>default exactly as the chat handler does, so a custom
-    # base set via litellm.api_base or OPENAI_BASE_URL/OPENAI_API_BASE isn't misread as the default and
-    # bridged to a /responses route it lacks. A whitespace-only base collapses to the default too.
-    resolved_api_base: Final = _resolve_openai_api_base(api_base)
-    on_constraint_enforcing_endpoint: Final = custom_llm_provider == "azure" or resolved_api_base.strip() in (
-        "",
-        _OPENAI_DEFAULT_API_BASE,
+    # The reasoning+tools constraint is enforced by the real OpenAI backend behind any api.openai.com
+    # host (the default URL or a PrivateLink hostname such as <region>.privatelink.api.openai.com) and
+    # by Azure OpenAI. Resolve the effective base arg>global>env>default exactly as the chat handler
+    # does, so a custom base set via litellm.api_base or OPENAI_BASE_URL/OPENAI_API_BASE isn't misread
+    # as the default and bridged to a /responses route it lacks. A whitespace-only base collapses to
+    # the default too.
+    resolved_api_base: Final = _resolve_openai_api_base(api_base).strip()
+    on_constraint_enforcing_endpoint: Final = (
+        custom_llm_provider == "azure" or resolved_api_base == "" or _is_openai_backed_api_base(resolved_api_base)
     )
     if (
         custom_llm_provider in ("openai", "azure")
@@ -7769,7 +7778,7 @@ def transcription(
         provider=LlmProviders(custom_llm_provider),
     )
 
-    if custom_llm_provider == "azure" and provider_config is None:
+    if custom_llm_provider in AZURE_OPENAI_AUDIO_PROVIDERS and provider_config is None:
         # azure configs
         api_base = api_base or litellm.api_base or get_secret_str("AZURE_API_BASE")
 
@@ -8056,7 +8065,10 @@ def speech(
         custom_llm_provider=custom_llm_provider,
     )
     response: HttpxBinaryResponseContent | Coroutine[object, object, HttpxBinaryResponseContent] | None = None
-    if custom_llm_provider == "openai" or custom_llm_provider in litellm.openai_compatible_providers:
+    if custom_llm_provider == "openai" or (
+        custom_llm_provider in litellm.openai_compatible_providers
+        and custom_llm_provider not in AZURE_OPENAI_AUDIO_PROVIDERS
+    ):
         if voice is None or not (isinstance(voice, str)):
             raise litellm.BadRequestError(
                 message="'voice' is required to be passed as a string for OpenAI TTS",
@@ -8110,7 +8122,7 @@ def speech(
             aspeech=aspeech,
             shared_session=shared_session,
         )
-    elif custom_llm_provider == "azure":
+    elif custom_llm_provider in AZURE_OPENAI_AUDIO_PROVIDERS:
         # Check if this is Azure Speech Service (Cognitive Services TTS)
         if model.startswith("speech/"):
             from litellm.llms.azure.text_to_speech.transformation import (

litellm/model_prices_and_context_window_backup.json

@@ -10305,6 +10305,24 @@
         "supports_vision": true,
         "supports_web_search": true
     },
+    "azure_ai/grok-4.6": {
+        "cache_read_input_token_cost": 5e-07,
+        "input_cost_per_token": 2e-06,
+        "litellm_provider": "azure_ai",
+        "max_input_tokens": 200000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "mode": "chat",
+        "output_cost_per_token": 6e-06,
+        "source": "https://techcommunity.microsoft.com/blog/azure-ai-foundry-blog/grok-4-6-comes-to-microsoft-foundry-models-built-for-long-horizon-reasoning-and-/4547578",
+        "supports_function_calling": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_tool_choice": true,
+        "supports_vision": true,
+        "supports_web_search": true
+    },
     "azure_ai/grok-4-fast-non-reasoning": {
         "deprecation_date": "2026-05-01",
         "input_cost_per_token": 2e-07,
@@ -29277,6 +29295,75 @@
         "supports_xhigh_reasoning_effort": true,
         "supports_minimal_reasoning_effort": true
     },
+    "gpt-6-astra": {
+        "cache_creation_input_token_cost": 1.25e-05,
+        "cache_creation_input_token_cost_above_272k_tokens": 2.5e-05,
+        "cache_creation_input_token_cost_above_272k_tokens_flex": 1.25e-05,
+        "cache_creation_input_token_cost_above_272k_tokens_priority": 5e-05,
+        "cache_creation_input_token_cost_flex": 6.25e-06,
+        "cache_creation_input_token_cost_priority": 2.5e-05,
+        "cache_read_input_token_cost": 1e-06,
+        "cache_read_input_token_cost_above_272k_tokens": 2e-06,
+        "cache_read_input_token_cost_above_272k_tokens_flex": 1e-06,
+        "cache_read_input_token_cost_above_272k_tokens_priority": 4e-06,
+        "cache_read_input_token_cost_flex": 5e-07,
+        "cache_read_input_token_cost_priority": 2e-06,
+        "input_cost_per_token": 1e-05,
+        "input_cost_per_token_above_272k_tokens": 2e-05,
+        "input_cost_per_token_above_272k_tokens_flex": 1e-05,
+        "input_cost_per_token_above_272k_tokens_priority": 4e-05,
+        "input_cost_per_token_batches": 5e-06,
+        "input_cost_per_token_flex": 5e-06,
+        "input_cost_per_token_priority": 2e-05,
+        "litellm_provider": "openai",
+        "max_input_tokens": 922000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "mode": "chat",
+        "output_cost_per_token": 5e-05,
+        "output_cost_per_token_above_272k_tokens": 7.5e-05,
+        "output_cost_per_token_above_272k_tokens_flex": 3.75e-05,
+        "output_cost_per_token_above_272k_tokens_priority": 0.00015,
+        "output_cost_per_token_batches": 2.5e-05,
+        "output_cost_per_token_flex": 2.5e-05,
+        "output_cost_per_token_priority": 0.0001,
+        "regional_processing_uplift_multiplier_eu": 1.1,
+        "regional_processing_uplift_multiplier_us": 1.1,
+        "search_context_cost_per_query": {
+            "search_context_size_high": 0.01,
+            "search_context_size_low": 0.01,
+            "search_context_size_medium": 0.01
+        },
+        "supported_endpoints": [
+            "/v1/chat/completions",
+            "/v1/batch",
+            "/v1/responses"
+        ],
+        "supported_modalities": [
+            "text",
+            "image"
+        ],
+        "supported_output_modalities": [
+            "text"
+        ],
+        "supports_computer_use": true,
+        "supports_function_calling": true,
+        "supports_max_reasoning_effort": true,
+        "supports_minimal_reasoning_effort": false,
+        "supports_native_streaming": true,
+        "supports_none_reasoning_effort": false,
+        "supports_parallel_function_calling": true,
+        "supports_pdf_input": true,
+        "supports_prompt_cache_breakpoint": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_system_messages": true,
+        "supports_tool_choice": true,
+        "supports_vision": true,
+        "supports_web_search": true,
+        "supports_xhigh_reasoning_effort": true
+    },
     "gpt-5.6": {
         "cache_creation_input_token_cost": 5e-06,
         "cache_creation_input_token_cost_above_272k_tokens": 1e-05,
@@ -52911,6 +52998,11 @@
         "cache_read_input_token_cost_above_272k_tokens": 1.1e-06,
         "output_cost_per_token": 3.3e-05,
         "output_cost_per_token_above_272k_tokens": 4.95e-05,
+        "search_context_cost_per_query": {
+            "search_context_size_high": 0.012,
+            "search_context_size_low": 0.012,
+            "search_context_size_medium": 0.012
+        },
         "litellm_provider": "bedrock_mantle",
         "max_input_tokens": 1050000,
         "max_output_tokens": 128000,
@@ -52945,6 +53037,11 @@
         "cache_read_input_token_cost_above_272k_tokens": 4.4e-07,
         "output_cost_per_token": 1.32e-05,
         "output_cost_per_token_above_272k_tokens": 1.98e-05,
+        "search_context_cost_per_query": {
+            "search_context_size_high": 0.012,
+            "search_context_size_low": 0.012,
+            "search_context_size_medium": 0.012
+        },
         "litellm_provider": "bedrock_mantle",
         "max_input_tokens": 1050000,
         "max_output_tokens": 128000,
@@ -53007,6 +53104,11 @@
         "cache_read_input_token_cost_above_272k_tokens": 4.4e-08,
         "output_cost_per_token": 1.32e-06,
         "output_cost_per_token_above_272k_tokens": 1.98e-06,
+        "search_context_cost_per_query": {
+            "search_context_size_high": 0.012,
+            "search_context_size_low": 0.012,
+            "search_context_size_medium": 0.012
+        },
         "litellm_provider": "bedrock_mantle",
         "max_input_tokens": 1050000,
         "max_output_tokens": 128000,
@@ -53195,6 +53297,11 @@
         "cache_read_input_token_cost_above_272k_tokens": 1.1e-06,
         "output_cost_per_token": 3.3e-05,
         "output_cost_per_token_above_272k_tokens": 4.95e-05,
+        "search_context_cost_per_query": {
+            "search_context_size_high": 0.012,
+            "search_context_size_low": 0.012,
+            "search_context_size_medium": 0.012
+        },
         "litellm_provider": "bedrock_mantle",
         "max_input_tokens": 1050000,
         "max_output_tokens": 128000,
@@ -53226,6 +53333,11 @@
         "cache_read_input_token_cost_above_272k_tokens": 5.5e-07,
         "output_cost_per_token": 1.65e-05,
         "output_cost_per_token_above_272k_tokens": 2.475e-05,
+        "search_context_cost_per_query": {
+            "search_context_size_high": 0.012,
+            "search_context_size_low": 0.012,
+            "search_context_size_medium": 0.012
+        },
         "litellm_provider": "bedrock_mantle",
         "max_input_tokens": 1050000,
         "max_output_tokens": 128000,

litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py

@@ -448,6 +448,17 @@ def _append_query_params(url: str, params: dict[str, str]) -> str:
     return urlunparse(parsed._replace(query=urlencode(query_params)))
 
 
+def _resolve_mcp_server_by_name_or_id(lookup: str, client_ip: str | None) -> MCPServer | None:
+    from litellm.proxy._experimental.mcp_server.mcp_server_manager import (
+        global_mcp_server_manager,
+    )
+
+    by_name: Final = global_mcp_server_manager.get_mcp_server_by_name(lookup, client_ip=client_ip)
+    if by_name is not None:
+        return by_name
+    return global_mcp_server_manager.get_mcp_server_by_id(lookup, client_ip=client_ip)
+
+
 def _resolve_oauth2_server_for_root_endpoints(
     client_ip: str | None = None,
 ) -> MCPServer | None:
@@ -1766,10 +1777,6 @@ async def authorize(
     resource: str | None = None,
 ):
     # Redirect to real OAuth provider with PKCE support
-    from litellm.proxy._experimental.mcp_server.mcp_server_manager import (
-        global_mcp_server_manager,
-    )
-
     if mcp_server_name is None and client_id and is_gateway_dcr_client_id(client_id):
         if is_proxy_api_resource(request, resource):
             return await native_client_authorize(
@@ -1797,9 +1804,7 @@ async def authorize(
 
     lookup_name: Final[str | None] = mcp_server_name or client_id
     client_ip: Final = IPAddressUtils.get_mcp_client_ip(request)
-    mcp_server = (
-        global_mcp_server_manager.get_mcp_server_by_name(lookup_name, client_ip=client_ip) if lookup_name else None
-    )
+    mcp_server = _resolve_mcp_server_by_name_or_id(lookup_name, client_ip) if lookup_name else None
     if mcp_server is None and mcp_server_name is None:
         mcp_server = _resolve_oauth2_server_for_root_endpoints(client_ip=client_ip)
     if mcp_server is None:
@@ -1855,10 +1860,6 @@ async def token_endpoint(
     3. Return the token
     4. Return a virtual key in this response
     """
-    from litellm.proxy._experimental.mcp_server.mcp_server_manager import (
-        global_mcp_server_manager,
-    )
-
     if mcp_server_name is None and is_gateway_dcr_client_id(client_id):
         from litellm.proxy.proxy_server import (  # noqa: PLC0415  # circular import at module load
             master_key,
@@ -1882,7 +1883,7 @@ async def token_endpoint(
 
     lookup_name: Final = mcp_server_name or client_id
     client_ip: Final = IPAddressUtils.get_mcp_client_ip(request)
-    mcp_server = global_mcp_server_manager.get_mcp_server_by_name(lookup_name, client_ip=client_ip)
+    mcp_server = _resolve_mcp_server_by_name_or_id(lookup_name, client_ip)
     if mcp_server is None and mcp_server_name is None:
         mcp_server = _resolve_oauth2_server_for_root_endpoints(client_ip=client_ip)
     if mcp_server is None:
@@ -2288,10 +2289,6 @@ async def _build_oauth_protected_resource_response(
     Returns:
         OAuth protected resource metadata dict
     """
-    from litellm.proxy._experimental.mcp_server.mcp_server_manager import (
-        global_mcp_server_manager,
-    )
-
     request_base_url: Final = get_request_base_url(request)
     client_ip: Final = IPAddressUtils.get_mcp_client_ip(request)
     explicitly_named: Final = mcp_server_name is not None
@@ -2304,7 +2301,7 @@ async def _build_oauth_protected_resource_response(
 
     mcp_server: MCPServer | None = None
     if mcp_server_name:
-        mcp_server = global_mcp_server_manager.get_mcp_server_by_name(mcp_server_name, client_ip=client_ip)
+        mcp_server = _resolve_mcp_server_by_name_or_id(mcp_server_name, client_ip)
 
     # Build resource URL based on the pattern
     if mcp_server_name:
@@ -2562,10 +2559,6 @@ def _build_oauth_authorization_server_response(
     registry lookups; unlike :func:`_build_oauth_protected_resource_response`
     it does not need to await any upstream IO.
     """
-    from litellm.proxy._experimental.mcp_server.mcp_server_manager import (
-        global_mcp_server_manager,
-    )
-
     request_base_url: Final = get_request_base_url(request)
     client_ip: Final = IPAddressUtils.get_mcp_client_ip(request)
     explicitly_named: Final = mcp_server_name is not None
@@ -2583,7 +2576,7 @@ def _build_oauth_authorization_server_response(
 
     mcp_server: MCPServer | None = None
     if mcp_server_name:
-        mcp_server = global_mcp_server_manager.get_mcp_server_by_name(mcp_server_name, client_ip=client_ip)
+        mcp_server = _resolve_mcp_server_by_name_or_id(mcp_server_name, client_ip)
 
     _raise_unless_oauth2_discovery_server(mcp_server, mcp_server_name, "not an OAuth authorization server")
 
@@ -2709,10 +2702,6 @@ async def oauth_authorization_server_legacy(request: Request, mcp_server_name: s
 @router.post("/{mcp_server_name}/register")
 @router.post("/register")
 async def register_client(request: Request, mcp_server_name: str | None = None):
-    from litellm.proxy._experimental.mcp_server.mcp_server_manager import (
-        global_mcp_server_manager,
-    )
-
     # Get the correct base URL considering X-Forwarded-* headers
     request_base_url: Final = get_request_base_url(request)
 
@@ -2748,7 +2737,7 @@ async def register_client(request: Request, mcp_server_name: str | None = None):
             )
         return dummy_return
 
-    mcp_server: Final = global_mcp_server_manager.get_mcp_server_by_name(mcp_server_name, client_ip=client_ip)
+    mcp_server: Final = _resolve_mcp_server_by_name_or_id(mcp_server_name, client_ip)
     if mcp_server is None:
         return dummy_return
     return await register_client_with_server(

litellm/proxy/_experimental/mcp_server/mcp_server_manager.py

@@ -47,7 +47,7 @@ from litellm.constants import (
     MCP_TOOL_LISTING_TIMEOUT,
 )
 from litellm.exceptions import BlockedPiiEntityError, GuardrailRaisedException
-from litellm.experimental_mcp_client.client import MCPClient, MCPSigV4Auth, strip_auth_scheme
+from litellm.experimental_mcp_client.client import MCPClient, MCPSigV4Auth, strip_auth_scheme, to_basic_credentials
 from litellm.integrations.custom_guardrail import (
     _sync_guardrail_info_to_logging_obj,  # pyright: ignore[reportPrivateUsage] - the same bridge @log_guardrail_information uses; reimplementing it here would fork the metadata-key logic
 )
@@ -2299,13 +2299,13 @@ class MCPServerManager:
                 from litellm.types.mcp import MCPAuth
 
                 if server.auth_type == MCPAuth.bearer_token:
-                    headers["Authorization"] = f"Bearer {server.authentication_token}"
+                    headers["Authorization"] = f"Bearer {strip_auth_scheme(server.authentication_token, 'Bearer')}"
                 elif server.auth_type == MCPAuth.api_key:
-                    headers["Authorization"] = f"ApiKey {server.authentication_token}"
+                    headers["Authorization"] = f"ApiKey {strip_auth_scheme(server.authentication_token, 'ApiKey')}"
                 elif server.auth_type == MCPAuth.basic:
-                    headers["Authorization"] = f"Basic {server.authentication_token}"
+                    headers["Authorization"] = f"Basic {to_basic_credentials(server.authentication_token)}"
                 elif server.auth_type == MCPAuth.token:
-                    headers["Authorization"] = f"token {server.authentication_token}"
+                    headers["Authorization"] = f"token {strip_auth_scheme(server.authentication_token, 'token')}"
 
             # Add any static headers from server config.
             #
@@ -3346,9 +3346,7 @@ class MCPServerManager:
             normalized: Final = {k.lower(): v for k, v in raw_headers.items()}
             auth_value = normalized.get("authorization")
         if auth_value:
-            if auth_value.startswith("Bearer "):
-                return auth_value[len("Bearer ") :]
-            return auth_value
+            return strip_auth_scheme(auth_value, "Bearer")
         return None
 
     @staticmethod
@@ -6147,13 +6145,13 @@ class MCPServerManager:
         internal_networks = IPAddressUtils.parse_internal_networks(general_settings.get("mcp_internal_ip_ranges"))
         return IPAddressUtils.is_internal_ip(client_ip, internal_networks)
 
-    def get_mcp_server_by_id(self, server_id: str) -> MCPServer | None:
-        """
-        Get the MCP Server from the server id
-        """
+    def get_mcp_server_by_id(self, server_id: str, client_ip: str | None = None) -> MCPServer | None:
+        """Get the MCP Server from the server id."""
         registry: Final = self.get_registry()
         for server in registry.values():
             if server.server_id == server_id:
+                if not self._is_server_accessible_from_ip(server, client_ip):
+                    return None
                 return server
         return None
 

litellm/proxy/_experimental/mcp_server/outbound_credentials/adapter.py

@@ -18,6 +18,7 @@ from fastapi import HTTPException
 from pydantic import SecretStr
 from typing_extensions import assert_never
 
+from litellm.experimental_mcp_client.client import strip_auth_scheme, to_basic_credentials
 from litellm.proxy._experimental.mcp_server.oauth_utils import resolve_upstream_resource
 from litellm.proxy._experimental.mcp_server.outbound_credentials.types import (
     DEFAULT_CREDENTIAL_HEADER,
@@ -213,7 +214,9 @@ def _shared_key_spec(
     token: Final = server.authentication_token
     if not token:
         return None  # no key configured -> defer to v1 (parity-safe)
-    value: Final = base64.b64encode(token.encode("utf-8")).decode() if encode else token
+    value: Final = (
+        to_basic_credentials(token) if encode else strip_auth_scheme(token, value_prefix) if value_prefix else token
+    )
     return ServerSpec(
         server_id=server.server_id,
         resource=resource,

litellm/proxy/agent_endpoints/auth/agent_permission_handler.py

@@ -5,10 +5,13 @@ Handles agent permission checking for keys and teams using object_permission_id.
 Follows the same pattern as MCP permission handling.
 """
 
+import asyncio
+from collections.abc import Awaitable, Callable, Sequence
 from dataclasses import dataclass
 from typing import Final, TypeAlias
 
 from litellm._logging import verbose_logger
+from litellm.proxy._experimental.mcp_server.ui_session_utils import build_effective_auth_contexts
 from litellm.proxy._types import (
     UI_TEAM_ID,
     LiteLLM_ObjectPermissionTable,
@@ -443,15 +446,47 @@ class AgentRequestHandler:
             return []
 
 
-async def accessible_agents(user_api_key_auth: UserAPIKeyAuth) -> tuple[AgentResponse, ...]:
-    """Every registry agent for proxy admins, else the agents the key's and team's grants reach."""
+def _granted_ids(access: AgentAccess) -> frozenset[str]:
+    match access:
+        case UnrestrictedAgentAccess():
+            return frozenset()
+        case RestrictedAgentAccess(agent_ids):
+            return agent_ids
+
+
+ResolveAgentAccess: TypeAlias = Callable[[UserAPIKeyAuth], Awaitable[AgentAccess]]
+EffectiveAuthContexts: TypeAlias = Callable[[UserAPIKeyAuth], Awaitable[Sequence[UserAPIKeyAuth]]]
+
+
+async def _granted_agent_ids(
+    user_api_key_auth: UserAPIKeyAuth,
+    resolve_access: ResolveAgentAccess,
+    effective_contexts: EffectiveAuthContexts,
+) -> frozenset[str]:
+    """Union of the explicit grants reachable from the key, its team, or (for a dashboard session)
+    the user's real teams and user row. No grant anywhere yields the empty set, unlike the
+    open-by-default ``resolve_agent_access`` that guards direct access."""
+    accesses: Final = await asyncio.gather(
+        *(resolve_access(auth_context) for auth_context in await effective_contexts(user_api_key_auth))
+    )
+    return frozenset().union(*(_granted_ids(access) for access in accesses))
+
+
+async def accessible_agents(
+    user_api_key_auth: UserAPIKeyAuth,
+    all_agents: tuple[AgentResponse, ...] | None = None,
+    resolve_access: ResolveAgentAccess | None = None,
+    effective_contexts: EffectiveAuthContexts = build_effective_auth_contexts,
+) -> tuple[AgentResponse, ...]:
+    """Every registry agent for proxy admins, else only the agents the caller was granted."""
     from litellm.proxy.agent_endpoints.agent_registry import global_agent_registry
 
-    all_agents: Final = global_agent_registry.get_agent_list()
+    agents: Final = global_agent_registry.get_agent_list() if all_agents is None else all_agents
     if user_api_key_auth.user_role in (LitellmUserRoles.PROXY_ADMIN, LitellmUserRoles.PROXY_ADMIN.value):
-        return all_agents
-    match await AgentRequestHandler.resolve_agent_access(user_api_key_auth=user_api_key_auth):
-        case UnrestrictedAgentAccess():
-            return all_agents
-        case RestrictedAgentAccess(allowed_agent_ids):
-            return tuple(agent for agent in all_agents if agent.agent_id in allowed_agent_ids)
+        return agents
+    allowed_agent_ids: Final = await _granted_agent_ids(
+        user_api_key_auth,
+        AgentRequestHandler.resolve_agent_access if resolve_access is None else resolve_access,
+        effective_contexts,
+    )
+    return tuple(agent for agent in agents if agent.agent_id in allowed_agent_ids)

litellm/proxy/auth/auth_checks.py

@@ -5788,7 +5788,7 @@ async def vector_store_access_check(
     vector_store_ids_to_run: Final = litellm.vector_store_registry.get_vector_store_ids_to_run(
         non_default_params=request_body, tools=request_body.get("tools", None)
     )
-    if vector_store_ids_to_run is None:
+    if not vector_store_ids_to_run:
         verbose_proxy_logger.debug("Vector store to run not found, skipping vector store access check")
         return True
 

litellm/proxy/auth/litellm_license.py

@@ -16,6 +16,10 @@ if TYPE_CHECKING:
     from litellm.proxy._types import EnterpriseLicenseData
 
 
+AUTO_ROUTER_LICENSE_FEATURE: Final = "auto_router"
+HEURISTIC_V2_LICENSE_REMEDY: Final = "A LiteLLM license with the 'auto_router' feature lifts the limit."
+
+
 class LicenseCheck:
     """
     - Check if license in env
@@ -149,6 +153,19 @@ class LicenseCheck:
             return False
         return team_count > _max_teams_in_license
 
+    def heuristic_v2_router_limit(self) -> int | None:
+        """
+        How many heuristic_v2 auto-routers this proxy may hold: unlimited (None) only when the
+        signed license lists the auto_router feature, otherwise one. A license verified through
+        the API carries no feature list, so it does not lift the limit either.
+        """
+        if self.airgapped_license_data is None:
+            return 1
+        allowed_features: Final = self.airgapped_license_data.get("allowed_features")
+        if isinstance(allowed_features, list) and AUTO_ROUTER_LICENSE_FEATURE in allowed_features:
+            return None
+        return 1
+
     def verify_license_without_api_request(self, public_key, license_key):
         try:
             from cryptography.hazmat.primitives import hashes
@@ -179,19 +196,21 @@ class LicenseCheck:
             # Decode and parse the data
             license_data: Final = json.loads(message.decode())
 
-            self.airgapped_license_data = EnterpriseLicenseData(**license_data)
-
             # debug information provided in license data
             verbose_proxy_logger.debug("License data: %s", license_data)
 
             # Check expiration date
             expiration_date: Final = datetime.strptime(license_data["expiration_date"], "%Y-%m-%d")
             if expiration_date < datetime.now():
+                self.airgapped_license_data = None
                 return False, "License has expired"
 
+            self.airgapped_license_data = EnterpriseLicenseData(**license_data)
+
             return True
 
         except Exception as e:
+            self.airgapped_license_data = None
             verbose_proxy_logger.debug(
                 "litellm.proxy.auth.litellm_license.py::verify_license_without_api_request - Unable to verify License locally. - %s",
                 e,

litellm/proxy/common_request_processing.py

@@ -467,6 +467,42 @@ def _getattr_object(value: object, name: str, default: object = None) -> object:
     return getattr(value, name, default)
 
 
+_OPENAI_ERROR_TYPE_BY_STATUS: Final[Mapping[int, str]] = MappingProxyType(
+    {
+        status.HTTP_401_UNAUTHORIZED: "authentication_error",
+        status.HTTP_403_FORBIDDEN: "permission_error",
+        status.HTTP_429_TOO_MANY_REQUESTS: "rate_limit_error",
+    }
+)
+
+
+def _error_status_code(exc: object, default: int) -> int:
+    """The HTTP status an exception carries, or ``default`` when it carries none."""
+    carried: Final = _getattr_object(exc, "status_code")
+    return carried if isinstance(carried, int) and not isinstance(carried, bool) else default
+
+
+def _openai_error_type(exc: object, status_code: int) -> str:
+    """OpenAI types ``error.type`` as a required string, so an exception carrying none
+    falls back to the type its status code stands for."""
+    carried: Final = _getattr_object(exc, "type")
+    if isinstance(carried, str):
+        return carried
+    mapped: Final = _OPENAI_ERROR_TYPE_BY_STATUS.get(status_code)
+    if mapped is not None:
+        return mapped
+    if status_code < status.HTTP_500_INTERNAL_SERVER_ERROR:
+        return "invalid_request_error"
+    return "internal_server_error"
+
+
+def _openai_error_param(exc: object) -> str | None:
+    """OpenAI types ``error.param`` as nullable, so an exception carrying none
+    serializes as JSON ``null``."""
+    carried: Final = _getattr_object(exc, "param")
+    return carried if isinstance(carried, str) else None
+
+
 class _UpstreamHttpResponse(Protocol):
     @property
     def status_code(self) -> int: ...
@@ -540,11 +576,12 @@ def proxy_exception_from_http_exception(exc: HTTPException, headers: dict[str, s
     message, structured_fields = serialize_http_exception_detail(raw_detail)
     existing_fields: Final = getattr(exc, "provider_specific_fields", None) or {}
     merged_fields: Final = {**existing_fields, **structured_fields} if structured_fields else (existing_fields or None)
+    error_status: Final = _error_status_code(exc, status.HTTP_400_BAD_REQUEST)
     return ProxyException(
         message=message,
-        type=getattr(exc, "type", "None"),
-        param=getattr(exc, "param", "None"),
-        code=getattr(exc, "status_code", status.HTTP_400_BAD_REQUEST),
+        type=_openai_error_type(exc, error_status),
+        param=_openai_error_param(exc),
+        code=error_status,
         provider_specific_fields=merged_fields,
         headers=headers,
     )
@@ -827,25 +864,22 @@ def sse_error_payload(exc: BaseException) -> tuple[int, Mapping[str, object]]:
     are byte-identical.
     """
     # Preserve status code from HTTPException (e.g. guardrail blocks)
-    error_status: Final = getattr(exc, "status_code", status.HTTP_500_INTERNAL_SERVER_ERROR)
+    error_status: Final = _error_status_code(exc, status.HTTP_500_INTERNAL_SERVER_ERROR)
     raw_detail: Final = _getattr_object(exc, "detail", "Error processing stream start")
     message, structured_fields = serialize_http_exception_detail(raw_detail)
 
     existing_fields: Final = getattr(exc, "provider_specific_fields", None) or {}
     merged_fields: Final = {**existing_fields, **structured_fields} if structured_fields else (existing_fields or None)
 
-    # Built in one statement then given its one optional key, rather than spread
-    # conditionally: the spread form costs two extra dict constructions, which
-    # type-discipline-budget.json's LIT002 ceiling has no room for.
     error_obj: Final = {
         "message": message,
-        "type": getattr(exc, "type", "None"),
-        "param": getattr(exc, "param", "None"),
+        "type": _openai_error_type(exc, error_status),
+        "param": _openai_error_param(exc),
         "code": str(error_status),
     }
-    if merged_fields:
-        error_obj["provider_specific_fields"] = merged_fields
-    return error_status, error_obj
+    if not merged_fields:
+        return error_status, error_obj
+    return error_status, {**error_obj, "provider_specific_fields": merged_fields}
 
 
 def _sse_error_frames(error_obj: Mapping[str, object]) -> tuple[str, str]:
@@ -922,7 +956,7 @@ async def create_response(
                 "error": {
                     "message": _CLIENT_DISCONNECT_DETAIL,
                     "type": "client_disconnect",
-                    "param": "None",
+                    "param": None,
                     "code": str(LITELLM_HTTP_STATUS_CLIENT_DISCONNECTED),
                 }
             },
@@ -3417,8 +3451,8 @@ class ProxyBaseLLMRequestProcessing:
             _code = status.HTTP_500_INTERNAL_SERVER_ERROR
         raise ProxyException(
             message=redact_internal_details_from_client_message(getattr(e, "message", error_msg)),
-            type=getattr(e, "type", "None"),
-            param=getattr(e, "param", "None"),
+            type=_openai_error_type(e, _code),
+            param=_openai_error_param(e),
             openai_code=getattr(e, "code", None),
             code=_code,
             provider_specific_fields=getattr(e, "provider_specific_fields", None),
@@ -3628,11 +3662,12 @@ class ProxyBaseLLMRequestProcessing:
 
             if isinstance(e, HTTPException):
                 raise e
+            stream_error_status: Final = _error_status_code(e, status.HTTP_500_INTERNAL_SERVER_ERROR)
             proxy_exception: Final = ProxyException(
                 message=redact_internal_details_from_client_message(getattr(e, "message", str(e))),
-                type=getattr(e, "type", "None"),
-                param=getattr(e, "param", "None"),
-                code=getattr(e, "status_code", 500),
+                type=_openai_error_type(e, stream_error_status),
+                param=_openai_error_param(e),
+                code=stream_error_status,
             )
             stream_completed = True
             yield serialize_error(proxy_exception)

litellm/proxy/common_utils/http_parsing_utils.py

@@ -1,10 +1,12 @@
 import json
 import re
-from collections.abc import Collection
-from typing import Any, Final
+from collections.abc import Collection, Mapping
+from types import MappingProxyType, UnionType
+from typing import Any, Final, Union, get_args, get_origin
 
 import orjson
 from fastapi import Request, UploadFile, status
+from typing_extensions import ReadOnly
 
 from litellm._logging import verbose_proxy_logger
 from litellm.constants import MAX_REQUEST_BODY_SIZE_TO_REPAIR_MB
@@ -40,6 +42,65 @@ def _is_json_content_type(content_type: str) -> bool:
     return _normalize_media_type(content_type) == "application/json"
 
 
+def _numeric_form_type(annotation: object) -> type[int] | type[float] | None:
+    """The scalar to parse an ``int``/``float``-typed field as, else ``None``."""
+    unwrapped: Final = get_args(annotation)[0] if get_origin(annotation) is ReadOnly else annotation
+    candidates: Final = (
+        tuple(arg for arg in get_args(unwrapped) if arg is not type(None))
+        if get_origin(unwrapped) in (Union, UnionType)
+        else (unwrapped,)
+    )
+    if len(candidates) != 1:
+        return None
+    if candidates[0] is int:
+        return int
+    if candidates[0] is float:
+        return float
+    return None
+
+
+def numeric_form_fields(annotations: Mapping[str, object]) -> Mapping[str, type[int] | type[float]]:
+    """
+    The numeric fields of a request schema, mapped to the scalar to parse them as.
+
+    Only a bare ``int``/``float`` or an optional one qualifies, so container and
+    literal fields are left alone and ``bool`` is excluded on purpose.
+    """
+    return MappingProxyType(
+        {
+            name: scalar
+            for name, annotation in annotations.items()
+            if (scalar := _numeric_form_type(annotation)) is not None
+        }
+    )
+
+
+def _numeric_form_value(value: object, scalar: type[int] | type[float]) -> object:
+    if not isinstance(value, str):
+        return value
+    try:
+        return scalar(value)
+    except ValueError:
+        return value
+
+
+def coerce_numeric_form_fields(
+    parsed_body: Mapping[str, object],
+    numeric_fields: Mapping[str, type[int] | type[float]],
+) -> Mapping[str, object]:
+    """
+    Parse the numeric fields of a form-encoded body back into numbers.
+
+    ``request.form()`` yields every field as a string, so a provider that puts the
+    value in a JSON body would send a string where its API requires a number. A
+    value that will not parse is left as-is for the provider to reject as before.
+    """
+    return {
+        name: _numeric_form_value(value, numeric_fields[name]) if name in numeric_fields else value
+        for name, value in parsed_body.items()
+    }
+
+
 async def _read_request_body(request: Request | None) -> dict:
     """
     Safely read the request body and parse it as JSON.

litellm/proxy/container_endpoints/endpoints.py

@@ -15,10 +15,11 @@ from litellm.proxy.common_utils.openai_endpoint_utils import (
     get_custom_llm_provider_from_request_headers,
     get_custom_llm_provider_from_request_query,
 )
+from litellm.proxy.common_utils.resource_ownership import is_proxy_admin
 from litellm.proxy.container_endpoints.ownership import (
     assert_user_can_access_container,
-    filter_container_list_response,
     get_container_forwarding_params,
+    list_owned_containers,
     record_container_owner,
 )
 
@@ -173,6 +174,9 @@ async def list_containers(
     request: Request,
     fastapi_response: Response,
     user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),
+    after: str | None = None,
+    limit: int | None = None,
+    order: str | None = None,
 ):
     """
     Container list endpoint for retrieving a list of containers.
@@ -206,55 +210,54 @@ async def list_containers(
         version,
     )
 
-    # Read query parameters
     query_params: Final = dict(request.query_params)
-    data: Final[dict[str, Any]] = {"query_params": query_params, "model": query_params.get("model")}
-
-    # Extract custom_llm_provider using priority chain
     custom_llm_provider: Final = (
         get_custom_llm_provider_from_request_headers(request=request)
         or get_custom_llm_provider_from_request_query(request=request)
         or "openai"
     )
+    data: Final[dict[str, Any]] = {
+        "query_params": query_params,
+        "model": query_params.get("model"),
+        "order": order,
+        "custom_llm_provider": custom_llm_provider,
+    }
 
-    # Add custom_llm_provider to data
-    data["custom_llm_provider"] = custom_llm_provider
+    async def fetch_page(page_after: str | None, page_limit: int | None) -> object:
+        processor: Final = ProxyBaseLLMRequestProcessing(data={**data, "after": page_after, "limit": page_limit})
+        try:
+            return await processor.base_process_llm_request(
+                request=request,
+                fastapi_response=fastapi_response,
+                user_api_key_dict=user_api_key_dict,
+                route_type="alist_containers",
+                proxy_logging_obj=proxy_logging_obj,
+                llm_router=llm_router,
+                general_settings=general_settings,
+                proxy_config=proxy_config,
+                select_data_generator=select_data_generator,
+                model=None,
+                user_model=user_model,
+                user_temperature=user_temperature,
+                user_request_timeout=user_request_timeout,
+                user_max_tokens=user_max_tokens,
+                user_api_base=user_api_base,
+                version=version,
+            )
+        except Exception as e:
+            raise await processor._handle_llm_api_exception(
+                e=e,
+                user_api_key_dict=user_api_key_dict,
+                proxy_logging_obj=proxy_logging_obj,
+                version=version,
+            )
 
-    # Process request using ProxyBaseLLMRequestProcessing
-    processor: Final = ProxyBaseLLMRequestProcessing(data=data)
-    try:
-        response: Final = await processor.base_process_llm_request(
-            request=request,
-            fastapi_response=fastapi_response,
-            user_api_key_dict=user_api_key_dict,
-            route_type="alist_containers",
-            proxy_logging_obj=proxy_logging_obj,
-            llm_router=llm_router,
-            general_settings=general_settings,
-            proxy_config=proxy_config,
-            select_data_generator=select_data_generator,
-            model=None,
-            user_model=user_model,
-            user_temperature=user_temperature,
-            user_request_timeout=user_request_timeout,
-            user_max_tokens=user_max_tokens,
-            user_api_base=user_api_base,
-            version=version,
-        )
-    except Exception as e:
-        raise await processor._handle_llm_api_exception(
-            e=e,
-            user_api_key_dict=user_api_key_dict,
-            proxy_logging_obj=proxy_logging_obj,
-            version=version,
-        )
-
-    # Ownership filtering runs OUTSIDE the LLM-exception scope: a DB error
-    # in the ownership lookup is not an LLM-API error and shouldn't be
-    # translated to a provider-shaped failure (which would also fire the
-    # post_call_failure_hook for what is in fact a successful upstream call).
-    return await filter_container_list_response(
-        response=response,
+    if is_proxy_admin(user_api_key_dict):
+        return await fetch_page(after, limit)
+    return await list_owned_containers(
+        fetch_page=fetch_page,
+        after=after,
+        limit=limit,
         user_api_key_dict=user_api_key_dict,
         custom_llm_provider=custom_llm_provider,
     )

litellm/proxy/container_endpoints/handler_factory.py

@@ -6,7 +6,9 @@ FastAPI route handlers for ALL container file endpoints.
 """
 
 import json
+from collections.abc import Mapping, Sequence
 from pathlib import Path
+from types import MappingProxyType
 from typing import Any, Final
 
 from fastapi import APIRouter, Depends, Request, Response
@@ -56,6 +58,7 @@ def _create_handler_for_path_params(
     route_type: str,
     returns_binary: bool = False,
     is_multipart: bool = False,
+    query_param_names: Sequence[str] = (),
 ):
     """
     Dynamically create a handler with the correct path parameter signature.
@@ -114,6 +117,7 @@ def _create_handler_for_path_params(
                 user_api_key_dict=user_api_key_dict,
                 route_type=route_type,
                 path_params={"container_id": container_id},
+                query_param_names=query_param_names,
             )
 
         return handler_container_id
@@ -133,6 +137,7 @@ def _create_handler_for_path_params(
                 user_api_key_dict=user_api_key_dict,
                 route_type=route_type,
                 path_params={"container_id": container_id, "file_id": file_id},
+                query_param_names=query_param_names,
             )
 
         return handler_container_file
@@ -150,6 +155,7 @@ def _create_handler_for_path_params(
                 user_api_key_dict=user_api_key_dict,
                 route_type=route_type,
                 path_params={},
+                query_param_names=query_param_names,
             )
 
         return handler_no_params
@@ -351,12 +357,17 @@ async def _process_multipart_upload_request(
         )
 
 
+def _declared_query_params(query_params: Mapping[str, str], query_param_names: Sequence[str]) -> Mapping[str, str]:
+    return MappingProxyType({name: query_params[name] for name in query_param_names if name in query_params})
+
+
 async def _process_request(
     request: Request,
     fastapi_response: Response,
     user_api_key_dict: UserAPIKeyAuth,
     route_type: str,
     path_params: dict[str, str],
+    query_param_names: Sequence[str] = (),
 ):
     """Common request processing logic."""
     from litellm.proxy.proxy_server import (
@@ -376,6 +387,7 @@ async def _process_request(
     query_params: Final = dict(request.query_params)
     data: Final[dict[str, Any]] = {
         "query_params": query_params,
+        **_declared_query_params(query_params, query_param_names),
         **path_params,
     }
 
@@ -452,7 +464,13 @@ def register_container_file_endpoints(router: APIRouter) -> None:
         is_multipart = endpoint_config.get("is_multipart", False)
 
         # Create handler with correct signature for path params
-        handler = _create_handler_for_path_params(path_params, route_type, returns_binary, is_multipart)
+        handler = _create_handler_for_path_params(
+            path_params,
+            route_type,
+            returns_binary,
+            is_multipart,
+            query_param_names=endpoint_config.get("query_params", ()),
+        )
 
         # Register routes
         route_method = getattr(router, method)

litellm/proxy/container_endpoints/ownership.py

@@ -1,9 +1,10 @@
 import json
-from collections.abc import Mapping, Sequence
+from collections.abc import Awaitable, Callable, Mapping, Sequence
 from collections.abc import Set as AbstractSet
-from typing import TYPE_CHECKING, Any, Final
+from typing import TYPE_CHECKING, Any, Final, TypeAlias
 
 from fastapi import HTTPException
+from pydantic import BaseModel
 
 from litellm._logging import verbose_proxy_logger
 from litellm.caching.in_memory_cache import InMemoryCache
@@ -46,6 +47,12 @@ _CONTAINER_STORED_ID_CACHE: Final = InMemoryCache(max_size_in_memory=10000, defa
 # different users with different scopes get disjoint cache entries.
 _ALLOWED_CONTAINER_IDS_CACHE: Final = InMemoryCache(max_size_in_memory=2048, default_ttl=60)
 
+DEFAULT_CONTAINER_LIST_LIMIT: Final = 20
+OWNED_CONTAINER_LIST_PAGE_SIZE: Final = 100
+OWNED_CONTAINER_LIST_MAX_PAGES: Final = 5
+
+FetchContainerListPage: TypeAlias = Callable[[str | None, int | None], Awaitable[object]]
+
 
 def _allowed_container_ids_cache_key(owner_scopes: Sequence[str]) -> str:
     """JSON-encode the sorted scope list — using a separator like ``|``
@@ -337,27 +344,23 @@ def _get_container_list_data(response: object) -> Sequence[object] | None:
     return data if isinstance(data, list) else None
 
 
-def _set_container_list_data(response: Any, data: list[object], removed_filtered_items: bool = False) -> object:
+def _get_has_more(response: object) -> bool:
     if isinstance(response, dict):
-        response["data"] = data
-        if data:
-            response["first_id"] = _get_response_id(data[0])
-            response["last_id"] = _get_response_id(data[-1])
-        else:
-            response["first_id"] = None
-            response["last_id"] = None
-            response["has_more"] = False
-        if removed_filtered_items:
-            response["has_more"] = False
-        return response
+        return response.get("has_more") is True
+    return getattr(response, "has_more", None) is True
 
-    response.data = data
-    response.first_id = _get_response_id(data[0]) if data else None
-    response.last_id = _get_response_id(data[-1]) if data else None
-    if not data and hasattr(response, "has_more"):
-        response.has_more = False
-    if removed_filtered_items and hasattr(response, "has_more"):
-        response.has_more = False
+
+def _with_container_list_page(response: object, data: Sequence[object], has_more: bool) -> object:
+    page: Final = {
+        "data": list(data),
+        "first_id": _get_response_id(data[0]) if data else None,
+        "last_id": _get_response_id(data[-1]) if data else None,
+        "has_more": has_more,
+    }
+    if isinstance(response, dict):
+        return {**response, **page}
+    if isinstance(response, BaseModel):
+        return response.model_copy(update=page)
     return response
 
 
@@ -366,16 +369,16 @@ async def _get_allowed_container_ids(
 ) -> AbstractSet[str]:
     owner_scopes: Final = get_resource_owner_scopes(user_api_key_dict)
     if not owner_scopes:
-        return set()
+        return frozenset()
 
     cache_key: Final = _allowed_container_ids_cache_key(owner_scopes)
     cached: Final = _ALLOWED_CONTAINER_IDS_CACHE.get_cache(cache_key)
     if cached is not None:
-        return set(cached)
+        return frozenset(cached)
 
     prisma_client: Final = await _get_prisma_client()
     if prisma_client is None:
-        return set()
+        return frozenset()
 
     table: Final = ManagedObjectRepository(prisma_client).table
     rows: Final[Sequence[prisma_models.LiteLLM_ManagedObjectTable]] = await table.find_many(
@@ -384,34 +387,69 @@ async def _get_allowed_container_ids(
             "created_by": {"in": owner_scopes},
         }
     )
-    allowed_ids: Final = {row.model_object_id for row in rows if getattr(row, "model_object_id", None) is not None}
-    # ``InMemoryCache.get_cache`` attempts ``json.loads`` on the stored
-    # value; passing a set would round-trip through that path
-    # unnecessarily. Store as a list and rehydrate above.
-    _ALLOWED_CONTAINER_IDS_CACHE.set_cache(cache_key, list(allowed_ids))
+    allowed_ids: Final = frozenset(
+        row.model_object_id for row in rows if getattr(row, "model_object_id", None) is not None
+    )
+    _ALLOWED_CONTAINER_IDS_CACHE.set_cache(cache_key, tuple(allowed_ids))
     return allowed_ids
 
 
-async def filter_container_list_response(
-    response: object,
+def _is_owned_container(item: object, allowed_container_ids: AbstractSet[str], custom_llm_provider: str) -> bool:
+    container_id: Final = _get_response_id(item)
+    if container_id is None:
+        return False
+    original_container_id, resolved_provider = decode_container_id_for_ownership(container_id, custom_llm_provider)
+    return _container_model_object_id(original_container_id, resolved_provider) in allowed_container_ids
+
+
+async def _collect_owned_containers(
+    fetch_page: FetchContainerListPage,
+    after: str | None,
+    needed: int,
+    allowed_container_ids: AbstractSet[str],
+    custom_llm_provider: str,
+    pages_left: int,
+    collected: tuple[object, ...],
+) -> tuple[object, tuple[object, ...]]:
+    page: Final = await fetch_page(after, OWNED_CONTAINER_LIST_PAGE_SIZE)
+    page_data: Final = _get_container_list_data(page) or ()
+    owned: Final = collected + tuple(
+        item for item in page_data if _is_owned_container(item, allowed_container_ids, custom_llm_provider)
+    )
+    upstream_last_id: Final = _get_response_id(page_data[-1]) if page_data else None
+    if len(owned) >= needed or upstream_last_id is None or pages_left <= 1 or not _get_has_more(page):
+        return page, owned
+    return await _collect_owned_containers(
+        fetch_page=fetch_page,
+        after=upstream_last_id,
+        needed=needed,
+        allowed_container_ids=allowed_container_ids,
+        custom_llm_provider=custom_llm_provider,
+        pages_left=pages_left - 1,
+        collected=owned,
+    )
+
+
+async def list_owned_containers(
+    fetch_page: FetchContainerListPage,
+    after: str | None,
+    limit: int | None,
     user_api_key_dict: UserAPIKeyAuth,
     custom_llm_provider: str,
 ) -> object:
-    if is_proxy_admin(user_api_key_dict):
-        return response
-
-    data: Final = _get_container_list_data(response)
-    if data is None:
-        return response
-
     allowed_container_ids: Final = await _get_allowed_container_ids(user_api_key_dict)
-    filtered: Final[list[object]] = []
-    for item in data:
-        container_id = _get_response_id(item)
-        if container_id is None:
-            continue
-        original_container_id, resolved_provider = decode_container_id_for_ownership(container_id, custom_llm_provider)
-        if _container_model_object_id(original_container_id, resolved_provider) in allowed_container_ids:
-            filtered.append(item)
-
-    return _set_container_list_data(response, filtered, removed_filtered_items=len(filtered) != len(data))
+    page_limit: Final = limit if limit is not None else DEFAULT_CONTAINER_LIST_LIMIT
+    last_page, owned = await _collect_owned_containers(
+        fetch_page=fetch_page,
+        after=after,
+        needed=page_limit + 1,
+        allowed_container_ids=allowed_container_ids,
+        custom_llm_provider=custom_llm_provider,
+        pages_left=OWNED_CONTAINER_LIST_MAX_PAGES,
+        collected=(),
+    )
+    return _with_container_list_page(
+        last_page,
+        owned[:page_limit],
+        has_more=len(owned) > page_limit or _get_has_more(last_page),
+    )

litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py

@@ -41,7 +41,7 @@ from litellm.llms.anthropic.chat.guardrail_translation.handler import AnthropicM
 from litellm.llms.base_llm.guardrail_translation.utils import (
     effective_scan_only_tool_results_for_guardrail,
 )
-from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM
+from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM, bedrock_bearer_token
 from litellm.llms.custom_httpx.http_handler import (
     get_async_httpx_client,
     httpxSpecialProvider,
@@ -56,7 +56,6 @@ from litellm.proxy.guardrails.anthropic_sse import (
     is_raw_sse_stream,
     model_response_text,
 )
-from litellm.secret_managers.main import get_secret_str
 from litellm.types.guardrails import (
     BedrockChecksConfigModel,
     BedrockGuardrailStreamingParams,
@@ -713,9 +712,7 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM):
     # logic becomes shared across providers.
 
     #### CALL HOOKS - proxy only ####
-    def _load_credentials(
-        self,
-    ):
+    def _load_credentials(self, bearer_token: str | None = None):
         try:
             from botocore.credentials import Credentials
         except ImportError:
@@ -737,17 +734,21 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM):
             aws_region_name=aws_region_name,
         )
 
-        credentials: Final[Credentials] = self.get_credentials(
-            aws_access_key_id=aws_access_key_id,
-            aws_secret_access_key=aws_secret_access_key,
-            aws_session_token=aws_session_token,
-            aws_region_name=aws_region_name,
-            aws_session_name=aws_session_name,
-            aws_profile_name=aws_profile_name,
-            aws_role_name=aws_role_name,
-            aws_web_identity_token=aws_web_identity_token,
-            aws_sts_endpoint=aws_sts_endpoint,
-            aws_external_id=aws_external_id,
+        credentials: Final[Credentials | None] = (
+            None
+            if bearer_token is not None
+            else self.get_credentials(
+                aws_access_key_id=aws_access_key_id,
+                aws_secret_access_key=aws_secret_access_key,
+                aws_session_token=aws_session_token,
+                aws_region_name=aws_region_name,
+                aws_session_name=aws_session_name,
+                aws_profile_name=aws_profile_name,
+                aws_role_name=aws_role_name,
+                aws_web_identity_token=aws_web_identity_token,
+                aws_sts_endpoint=aws_sts_endpoint,
+                aws_external_id=aws_external_id,
+            )
         )
         return credentials, aws_region_name
 
@@ -779,13 +780,9 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM):
         proxy_endpoint_url = f"{proxy_endpoint_url}{request_path}"
         encoded_data: Final = json.dumps(data).encode("utf-8")
 
-        # first check api-key, if none, fall back to sigV4
-        if api_key is not None:
-            aws_bearer_token: str | None = api_key
-        else:
-            aws_bearer_token = get_secret_str("AWS_BEARER_TOKEN_BEDROCK")
+        aws_bearer_token: Final = bedrock_bearer_token(api_key)
 
-        if aws_bearer_token:
+        if aws_bearer_token is not None:
             try:
                 from botocore.awsrequest import AWSRequest
             except ImportError:
@@ -916,7 +913,7 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM):
                 source,
             )
             return BedrockGuardrailResponse()
-        credentials, aws_region_name = self._load_credentials()
+        credentials, aws_region_name = self._load_credentials(bearer_token=bedrock_bearer_token(api_key))
         allow_chunking: Final = not self._content_uses_contextual_grounding(content)
 
         completed_chunk_usages: Final[list[BedrockGuardrailUsage]] = []  # mutable-ok: billed-chunk usage accumulator
@@ -958,7 +955,7 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM):
         self,
         content: Sequence[BedrockContentItem],
         base_request_data: Mapping[str, object],
-        credentials: "Credentials",
+        credentials: "Credentials | None",
         aws_region_name: str,
         api_key: str | None,
         request_data: dict | None,  # mutable-ok: proxy request body dict, mutated by the logging helper
@@ -1096,7 +1093,7 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM):
         self,
         content: Sequence[BedrockContentItem],
         base_request_data: Mapping[str, object],
-        credentials: "Credentials",
+        credentials: "Credentials | None",
         aws_region_name: str,
         api_key: str | None,
         request_data: dict | None,  # mutable-ok: proxy request body dict, mutated by the logging helper
@@ -1146,7 +1143,7 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM):
         self,
         content: Sequence[BedrockContentItem],
         base_request_data: Mapping[str, object],
-        credentials: "Credentials",
+        credentials: "Credentials | None",
         aws_region_name: str,
         api_key: str | None,
         request_data: dict | None,  # mutable-ok: proxy request body dict, mutated by the logging helper
@@ -1873,9 +1870,9 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM):
             # Nothing to scan (e.g. tool-only turn) -> allow, like ApplyGuardrail does.
             return BedrockGuardrailResponse()
 
-        credentials, aws_region_name = self._load_credentials()
-        body: Final[dict[str, object]] = {"messages": checks_messages, "checks": self.checks}
         api_key: Final[str | None] = request_data.get("api_key") if request_data else None
+        credentials, aws_region_name = self._load_credentials(bearer_token=bedrock_bearer_token(api_key))
+        body: Final[dict[str, object]] = {"messages": checks_messages, "checks": self.checks}
 
         prepared_request: Final = self._prepare_request(
             credentials=credentials,

litellm/proxy/guardrails/guardrail_hooks/compresr/compresr.py

@@ -916,9 +916,10 @@ class CompresrGuardrail(CustomGuardrail):
     def _mirror_texts_channel(input_texts: object, applied: _CompressionResult) -> list[object] | None:
         """Compressed content mirrored into the Responses `texts` channel.
 
-        The chat/Anthropic handlers round-trip ``structured_messages``; the
-        Responses translation cannot rebuild its input from chat messages and
-        instead writes back through ``texts``. This matches by value, so a
+        The chat/Anthropic/Responses handlers round-trip
+        ``structured_messages``; translations without that round-trip write
+        back through ``texts``, so the compressed content is mirrored there
+        too. This matches by value, so a
         replacement is applied only when it is unambiguous: one compression per
         text, and every occurrence in ``texts`` accounted for by a compressed
         target. Anything else is left uncompressed rather than risk a wrong or

litellm/proxy/guardrails/guardrail_hooks/headroom/headroom.py

@@ -50,6 +50,9 @@ if TYPE_CHECKING:
     from litellm.types.proxy.guardrails.guardrail_hooks.base import GuardrailConfigModel
 
 BYPASS_HEADER: Final = "x-headroom-bypass"
+_STREAM_CONVERTIBLE_CALL_TYPES: Final = frozenset(
+    (CallTypes.completion, CallTypes.acompletion, CallTypes.responses, CallTypes.aresponses)
+)
 HEADROOM_RETRIEVE_TOOL_NAME: Final = "headroom_retrieve"
 _HASH_PATTERN: Final = re.compile(r"hash=([a-f0-9]{24})")
 _HASH_CACHE_TTL_SECONDS: Final = 15 * 60
@@ -725,6 +728,10 @@ class HeadroomGuardrail(CustomGuardrail):
             verbose_proxy_logger.debug("Headroom: %s header set; skipping compression", BYPASS_HEADER)
             return inputs
 
+        if request_data.get("background"):
+            verbose_proxy_logger.debug("Headroom: background request; skipping compression")
+            return inputs
+
         structured_messages: Final = inputs.get("structured_messages")
         if not _is_object_list(structured_messages) or not structured_messages:
             return inputs
@@ -826,9 +833,9 @@ class HeadroomGuardrail(CustomGuardrail):
     ) -> dict[str, Any] | None:  # mutable-ok: overrides CustomLogger hook whose contract is a plain dict
         base_result: Final = await super().async_pre_call_deployment_hook(kwargs, call_type)
         effective: Final = base_result if base_result is not None else kwargs
-        if call_type not in (CallTypes.completion, CallTypes.acompletion):
+        if call_type not in _STREAM_CONVERTIBLE_CALL_TYPES:
             return base_result
-        if not effective.get("stream"):
+        if not effective.get("stream") or effective.get("background"):
             return base_result
         if not has_headroom_retrieve_tool(effective.get("tools")):
             return base_result

litellm/proxy/guardrails/guardrail_hooks/unified_guardrail/unified_guardrail.py

@@ -121,9 +121,6 @@ def _a2a_jsonrpc_error_chunk(exc: HTTPException, request_id: str | None) -> Mapp
     }
 
 
-endpoint_guardrail_translation_mappings = None
-
-
 def _ensure_litellm_metadata(data: dict, user_api_key_dict: UserAPIKeyAuth) -> None:
     """Populate data['litellm_metadata'] from user_api_key_dict if absent."""
     if "litellm_metadata" not in data:
@@ -164,7 +161,6 @@ class UnifiedLLMGuardrails(CustomLogger):
         Use this if you want to MODIFY the input
         """
 
-        global endpoint_guardrail_translation_mappings
         from litellm.proxy.common_utils.callback_utils import (
             add_guardrail_to_applied_guardrails_header,
         )
@@ -186,18 +182,15 @@ class UnifiedLLMGuardrails(CustomLogger):
             )
             return data
 
-        if endpoint_guardrail_translation_mappings is None:
-            endpoint_guardrail_translation_mappings = load_guardrail_translation_mappings()
+        mappings: Final = load_guardrail_translation_mappings()
 
         try:
-            if CallTypes(call_type) not in endpoint_guardrail_translation_mappings:
+            if CallTypes(call_type) not in mappings:
                 return data
         except ValueError:
             return data  # handle unmapped call types
 
-        endpoint_translation: Final = _as_endpoint_translation(
-            endpoint_guardrail_translation_mappings[CallTypes(call_type)]()
-        )
+        endpoint_translation: Final = _as_endpoint_translation(mappings[CallTypes(call_type)]())
 
         _ensure_litellm_metadata(data, user_api_key_dict)
 
@@ -222,8 +215,6 @@ class UnifiedLLMGuardrails(CustomLogger):
 
         This can NOT modify the input, only used to reject or accept a call before going to LLM API
         """
-        global endpoint_guardrail_translation_mappings
-
         verbose_proxy_logger.debug("Running UnifiedLLMGuardrails moderation hook")
 
         guardrail_to_apply: Final[CustomGuardrail] = data.pop("guardrail_to_apply", None)
@@ -241,14 +232,11 @@ class UnifiedLLMGuardrails(CustomLogger):
             )
             return data
 
-        if endpoint_guardrail_translation_mappings is None:
-            endpoint_guardrail_translation_mappings = load_guardrail_translation_mappings()
-        if call_type is not None and CallTypes(call_type) not in endpoint_guardrail_translation_mappings:
+        mappings: Final = load_guardrail_translation_mappings()
+        if call_type is not None and CallTypes(call_type) not in mappings:
             return data
 
-        endpoint_translation: Final = _as_endpoint_translation(
-            endpoint_guardrail_translation_mappings[CallTypes(call_type)]()
-        )
+        endpoint_translation: Final = _as_endpoint_translation(mappings[CallTypes(call_type)]())
 
         _ensure_litellm_metadata(data, user_api_key_dict)
 
@@ -271,7 +259,6 @@ class UnifiedLLMGuardrails(CustomLogger):
 
         Uses Enkrypt AI guardrails to check the response for policy violations, PII, and injection attacks
         """
-        global endpoint_guardrail_translation_mappings
         # Local import avoids a module-level cyclic import with
         # litellm.integrations.custom_guardrail.
         from litellm.integrations.custom_guardrail import ModifyResponseException
@@ -319,10 +306,9 @@ class UnifiedLLMGuardrails(CustomLogger):
             )
             return response
 
-        if endpoint_guardrail_translation_mappings is None:
-            endpoint_guardrail_translation_mappings = load_guardrail_translation_mappings()
+        mappings: Final = load_guardrail_translation_mappings()
 
-        if CallTypes(call_type) not in endpoint_guardrail_translation_mappings:
+        if CallTypes(call_type) not in mappings:
             verbose_proxy_logger.warning(
                 "Guardrail '%s' selected for route '%s' but call type '%s' has no guardrail translation handler; "
                 "skipping post-call scanning.",
@@ -332,9 +318,7 @@ class UnifiedLLMGuardrails(CustomLogger):
             )
             return response
 
-        endpoint_translation: Final = _as_endpoint_translation(
-            endpoint_guardrail_translation_mappings[CallTypes(call_type)]()
-        )
+        endpoint_translation: Final = _as_endpoint_translation(mappings[CallTypes(call_type)]())
 
         try:
             response = await endpoint_translation.process_output_response(
@@ -906,8 +890,6 @@ class UnifiedLLMGuardrails(CustomLogger):
         sampling_rate=1 means every chunk, sampling_rate=5 means every 5th chunk, etc.
         """
 
-        global endpoint_guardrail_translation_mappings
-
         # Local import avoids a module-level cyclic import with
         # litellm.integrations.custom_guardrail.
         from litellm.integrations.custom_guardrail import ModifyResponseException
@@ -978,9 +960,7 @@ class UnifiedLLMGuardrails(CustomLogger):
                 yield item
             return
 
-        # Initialize translation mappings if needed
-        if endpoint_guardrail_translation_mappings is None:
-            endpoint_guardrail_translation_mappings = load_guardrail_translation_mappings()
+        mappings: Final = load_guardrail_translation_mappings()
 
         # Streaming text transformation (incremental_diff) diverges enough from the
         # block_only path that it runs as its own iterator. It requires a route we
@@ -989,7 +969,7 @@ class UnifiedLLMGuardrails(CustomLogger):
         if streaming_transform_mode == "incremental_diff":
             transform_call_type: Final = self._resolve_transform_call_type(
                 user_api_key_dict=user_api_key_dict,
-                mappings=endpoint_guardrail_translation_mappings,
+                mappings=mappings,
             )
             if transform_call_type is not None:
                 async for transformed_item in self._run_incremental_transform_stream(
@@ -1000,7 +980,7 @@ class UnifiedLLMGuardrails(CustomLogger):
                     call_type=transform_call_type,
                     sampling_rate=sampling_rate,
                     end_of_stream_only=end_of_stream_only,
-                    mappings=endpoint_guardrail_translation_mappings,
+                    mappings=mappings,
                 ):
                     yield transformed_item
                 return
@@ -1037,7 +1017,7 @@ class UnifiedLLMGuardrails(CustomLogger):
                 call_type = _infer_call_type(call_type=None, completion_response=item)
 
             # If call type not supported, just pass through all chunks
-            if call_type is None or CallTypes(call_type) not in endpoint_guardrail_translation_mappings:
+            if call_type is None or CallTypes(call_type) not in mappings:
                 yield item
                 async for remaining_item in response:
                     yield remaining_item
@@ -1049,7 +1029,7 @@ class UnifiedLLMGuardrails(CustomLogger):
             # moderation runs below.
             if end_of_stream_only:
                 if not buffer_until_moderated:
-                    endpoint_translation = endpoint_guardrail_translation_mappings[CallTypes(call_type)]()
+                    endpoint_translation = mappings[CallTypes(call_type)]()
                     stream_has_ended = hasattr(
                         endpoint_translation, "_check_streaming_has_ended"
                     ) and endpoint_translation._check_streaming_has_ended(responses_so_far)
@@ -1063,7 +1043,7 @@ class UnifiedLLMGuardrails(CustomLogger):
 
             # Process chunk based on sampling rate
             if chunk_counter % sampling_rate == 0:
-                endpoint_translation = endpoint_guardrail_translation_mappings[CallTypes(call_type)]()
+                endpoint_translation = mappings[CallTypes(call_type)]()
                 scan_key = endpoint_translation.get_streaming_scan_key(responses_so_far)
                 if _is_redundant_scan(scan_key, last_scan_key):
                     verbose_proxy_logger.debug(
@@ -1143,14 +1123,14 @@ class UnifiedLLMGuardrails(CustomLogger):
                 yield item
 
         # Stream has ended - do final processing with all collected chunks
-        if call_type is not None and CallTypes(call_type) in endpoint_guardrail_translation_mappings:
+        if call_type is not None and CallTypes(call_type) in mappings:
             verbose_proxy_logger.debug(
                 "Processing final streaming response with all %s chunks for guardrail %s",
                 len(responses_so_far),
                 guardrail_to_apply.guardrail_name,
             )
 
-            endpoint_translation = endpoint_guardrail_translation_mappings[CallTypes(call_type)]()
+            endpoint_translation = mappings[CallTypes(call_type)]()
 
             # When buffering, snapshot the original chunks before moderation.
             # A shallow copy suffices: end-of-stream

litellm/proxy/guardrails/guardrail_registry.py

@@ -654,23 +654,16 @@ class InMemoryGuardrailHandler:
         source: Literal["db", "config"] = "db",
     ) -> None:
         """
-        Update a guardrail in memory
-
-        - updates the guardrail in memory
-        - updates the guardrail params in litellm.callback_manager
+        Update a guardrail in memory: a changed name or litellm_params rebuilds the
+        live callback from the new row (fail-closed: an invalid row keeps the
+        previous instance and raises), anything else only refreshes the stored row
         """
-        self.IN_MEMORY_GUARDRAILS[guardrail_id] = guardrail
-        self._sources[guardrail_id] = source
-
-        tracked_callbacks: Final = self._tracked_callbacks(guardrail_id)
-        if not tracked_callbacks:
+        updated_guardrail: Final = cast(Guardrail, {**guardrail, "guardrail_id": guardrail_id})
+        if self._has_guardrail_params_changed(guardrail_id, updated_guardrail):
+            self.reinitialize_guardrail(guardrail=updated_guardrail, source=source)
             return
-        updated_litellm_params: Final = cast(LitellmParams, guardrail.get("litellm_params", {}))
-        tracked_callbacks[0].update_in_memory_litellm_params(litellm_params=updated_litellm_params)
-        for sibling_callback in tracked_callbacks[1:]:
-            sibling_stage = sibling_callback.event_hook
-            sibling_callback.update_in_memory_litellm_params(litellm_params=updated_litellm_params)
-            sibling_callback.event_hook = sibling_stage
+        self.IN_MEMORY_GUARDRAILS[guardrail_id] = updated_guardrail
+        self._sources[guardrail_id] = source
 
     def delete_in_memory_guardrail(self, guardrail_id: str) -> None:
         """

litellm/proxy/hooks/responses_id_security.py

@@ -5,10 +5,11 @@ This hook uses the DBSpendUpdateWriter to batch-write response IDs to the databa
 instead of writing immediately on each request.
 """
 
-from collections.abc import AsyncGenerator
+from collections.abc import AsyncGenerator, Callable, Mapping
 from typing import TYPE_CHECKING, Any, Final, cast
 
 from fastapi import HTTPException
+from pydantic import TypeAdapter, ValidationError
 
 from litellm._logging import verbose_proxy_logger
 from litellm.integrations.custom_logger import CustomLogger
@@ -32,6 +33,44 @@ _RESPONSES_API_PROVIDER_PREFIX: Final = "/openai"
 _RESPONSES_API_CREATE_ROUTES: Final = frozenset({"/v1/responses", "/responses"})
 
 
+_RESPONSE_PAYLOAD_ADAPTER: Final = TypeAdapter(Mapping[str, object])
+
+
+def _response_payload(response_obj: object) -> Mapping[str, object] | None:
+    try:
+        return _RESPONSE_PAYLOAD_ADAPTER.validate_python(response_obj)
+    except ValidationError:
+        return None
+
+
+def _rewrite_advertised_id(
+    event: BaseLiteLLMOpenAIResponseObject,
+    rewrite: Callable[[str], str],
+) -> BaseLiteLLMOpenAIResponseObject:
+    event_id: Final = getattr(event, "id", None)
+    if isinstance(event_id, str) and event_id.startswith("resp_"):
+        setattr(event, "id", rewrite(event_id))
+        return event
+
+    nested: Final = getattr(event, "response", None)
+    if isinstance(nested, ResponsesAPIResponse):
+        setattr(nested, "id", rewrite(nested.id))
+        setattr(event, "response", nested)
+        return event
+
+    payload: Final = _response_payload(nested)
+    if payload is None:
+        return event
+
+    payload_id: Final = payload.get("id")
+    if not isinstance(payload_id, str):
+        return event
+
+    rewritten: Final = {**payload, "id": rewrite(payload_id)}  # mutable-ok: pydantic cannot serialize a frozen map
+    setattr(event, "response", rewritten)
+    return event
+
+
 def _is_responses_api_create_route(request_route: str | None) -> bool:
     if request_route is None:
         return False
@@ -196,10 +235,6 @@ class ResponsesIDSecurity(CustomLogger):
         user_api_key_dict: "UserAPIKeyAuth",
         request_cache: dict[str, str] | None = None,
     ) -> BaseLiteLLMOpenAIResponseObject:
-        # encrypt the response id using the symmetric key
-        # encrypt the response id, and encode the user id and response id in base64
-
-        # Check if signing key is available
         signing_key: Final = self._get_signing_key()
         if signing_key is None:
             verbose_proxy_logger.debug(
@@ -210,43 +245,22 @@ class ResponsesIDSecurity(CustomLogger):
             )
             return response
 
-        response_id: Final = getattr(response, "id", None)
-        response_obj: Final = getattr(response, "response", None)
+        def encrypt(original_id: str) -> str:
+            cached: Final = request_cache.get(original_id) if request_cache is not None else None
+            if cached is not None:
+                return cached
 
-        if response_id and isinstance(response_id, str) and response_id.startswith("resp_"):
-            # Check request-scoped cache first (for streaming consistency)
-            if request_cache is not None and response_id in request_cache:
-                setattr(response, "id", request_cache[response_id])
-            else:
-                encrypted_response_id = SpecialEnums.LITELLM_MANAGED_RESPONSE_API_RESPONSE_ID_COMPLETE_STR.value.format(
-                    response_id,
-                    user_api_key_dict.user_id or "",
-                    user_api_key_dict.team_id or "",
-                )
+            managed_id: Final = SpecialEnums.LITELLM_MANAGED_RESPONSE_API_RESPONSE_ID_COMPLETE_STR.value.format(
+                original_id,
+                user_api_key_dict.user_id or "",
+                user_api_key_dict.team_id or "",
+            )
+            encrypted_id: Final = f"resp_{encrypt_value_helper(value=managed_id)}"
+            if request_cache is not None:
+                request_cache[original_id] = encrypted_id
+            return encrypted_id
 
-                encoded_user_id_and_response_id = encrypt_value_helper(value=encrypted_response_id)
-                encrypted_id = f"resp_{encoded_user_id_and_response_id}"
-                if request_cache is not None:
-                    request_cache[response_id] = encrypted_id
-                setattr(response, "id", encrypted_id)
-
-        elif response_obj and isinstance(response_obj, ResponsesAPIResponse):
-            # Check request-scoped cache first (for streaming consistency)
-            if request_cache is not None and response_obj.id in request_cache:
-                setattr(response_obj, "id", request_cache[response_obj.id])
-            else:
-                encrypted_response_id = SpecialEnums.LITELLM_MANAGED_RESPONSE_API_RESPONSE_ID_COMPLETE_STR.value.format(
-                    response_obj.id,
-                    user_api_key_dict.user_id or "",
-                    user_api_key_dict.team_id or "",
-                )
-                encoded_user_id_and_response_id = encrypt_value_helper(value=encrypted_response_id)
-                encrypted_id = f"resp_{encoded_user_id_and_response_id}"
-                if request_cache is not None:
-                    request_cache[response_obj.id] = encrypted_id
-                setattr(response_obj, "id", encrypted_id)
-            setattr(response, "response", response_obj)
-        return response
+        return _rewrite_advertised_id(response, encrypt)
 
     async def async_post_call_success_hook(
         self,

litellm/proxy/image_endpoints/endpoints.py

@@ -2,7 +2,7 @@ import asyncio
 import io
 import traceback
 from collections.abc import Sequence
-from typing import Final
+from typing import Final, get_type_hints
 
 import orjson
 from fastapi import APIRouter, Depends, File, HTTPException, Request, Response, UploadFile, status
@@ -16,11 +16,18 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import (
 from litellm.proxy._types import *
 from litellm.proxy.auth.user_api_key_auth import UserAPIKeyAuth, user_api_key_auth
 from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing
+from litellm.proxy.common_utils.http_parsing_utils import (
+    coerce_numeric_form_fields,
+    numeric_form_fields,
+)
 from litellm.proxy.route_llm_request import route_request
+from litellm.types.images.main import ImageEditRequestParams
 from litellm.types.llms.openai import ChatCompletionUserMessage
 
 router: Final = APIRouter()
 
+IMAGE_EDIT_NUMERIC_FORM_FIELDS: Final = numeric_form_fields(get_type_hints(ImageEditRequestParams))
+
 
 async def uploadfile_to_bytesio(upload: UploadFile) -> io.BytesIO:
     """
@@ -279,7 +286,12 @@ async def image_edit_api(
     #########################################################
     # Read request body and convert UploadFiles to BytesIO
     #########################################################
-    data: Final = await _read_request_body(request=request)
+    data: Final = dict(
+        coerce_numeric_form_fields(
+            parsed_body=await _read_request_body(request=request),
+            numeric_fields=IMAGE_EDIT_NUMERIC_FORM_FIELDS,
+        )
+    )
     image_files: Final = await batch_to_bytesio(image)
     mask_files: Final = await batch_to_bytesio(mask)
     if image_files:

litellm/proxy/management_endpoints/access_group_endpoints.py

@@ -1,4 +1,5 @@
 from collections.abc import Mapping, Sequence
+from types import MappingProxyType
 from typing import Final, Protocol
 
 from fastapi import APIRouter, Depends, HTTPException, status
@@ -20,7 +21,7 @@ from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
 from litellm.proxy.db.exception_handler import PrismaDBExceptionHandler
 from litellm.proxy.management_helpers.access_group_team_sync import invalidate_access_group_cache
 from litellm.proxy.utils import get_prisma_client_or_throw
-from litellm.repositories.table_repositories import AccessGroupRepository
+from litellm.repositories.table_repositories import AccessGroupRepository, TeamRepository
 from litellm.types.access_group import (
     AccessGroupCreateRequest,
     AccessGroupResponse,
@@ -74,11 +75,11 @@ class _AccessGroupTable(Protocol):
 
 
 class _TeamTable(Protocol):
-    async def find_unique(self, where: Mapping[str, object]) -> _TeamRecord | None: ...
+    async def find_unique(self, *, where: Mapping[str, object]) -> _TeamRecord | None: ...
 
-    async def find_many(self, where: Mapping[str, object]) -> Sequence[_TeamRecord]: ...
+    async def find_many(self, *, where: Mapping[str, object]) -> Sequence[_TeamRecord]: ...
 
-    async def update(self, where: Mapping[str, object], data: Mapping[str, object]) -> object: ...
+    async def update(self, *, where: Mapping[str, object], data: Mapping[str, object]) -> object: ...
 
 
 class _KeyTable(Protocol):
@@ -119,8 +120,56 @@ def _require_admin_view(user_api_key_dict: UserAPIKeyAuth) -> None:
         )
 
 
-def _record_to_response(record: _AccessGroupRecord) -> AccessGroupResponse:
-    return AccessGroupResponse.model_validate(record.dict())
+def _record_to_response(
+    record: _AccessGroupRecord, *, assigned_team_ids: Sequence[str] | None = None
+) -> AccessGroupResponse:
+    stored: Final = record.dict()
+    payload: Final = (
+        stored if assigned_team_ids is None else MappingProxyType({**stored, "assigned_team_ids": assigned_team_ids})
+    )
+    return AccessGroupResponse.model_validate(payload)
+
+
+def _attached_team_ids_by_group(
+    records: Sequence[_AccessGroupRecord], teams: Sequence[_TeamRecord]
+) -> Mapping[str, tuple[str, ...]]:
+    """Teams really attached to each group: the stored column minus ghosts, plus teams the mirror missed."""
+    real_team_ids: Final = frozenset(team.team_id for team in teams)
+
+    def attached(record: _AccessGroupRecord) -> tuple[str, ...]:
+        stored: Final = (team_id for team_id in (record.assigned_team_ids or ()) if team_id in real_team_ids)
+        carrying: Final = (team.team_id for team in teams if record.access_group_id in (team.access_group_ids or ()))
+        return tuple(dict.fromkeys((*stored, *carrying)))
+
+    return MappingProxyType({record.access_group_id: attached(record) for record in records})
+
+
+async def _attached_team_ids_for(
+    team_table: _TeamTable, records: Sequence[_AccessGroupRecord]
+) -> Mapping[str, tuple[str, ...]]:
+    if not records:
+        return MappingProxyType({})
+    group_ids: Final = tuple(record.access_group_id for record in records)
+    stored_team_ids: Final = tuple(
+        dict.fromkeys(team_id for record in records for team_id in (record.assigned_team_ids or ()))
+    )
+    carrying: Final = {"access_group_ids": {"hasSome": group_ids}}  # mutable-ok: prisma where is a dict
+    listed: Final = {"team_id": {"in": stored_team_ids}}  # mutable-ok: prisma where is a dict
+    teams: Final = await team_table.find_many(where={"OR": (carrying, listed)})  # mutable-ok: prisma where is a dict
+    return _attached_team_ids_by_group(records, teams)
+
+
+async def _require_teams_exist(tx: _AccessGroupTx, team_ids: Sequence[str]) -> None:
+    if not team_ids:
+        return
+    where: Final = {"team_id": {"in": team_ids}}  # mutable-ok: prisma where is a dict
+    found: Final = await tx.litellm_teamtable.find_many(where=where)
+    missing: Final = frozenset(team_ids) - frozenset(team.team_id for team in found)
+    if missing:
+        raise HTTPException(
+            status_code=status.HTTP_400_BAD_REQUEST,
+            detail=f"Unknown team ids: {', '.join(sorted(missing))}",
+        )
 
 
 def _record_to_access_group_table(record: _AccessGroupRecord) -> LiteLLM_AccessGroupTable:
@@ -330,6 +379,7 @@ async def create_access_group(
                     status_code=status.HTTP_409_CONFLICT,
                     detail=f"Access group '{data.access_group_name}' already exists",
                 )
+            await _require_teams_exist(tx, data.assigned_team_ids or ())
 
             record: Final = await tx.litellm_accessgrouptable.create(
                 data={
@@ -390,7 +440,8 @@ async def list_access_groups(
 
     table: Final = AccessGroupRepository(prisma_client).table
     records: Final = await table.find_many(order={"created_at": "desc"})
-    return [_record_to_response(r) for r in records]
+    attached: Final = await _attached_team_ids_for(TeamRepository(prisma_client).table, records)
+    return [_record_to_response(r, assigned_team_ids=attached[r.access_group_id]) for r in records]
 
 
 @router.get(
@@ -411,7 +462,8 @@ async def get_access_group(
             status_code=status.HTTP_404_NOT_FOUND,
             detail=f"Access group '{access_group_id}' not found",
         )
-    return _record_to_response(record)
+    attached: Final = await _attached_team_ids_for(TeamRepository(prisma_client).table, (record,))
+    return _record_to_response(record, assigned_team_ids=attached[record.access_group_id])
 
 
 @router.put(
@@ -461,8 +513,10 @@ async def update_access_group(
                     status_code=status.HTTP_404_NOT_FOUND,
                     detail=f"Access group '{access_group_id}' not found",
                 )
+            await _require_teams_exist(tx, data.assigned_team_ids or ())
 
-            old_team_ids: Final[set[str]] = set(existing.assigned_team_ids or [])
+            attached: Final = await _attached_team_ids_for(tx.litellm_teamtable, (existing,))
+            old_team_ids: Final[set[str]] = set(attached[access_group_id])
             old_key_ids: Final[set[str]] = set(existing.assigned_key_ids or [])
             new_team_ids: Final[set[str]] = (
                 set(update_fields["assigned_team_ids"] or []) if "assigned_team_ids" in update_fields else old_team_ids

litellm/proxy/management_endpoints/internal_user_endpoints.py

@@ -17,6 +17,7 @@ import json
 import traceback
 from collections.abc import Awaitable, Mapping, Sequence
 from datetime import datetime, timezone
+from types import MappingProxyType
 from typing import Any, Final, Literal, Protocol, cast, overload
 
 import fastapi
@@ -77,6 +78,7 @@ from litellm.types.proxy.management_endpoints.internal_user_endpoints import (
     BulkUpdateUserRequest,
     BulkUpdateUserResponse,
     UserListResponse,
+    UserSearchWhere,
     UserUpdateResult,
 )
 from litellm.types.proxy.management_endpoints.scim_v2 import (
@@ -2080,6 +2082,22 @@ async def _authorize_user_list_request(
     return ",".join(allowed_org_ids)
 
 
+_NO_SEARCH_WHERE: Final[Mapping[str, object]] = MappingProxyType({})
+
+
+def _user_search_where(search: str | None) -> Mapping[str, object]:
+    """Prisma predicate for `/user/list?search=`: user_id or user_email contains it, case-insensitive."""
+    if not search:
+        return _NO_SEARCH_WHERE
+    search_where: Final[UserSearchWhere] = {
+        "OR": (
+            {"user_id": {"contains": search, "mode": "insensitive"}},
+            {"user_email": {"contains": search, "mode": "insensitive"}},
+        )
+    }
+    return search_where
+
+
 @router.get(
     "/user/list",
     tags=["Internal User management"],
@@ -2091,6 +2109,10 @@ async def get_users(
     user_ids: str | None = fastapi.Query(default=None, description="Get list of users by user_ids"),
     sso_user_ids: str | None = fastapi.Query(default=None, description="Get list of users by sso_user_id"),
     user_email: str | None = fastapi.Query(default=None, description="Filter users by partial email match"),
+    search: str | None = fastapi.Query(
+        default=None,
+        description="Combined search: matches users whose 'user_id' or 'user_email' contains the value (case-insensitive).",
+    ),
     team: str | None = fastapi.Query(default=None, description="Filter users by team id"),
     page: int = fastapi.Query(default=1, ge=1, description="Page number"),
     page_size: int = fastapi.Query(default=25, ge=1, le=100, description="Number of items per page"),
@@ -2121,6 +2143,8 @@ async def get_users(
             Get list of users by sso_ids. Comma separated list of sso_ids.
         user_email: Optional[str]
             Filter users by partial email match
+        search: Optional[str]
+            Combined search: matches users whose user_id or user_email contains the value (case-insensitive)
         team: Optional[str]
             Filter users by team id. Will match if user has this team in their teams array.
         page: int
@@ -2197,7 +2221,11 @@ async def get_users(
             where_conditions["organization_memberships"] = {"some": {"organization_id": {"in": org_id_list}}}
 
     ## Filter any none fastapi.Query params - e.g. where_conditions: {'user_email': {'contains': Query(None), 'mode': 'insensitive'}, 'teams': {'has': Query(None)}}
-    where_conditions = {k: v for k, v in where_conditions.items() if v is not None}
+    where: Final[Mapping[str, object]] = {
+        key: value
+        for key, value in (*where_conditions.items(), *_user_search_where(search).items())
+        if value is not None
+    }
 
     # Build order_by conditions
 
@@ -2206,14 +2234,14 @@ async def get_users(
     )
 
     users: Final[Sequence[prisma_models.LiteLLM_UserTable]] = await UserRepository(prisma_client).table.find_many(
-        where=where_conditions,
+        where=where,
         skip=skip,
         take=page_size,
         order=(order_by if order_by else {"created_at": "desc"}),  # Default to created_at desc if no sort specified
     )
 
     # Get total count of user rows
-    total_count: Final[int] = await UserRepository(prisma_client).table.count(where=where_conditions)
+    total_count: Final[int] = await UserRepository(prisma_client).table.count(where=where)
 
     # Get key count for each user
     user_key_counts: Final = await get_user_key_counts(prisma_client, [user.user_id for user in users])

litellm/proxy/management_endpoints/key_management_endpoints.py

@@ -147,6 +147,7 @@ from litellm.types.proxy.management_endpoints.key_management_endpoints import (
     BulkUpdateKeyResponse,
     BulkUpdateTeamKeysRequest,
     FailedKeyUpdate,
+    KeySearchWhere,
     SuccessfulKeyUpdate,
 )
 from litellm.types.router import Deployment
@@ -5800,6 +5801,10 @@ async def list_keys(
         None,
         description="Filter keys by key alias. Exact match by default; set substring_matching=true (admin only) for case-insensitive substring matching.",
     ),
+    search: str | None = Query(
+        None,
+        description="Combined search: matches keys whose token (key hash) equals the value OR whose key_alias contains it (case-insensitive).",
+    ),
     return_full_object: bool = Query(False, description="Return full key object"),
     include_team_keys: bool = Query(False, description="Include all keys for teams that user is an admin of."),
     include_created_by_keys: bool = Query(False, description="Include keys created by the user"),
@@ -5943,6 +5948,7 @@ async def list_keys(
             agent_id=agent_id,
             use_substring_matching=use_substring_matching,
             expires_filter=expires if isinstance(expires, str) else None,
+            search=search,
         )
 
         verbose_proxy_logger.debug("Successfully prepared response")
@@ -6162,6 +6168,16 @@ def _build_expires_where_clause(expires_filter: str, now: datetime) -> dict[str,
     return {"OR": [{"expires": None}, {"expires": {"gte": now}}]}
 
 
+def _build_key_search_where(search: str) -> KeySearchWhere:
+    search_where: Final[KeySearchWhere] = {
+        "OR": (
+            {"token": search},
+            {"key_alias": {"contains": search, "mode": "insensitive"}},
+        )
+    }
+    return search_where
+
+
 def _build_key_filter_conditions(
     user_id: str | None,
     team_id: str | None,
@@ -6177,6 +6193,7 @@ def _build_key_filter_conditions(
     agent_id: str | None = None,
     use_substring_matching: bool = False,
     expires_filter: str | None = None,
+    search: str | None = None,
 ) -> Mapping[str, object]:
     """Build filter conditions for key listing.
 
@@ -6266,7 +6283,7 @@ def _build_key_filter_conditions(
 
     # Apply team_id, project_id and access_group_id as global AND filters so they
     # narrow results across all visibility conditions (own keys, team keys, etc.)
-    global_filters: Final[tuple[dict[str, object], ...]] = (
+    global_filters: Final[tuple[Mapping[str, object], ...]] = (
         *(
             (
                 {"key_alias": {"contains": key_alias, "mode": "insensitive"}}
@@ -6277,6 +6294,7 @@ def _build_key_filter_conditions(
             else ()
         ),
         *(({"token": key_hash},) if key_hash and isinstance(key_hash, str) else ()),
+        *((_build_key_search_where(search),) if isinstance(search, str) and search else ()),
         *(({"team_id": team_id},) if team_id and isinstance(team_id, str) else ()),
         *(({"project_id": project_id},) if project_id else ()),
         *(({"access_group_ids": {"hasSome": [access_group_id]}},) if access_group_id else ()),
@@ -6316,6 +6334,7 @@ async def _list_key_helper(
     agent_id: str | None = None,
     use_substring_matching: bool = False,
     expires_filter: str | None = None,
+    search: str | None = None,
 ) -> KeyListResponseObject:
     """
     Helper function to list keys
@@ -6354,6 +6373,7 @@ async def _list_key_helper(
         agent_id=agent_id,
         use_substring_matching=use_substring_matching,
         expires_filter=expires_filter,
+        search=search,
     )
 
     # Calculate skip for pagination

litellm/proxy/management_endpoints/model_management_endpoints.py

@@ -13,10 +13,11 @@ model/{model_id}/update - PATCH endpoint for model update.
 import asyncio
 import datetime
 import json
-from collections.abc import Awaitable, Mapping, Sequence
+from collections.abc import AsyncGenerator, Awaitable, Callable, Mapping, Sequence
+from contextlib import AbstractAsyncContextManager, asynccontextmanager
 from json import JSONDecodeError
 from types import MappingProxyType
-from typing import TYPE_CHECKING, Annotated, Final, Literal, Protocol, cast
+from typing import TYPE_CHECKING, Annotated, Final, Literal, Protocol, TypeVar, cast
 
 from fastapi import APIRouter, Depends, Header, HTTPException, Request, status
 from pydantic import BaseModel, ConfigDict, Field, ValidationError, field_validator
@@ -49,6 +50,7 @@ from litellm.proxy._types import (
     TeamModelDeleteRequest,
     UserAPIKeyAuth,
 )
+from litellm.proxy.auth.litellm_license import HEURISTIC_V2_LICENSE_REMEDY
 from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
 from litellm.proxy.common_utils.config_sync_pubsub import (
     coordination_redis_cache,
@@ -96,6 +98,9 @@ from litellm.router_strategy.complexity_router import (
 from litellm.router_utils.auto_router_model_naming import (
     STRATEGY_ROUTER_PARAM_FIELDS,
     carries_complexity_router_settings,
+    count_heuristic_v2_routers,
+    heuristic_v2_limit_violation,
+    uses_heuristic_v2_classifier,
     validate_complexity_router_config_placement,
     validate_complexity_router_config_write,
     validate_strategy_router_model_write,
@@ -153,6 +158,8 @@ class _ProxyModelTable(Protocol):
 
     def find_many(self, *, where: Mapping[str, object]) -> Awaitable[Sequence[_ProxyModelRow]]: ...
 
+    def create(self, *, data: Mapping[str, object]) -> Awaitable[_ProxyModelRow]: ...
+
     def update(
         self, *, where: Mapping[str, object], data: Mapping[str, object]
     ) -> Awaitable[_ProxyModelRow | None]: ...
@@ -166,6 +173,9 @@ class _TxModelTables(Protocol):
     litellm_proxymodeltable: _ProxyModelTable
 
 
+_RowT = TypeVar("_RowT")
+
+
 class _ExistingModelRow(Protocol):
     @property
     def litellm_params(self) -> Mapping[str, object]: ...
@@ -269,6 +279,66 @@ def _raise_on_strategy_router_write_violation(
     )
 
 
+HEURISTIC_V2_SLOT_LOCK_KEY: Final = 5_872_301
+_HEURISTIC_V2_LOCK_SQL: Final = "SELECT 1 AS locked FROM pg_advisory_xact_lock($1)"
+_HEURISTIC_V2_DB_ROWS_SQL: Final = """
+SELECT count(*)::int AS held FROM "LiteLLM_ProxyModelTable"
+WHERE model_id <> $1
+  AND (CASE jsonb_typeof(litellm_params) WHEN 'string' THEN (litellm_params #>> '{}')::jsonb ELSE litellm_params END)
+      -> 'complexity_router_config' ->> 'classifier_type' = 'heuristic_v2'
+"""
+
+
+def _effective_complexity_router_config(
+    incoming_params: GenericLiteLLMParams | None, existing_params: GenericLiteLLMParams | None
+) -> object:
+    """The complexity config a write leaves on the row: the incoming one when the write carries it, else the stored one."""
+    incoming: Final = None if incoming_params is None else incoming_params.complexity_router_config
+    if incoming is not None or existing_params is None:
+        return incoming
+    return existing_params.complexity_router_config
+
+
+@asynccontextmanager
+async def _heuristic_v2_slot(
+    prisma_client: PrismaClient, *, effective_config: object, model_id: str | None
+) -> AsyncGenerator[_ProxyModelTable, None]:
+    """Hand out the model table to write through while the row's claim on a heuristic_v2 slot is settled.
+
+    A write that leaves the row on classifier_type heuristic_v2 under a limited license runs
+    inside one transaction that takes an advisory lock in its own statement before counting
+    (a statement's snapshot predates anything it locks), so pods cannot both pass the count:
+    the DB rows (any pod, either JSON shape) plus this proxy's config.yaml routers are judged
+    against the license limit and the write is refused with a 403 before it happens. The row
+    being edited keeps its own slot through ``model_id``. Every other write, and every write on
+    an unlimited license, goes through the repository table with no lock. Only the row write
+    itself may run inside: anything that needs a second connection (the team model bookkeeping)
+    must wait until the transaction has committed and the lock is released. The transaction
+    writes bypass the repository's publish-on-write, so the config change is published once
+    after commit, the way delete_team_models does.
+    """
+    from litellm.proxy.proxy_server import _license_check, llm_router
+
+    limit: Final = _license_check.heuristic_v2_router_limit()
+    if limit is None or not uses_heuristic_v2_classifier(effective_config):
+        yield _proxy_model_table(prisma_client)
+        return
+    async with prisma_client.db.tx() as tx_ctx:
+        tables: Final[_TxModelTables] = tx_ctx
+        await tx_ctx.query_raw(_HEURISTIC_V2_LOCK_SQL, HEURISTIC_V2_SLOT_LOCK_KEY)
+        rows: Sequence[Mapping[str, object]] = await tx_ctx.query_raw(_HEURISTIC_V2_DB_ROWS_SQL, model_id or "")
+        db_held: Final = rows[0].get("held") if rows else 0
+        config_rows: Final = () if llm_router is None else tuple(llm_router.config_deployments())
+        held: Final = (db_held if isinstance(db_held, int) else 0) + count_heuristic_v2_routers(config_rows)
+        violation: Final = heuristic_v2_limit_violation(held=held + 1, limit=limit)
+        if violation is not None:
+            raise HTTPException(
+                status_code=status.HTTP_403_FORBIDDEN, detail=f"{violation} {HEURISTIC_V2_LICENSE_REMEDY}"
+            )
+        yield tables.litellm_proxymodeltable
+    await publish_config_change(redis_cache=coordination_redis_cache(), object_type="litellm_proxymodeltable")
+
+
 ENFORCE_RPM_TPM_ON_MODEL_ADD_SETTING: Final = "enforce_rpm_tpm_on_model_add"
 _REQUIRED_RATE_LIMIT_FIELDS: Final = ("rpm", "tpm")
 
@@ -720,22 +790,29 @@ async def patch_model(
         )
 
         requested_model_name: Final = patch_data.model_name
+        stored_model_name: str | None = None
+
+        async def write_row(update_data: PrismaCompatibleUpdateDBModel) -> _ProxyModelRow | None:
+            nonlocal stored_model_name
+            stored_model_name = update_data.get("model_name")
+            update_data["updated_by"] = user_api_key_dict.user_id or litellm_proxy_admin_name
+            update_data["updated_at"] = cast(str, get_utc_datetime())
+            async with _heuristic_v2_slot(
+                prisma_client,
+                effective_config=_effective_complexity_router_config(
+                    patch_data.litellm_params, db_model.litellm_params
+                ),
+                model_id=model_id,
+            ) as table:
+                return await table.update(where={"model_id": model_id}, data=update_data)
+
         # Handle team model updates with proper alias management
-        update_data: Final = await _update_team_model_in_db(
+        updated_model: Final = await _update_team_model_in_db(
             db_model=db_model,
             patch_data=patch_data,
             user_api_key_dict=user_api_key_dict,
             prisma_client=prisma_client,
-        )
-
-        # Add metadata about update
-        update_data["updated_by"] = user_api_key_dict.user_id or litellm_proxy_admin_name
-        update_data["updated_at"] = cast(str, get_utc_datetime())
-
-        # Perform partial update
-        updated_model: Final = await _proxy_model_table(prisma_client).update(
-            where={"model_id": model_id},
-            data=update_data,
+            write_row=write_row,
         )
 
         if updated_model is None:
@@ -746,7 +823,6 @@ async def patch_model(
                 param=None,
             )
 
-        stored_model_name: Final = update_data.get("model_name")
         if (
             stored_model_name is not None
             and stored_model_name == requested_model_name
@@ -980,7 +1056,8 @@ async def _add_model_to_db(
     prisma_client: PrismaClient,
     new_encryption_key: str | None = None,
     should_create_model_in_db: bool = True,
-) -> "prisma_models.LiteLLM_ProxyModelTable | LiteLLM_ProxyModelTable | None":
+    slot: AbstractAsyncContextManager[_ProxyModelTable] | None = None,
+) -> "_ProxyModelRow | LiteLLM_ProxyModelTable":
     # encrypt litellm params #
     _litellm_params_dict: Final = model_params.litellm_params.dict(exclude_none=True)
     _original_litellm_model_name: Final = model_params.litellm_params.model
@@ -998,18 +1075,20 @@ async def _add_model_to_db(
     if model_params.model_info.id is not None:
         _data["model_id"] = model_params.model_info.id
     _create_data: Final = cast("Mapping[str, object]", _data)  # cast-ok: str-keyed json payload built just above
-    if should_create_model_in_db:
-        model_response = await ModelRepository(prisma_client).table.create(data=_create_data)
-    else:
-        model_response = LiteLLM_ProxyModelTable(**_data)
-    return model_response
+    if not should_create_model_in_db:
+        return LiteLLM_ProxyModelTable(**_data)
+    if slot is None:
+        return await _proxy_model_table(prisma_client).create(data=_create_data)
+    async with slot as table:
+        return await table.create(data=_create_data)
 
 
 async def _add_team_model_to_db(
     model_params: Deployment,
     user_api_key_dict: UserAPIKeyAuth,
     prisma_client: PrismaClient,
-) -> "prisma_models.LiteLLM_ProxyModelTable | LiteLLM_ProxyModelTable | None":
+    slot: AbstractAsyncContextManager[_ProxyModelTable] | None = None,
+) -> "_ProxyModelRow | LiteLLM_ProxyModelTable":
     """
     If 'team_id' is provided,
 
@@ -1040,6 +1119,7 @@ async def _add_team_model_to_db(
         model_params=model_params,
         user_api_key_dict=user_api_key_dict,
         prisma_client=prisma_client,
+        slot=slot,
     )
 
     if original_model_name:
@@ -1060,7 +1140,8 @@ async def _update_team_model_in_db(
     patch_data: updateDeployment,
     user_api_key_dict: UserAPIKeyAuth,
     prisma_client: PrismaClient,
-) -> PrismaCompatibleUpdateDBModel:
+    write_row: Callable[[PrismaCompatibleUpdateDBModel], Awaitable[_RowT]],
+) -> _RowT:
     """
     Handle team model updates with proper alias management.
 
@@ -1068,6 +1149,9 @@ async def _update_team_model_in_db(
     - Creates unique internal model_name and team alias
     - Adds model to team object
     - Preserves team_public_model_name for external reference
+
+    The row is written through ``write_row`` before the team's model list is touched, so a
+    refused or failed write leaves the team as it was (the create path orders itself the same way).
     """
     # Validate team_id if present in patch_data
     from litellm.proxy.proxy_server import premium_user
@@ -1079,9 +1163,7 @@ async def _update_team_model_in_db(
         premium_user=premium_user,
     )
 
-    # Validated before any write, beside the premium check the create path already runs
-    # here. The team ACL is updated below and autocommits, so a validator that raises
-    # further down would leave the team mutated and the deployment row never written.
+    # Validated before the row write, beside the premium check the create path already runs here.
     #
     # The merged view is what gets stored, so that is what has to satisfy the invariants.
     # Validating the patch alone rejected a partial edit of an already valid deployment:
@@ -1101,7 +1183,7 @@ async def _update_team_model_in_db(
 
     # No team_id in patch, proceed with standard update
     if patch_team_id is None:
-        return update_db_model(db_model=db_model, updated_patch=patch_data)
+        return await write_row(update_db_model(db_model=db_model, updated_patch=patch_data))
 
     # Determine public model name
     public_model_name: Final = _get_public_model_name(
@@ -1120,11 +1202,14 @@ async def _update_team_model_in_db(
     db_team_id: Final = db_model.model_info.team_id if db_model.model_info else None
     is_new_team_assignment: Final = db_team_id != patch_team_id
 
+    # Team rows keep their internal UUID-based model_name; the public name lives in model_info
+    patch_data.model_name = f"model_name_{patch_team_id}_{uuid.uuid4()}" if is_new_team_assignment else None
+    row: Final = await write_row(update_db_model(db_model=db_model, updated_patch=patch_data))
+
     if is_new_team_assignment:
         await _setup_new_team_model_assignment(
             team_id=patch_team_id,
             public_model_name=public_model_name,
-            patch_data=patch_data,
             user_api_key_dict=user_api_key_dict,
         )
     else:
@@ -1132,12 +1217,11 @@ async def _update_team_model_in_db(
             team_id=patch_team_id,
             public_model_name=public_model_name,
             db_model=db_model,
-            patch_data=patch_data,
             user_api_key_dict=user_api_key_dict,
             prisma_client=prisma_client,
         )
 
-    return update_db_model(db_model=db_model, updated_patch=patch_data)
+    return row
 
 
 def _get_public_model_name(
@@ -1189,13 +1273,9 @@ def _get_public_model_name(
 async def _setup_new_team_model_assignment(
     team_id: str,
     public_model_name: str,
-    patch_data: updateDeployment,
     user_api_key_dict: UserAPIKeyAuth,
 ) -> None:
-    """Set up a new team model with unique name and team membership."""
-    unique_model_name: Final = f"model_name_{team_id}_{uuid.uuid4()}"
-    patch_data.model_name = unique_model_name
-
+    """Register a newly team-assigned model's public name on the team."""
     await team_model_add(
         data=TeamModelAddRequest(
             team_id=team_id,
@@ -1385,7 +1465,6 @@ async def _update_existing_team_model_assignment(
     team_id: str,
     public_model_name: str,
     db_model: Deployment,
-    patch_data: updateDeployment,
     user_api_key_dict: UserAPIKeyAuth,
     prisma_client: PrismaClient | None,
 ) -> None:
@@ -1409,9 +1488,6 @@ async def _update_existing_team_model_assignment(
     old_public_name: Final = db_model.model_info.team_public_model_name if db_model.model_info else None
 
     if old_public_name and public_model_name != old_public_name:
-        # Clear user-supplied public name from patch before any early return so the
-        # caller does not overwrite the internal UUID-based model_name in the DB.
-        patch_data.model_name = None
         if prisma_client is None:
             verbose_proxy_logger.warning(
                 "prisma_client not initialized; skipping public name update entirely to avoid orphaned entries"
@@ -1459,10 +1535,6 @@ async def _update_existing_team_model_assignment(
     # else: old_public_name == public_model_name (no rename needed)
     # No team_model_add/delete calls required; public name is already registered
 
-    # Always clear patch_data.model_name to prevent caller from overwriting
-    # the internal UUID-based model_name in the DB with the user-supplied public name
-    patch_data.model_name = None
-
 
 class ModelManagementAuthChecks:
     """
@@ -1878,18 +1950,19 @@ async def add_new_model(
             reload_outcome: ReconcileOutcome = ReconcileOutcome(still_desired=None, live_after=None)
             try:
                 _original_litellm_model_name: Final = model_params.model_name
-                if model_params.model_info.team_id is None:
-                    model_response = await _add_model_to_db(
-                        model_params=priced_model_params,
-                        user_api_key_dict=user_api_key_dict,
-                        prisma_client=prisma_client,
-                    )
-                else:
-                    model_response = await _add_team_model_to_db(
-                        model_params=priced_model_params,
-                        user_api_key_dict=user_api_key_dict,
-                        prisma_client=prisma_client,
-                    )
+                add_model: Final = (
+                    _add_model_to_db if model_params.model_info.team_id is None else _add_team_model_to_db
+                )
+                model_response = await add_model(
+                    model_params=priced_model_params,
+                    user_api_key_dict=user_api_key_dict,
+                    prisma_client=prisma_client,
+                    slot=_heuristic_v2_slot(
+                        prisma_client,
+                        effective_config=priced_model_params.litellm_params.complexity_router_config,
+                        model_id=priced_model_params.model_info.id,
+                    ),
+                )
                 reload_outcome = await proxy_config.add_deployment(
                     prisma_client=prisma_client, proxy_logging_obj=proxy_logging_obj
                 )
@@ -1903,6 +1976,8 @@ async def add_new_model(
                         passed_model_info=priced_model_params.model_info,
                     )
             except Exception as e:
+                if isinstance(e, HTTPException):
+                    raise
                 verbose_proxy_logger.exception("Exception in add_new_model: %s", e)
 
         else:
@@ -2070,10 +2145,17 @@ async def update_model(
                 "updated_by": user_api_key_dict.user_id or LITELLM_PROXY_ADMIN_NAME,
                 **({} if renamed_to is None else {"model_name": renamed_to}),
             }
-            model_response: Final = await _proxy_model_table(prisma_client).update(
-                where={"model_id": _model_id},
-                data=_data,
-            )
+            async with _heuristic_v2_slot(
+                prisma_client,
+                effective_config=_effective_complexity_router_config(
+                    model_params.litellm_params, deployment.litellm_params
+                ),
+                model_id=_model_id,
+            ) as table:
+                model_response: Final = await table.update(
+                    where={"model_id": _model_id},
+                    data=_data,
+                )
             if renamed_to is not None:
                 await sync_access_groups_for_renamed_model(
                     prisma_client=prisma_client,

litellm/proxy/management_endpoints/scim/scim_v2.py

@@ -1123,13 +1123,15 @@ async def _create_user_if_not_exists(user_id: str, created_via: str = "scim_grou
             user_id=user_id,
             user_email=user_id,  # We don't have email from group membership
             user_alias=None,
-            teams=[],  # Teams will be added separately
             metadata={"created_via": created_via},
             auto_create_key=False,
             user_role=default_role,
         )
 
-        created_user: Final = await new_user(data=new_user_request)
+        created_user: Final = await new_user(
+            data=new_user_request,
+            user_api_key_dict=UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN),
+        )
         verbose_proxy_logger.info("Created user %s via %s", user_id, created_via)
         return created_user
 
@@ -1699,7 +1701,7 @@ async def create_user(
             user_id=user_id,
             user_email=user_data["user_email"],
             user_alias=user_data["user_alias"],
-            teams=user_data["teams"],
+            teams=user_data["teams"] or None,
             metadata=metadata,
             auto_create_key=False,
             user_role=resolved_role if admin_group is not None else default_role,
@@ -1717,6 +1719,7 @@ async def create_user(
 
         created_user: Final = await new_user(
             data=new_user_request,
+            user_api_key_dict=UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN),
         )
 
         scim_user: Final = await ScimTransformations.transform_litellm_user_to_scim_user(user=created_user)
@@ -1771,22 +1774,25 @@ async def update_user(
             roles=user_data["roles"],
         )
 
+        # SCIM User.groups is readOnly (RFC 7643 4.1.2): IdPs sync membership via /Groups and send
+        # no groups or `groups: []` on profile PUTs, so empty means unspecified, not "remove from every team"
+        target_teams: Final = user_data["teams"] or existing_user.teams
         await _handle_team_membership_changes(
             user_id=user_id,
-            existing_teams=existing_user.teams or [],
-            new_teams=user_data["teams"],
+            existing_teams=existing_user.teams,
+            new_teams=target_teams,
         )
 
         update_data: Final = {
             "user_email": user_data["user_email"],
             "user_alias": user_data["user_alias"],
             "sso_user_id": user_data["sso_user_id"],
-            "teams": user_data["teams"],
+            "teams": target_teams,
             "metadata": safe_dumps(metadata),
         }
 
         admin_group: Final = await _get_scim_admin_group()
-        if admin_group is not None:
+        if admin_group is not None and user_data["teams"]:
             update_data["user_role"] = _resolve_scim_user_role(
                 user.groups or [], admin_group, _default_scim_user_role()
             )

litellm/proxy/management_endpoints/team_endpoints.py

@@ -10,6 +10,7 @@ All /team management endpoints
 """
 
 import asyncio
+import copy
 import json
 import math
 import traceback
@@ -2189,6 +2190,26 @@ async def update_team(
             if field in updated_kv
         }
 
+        _writes_metadata_backed_field: Final = any(
+            field in updated_kv
+            for field in (
+                *LiteLLM_ManagementEndpoint_MetadataFields,
+                *LiteLLM_ManagementEndpoint_MetadataFields_Premium,
+            )
+        )
+        if isinstance(existing_team_row.metadata, dict):
+            if "metadata" not in updated_kv and (_team_member_fields_in_request or _writes_metadata_backed_field):
+                updated_kv["metadata"] = copy.deepcopy(existing_team_row.metadata)
+            elif isinstance(updated_kv.get("metadata"), dict):
+                updated_kv["metadata"] = {
+                    **updated_kv["metadata"],
+                    **{
+                        key: existing_team_row.metadata[key]
+                        for key in TeamMemberBudgetHandler.SYSTEM_MANAGED_METADATA_KEYS
+                        if key in existing_team_row.metadata
+                    },
+                }
+
         if _team_member_fields_in_request and TeamMemberBudgetHandler.should_create_budget(
             team_member_budget=data.team_member_budget,
             team_member_rpm_limit=data.team_member_rpm_limit,

litellm/proxy/memory/memory_endpoints.py

@@ -22,6 +22,7 @@ from collections.abc import Mapping
 from typing import TYPE_CHECKING, Final
 
 from fastapi import APIRouter, Depends, HTTPException, Query
+from typing_extensions import ReadOnly, TypedDict
 
 from litellm._logging import verbose_proxy_logger
 from litellm.proxy._types import (
@@ -91,6 +92,36 @@ def _visibility_filter(user_api_key_dict: UserAPIKeyAuth) -> Mapping[str, object
     return {"OR": ors}
 
 
+class _StartsWith(TypedDict):
+    startsWith: ReadOnly[str]
+
+
+class _MemoryKeyWhere(TypedDict):
+    key: ReadOnly[str | _StartsWith]
+
+
+class _MemoryIdWhere(TypedDict):
+    memory_id: ReadOnly[str]
+
+
+class _MemorySearchWhere(TypedDict):
+    OR: ReadOnly[tuple[_MemoryKeyWhere, _MemoryIdWhere]]
+
+
+def _key_filter(search: str | None, key_prefix: str | None, key: str | None) -> Mapping[str, object] | None:
+    """`search` matches a key prefix or an exact memory_id; otherwise `key_prefix` wins over `key`."""
+    if search is not None:
+        search_where: Final[_MemorySearchWhere] = {"OR": ({"key": {"startsWith": search}}, {"memory_id": search})}
+        return search_where
+    if key_prefix is not None:
+        prefix_where: Final[_MemoryKeyWhere] = {"key": {"startsWith": key_prefix}}
+        return prefix_where
+    if key is not None:
+        exact_where: Final[_MemoryKeyWhere] = {"key": key}
+        return exact_where
+    return None
+
+
 def _row_to_model(row: "prisma_models.LiteLLM_MemoryTable") -> LiteLLM_MemoryRow:
     return LiteLLM_MemoryRow(
         memory_id=row.memory_id,
@@ -326,6 +357,13 @@ async def list_memory(
             "Mutually exclusive with `key`; if both are provided, `key_prefix` wins."
         ),
     ),
+    search: str | None = Query(
+        None,
+        description=(
+            "Match entries whose key starts with this value or whose memory_id equals it. "
+            "Takes precedence over `key_prefix` and `key` when provided."
+        ),
+    ),
     page: int = Query(1, ge=1),
     page_size: int = Query(50, ge=1, le=500),
     user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),
@@ -333,22 +371,16 @@ async def list_memory(
     """List memory entries visible to the caller."""
     prisma_client: Final = _require_prisma()
 
-    # Build the key filter first (prefix wins if both `key` and `key_prefix`
-    # are passed). Then AND it with the visibility filter via an explicit
-    # top-level "AND" — safer than `dict.update` since future visibility
-    # filters could grow an "OR" key that would clobber this one if merged
-    # by key.
-    key_filter: Final[dict[str, object]] = {}
-    if key_prefix is not None:
-        key_filter["key"] = {"startsWith": key_prefix}
-    elif key is not None:
-        key_filter["key"] = key
+    # AND the key filter with the visibility filter via an explicit top-level
+    # "AND": both sides can carry an "OR" key (`search`, non-admin visibility),
+    # so merging them by key would let one clobber the other and leak rows.
+    key_filter: Final = _key_filter(search=search, key_prefix=key_prefix, key=key)
 
     vis: Final = _visibility_filter(user_api_key_dict)
-    where: Mapping[str, object]
+    where: Mapping[str, object] | None
     if vis is None:
         where = key_filter
-    elif not key_filter:
+    elif key_filter is None:
         where = vis
     else:
         where = {"AND": [key_filter, vis]}

litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py

@@ -1730,6 +1730,16 @@ def get_vertex_ai_allowed_incoming_headers(request: Request) -> dict:
     return headers
 
 
+def _is_vertex_anthropic_count_tokens_route(endpoint: str) -> bool:
+    return endpoint.rsplit("/", 1)[-1].split(":", 1)[0] == "count-tokens"
+
+
+def _upstream_headers_for_vertex_route(endpoint: str, headers: Mapping[str, str]) -> Mapping[str, str]:
+    if not _is_vertex_anthropic_count_tokens_route(endpoint):
+        return headers
+    return MappingProxyType({name: value for name, value in headers.items() if name.lower() != "anthropic-beta"})
+
+
 def get_vertex_pass_through_handler(
     call_type: Literal["discovery", "aiplatform"],  # noqa: UP037  # ruff reports quoted Literal values here
 ) -> BaseVertexAIPassThroughHandler:
@@ -2128,7 +2138,7 @@ async def _base_vertex_proxy_route(
     endpoint_func: Final = create_pass_through_route(
         endpoint=endpoint,
         target=target,
-        custom_headers=headers,
+        custom_headers=_upstream_headers_for_vertex_route(endpoint, headers),
         is_streaming_request=is_streaming_request,
     )  # dynamically construct pass-through endpoint based on incoming path
 

litellm/proxy/pass_through_endpoints/llm_provider_handlers/anthropic_passthrough_logging_handler.py

@@ -107,6 +107,7 @@ class AnthropicPassthroughLoggingHandler:
             start_time=start_time,
             end_time=end_time,
             logging_obj=logging_obj,
+            response_id=optional_str(response_body.get("id")),
         )
 
         return {
@@ -148,8 +149,9 @@ class AnthropicPassthroughLoggingHandler:
         return model
 
     @staticmethod
-    def _extract_model_from_anthropic_chunks(
+    def _extract_message_start_field(
         all_chunks: Sequence[str | bytes],
+        field: str,
     ) -> str | None:
         for raw in all_chunks:
             text = raw.decode("utf-8") if isinstance(raw, bytes) else raw
@@ -163,11 +165,23 @@ class AnthropicPassthroughLoggingHandler:
                 if not isinstance(data, dict):
                     continue
                 if data.get("type") == "message_start":
-                    model = (data.get("message") or {}).get("model")
-                    if model:
-                        return model
+                    value = (data.get("message") or {}).get(field)
+                    if isinstance(value, str) and value:
+                        return value
         return None
 
+    @staticmethod
+    def _extract_model_from_anthropic_chunks(
+        all_chunks: Sequence[str | bytes],
+    ) -> str | None:
+        return AnthropicPassthroughLoggingHandler._extract_message_start_field(all_chunks, "model")
+
+    @staticmethod
+    def _extract_response_id_from_anthropic_chunks(
+        all_chunks: Sequence[str | bytes],
+    ) -> str | None:
+        return AnthropicPassthroughLoggingHandler._extract_message_start_field(all_chunks, "id")
+
     @staticmethod
     def _stream_was_interrupted(
         all_chunks: Sequence[str | bytes],
@@ -251,6 +265,7 @@ class AnthropicPassthroughLoggingHandler:
         start_time: datetime,
         end_time: datetime,
         logging_obj: LiteLLMLoggingObj,
+        response_id: str | None = None,
     ):
         """
         Create the standard logging object for Anthropic passthrough
@@ -312,8 +327,7 @@ class AnthropicPassthroughLoggingHandler:
                 json.dumps(kwargs, indent=4, default=str),
             )
 
-            # set litellm_call_id to logging response object
-            litellm_model_response.id = logging_obj.litellm_call_id
+            litellm_model_response.id = response_id or logging_obj.litellm_call_id
             litellm_model_response.model = model
             logging_obj.model_call_details["model"] = model
             if not logging_obj.model_call_details.get("custom_llm_provider"):
@@ -413,6 +427,7 @@ class AnthropicPassthroughLoggingHandler:
             start_time=start_time,
             end_time=end_time,
             logging_obj=litellm_logging_obj,
+            response_id=AnthropicPassthroughLoggingHandler._extract_response_id_from_anthropic_chunks(all_chunks),
         )
 
         return {

litellm/proxy/prompts/prompt_endpoints.py

@@ -29,6 +29,12 @@ from litellm.proxy._types import (
 from litellm.proxy.auth.auth_utils import is_request_body_safe
 from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
 from litellm.proxy.common_utils.path_utils import safe_filename
+from litellm.proxy.prompts.prompt_registry import (
+    DEFAULT_PROMPT_ENVIRONMENT,
+    get_base_prompt_id,
+    get_version_number,
+    prompt_environment_or_default,
+)
 from litellm.repositories.table_repositories import PromptRepository
 from litellm.types.prompts.init_prompts import (
     ListPromptsResponse,
@@ -102,165 +108,20 @@ def _prompt_table(prisma_client: "PrismaClient") -> _PromptTableActions:
     return PromptRepository(prisma_client).table
 
 
-def get_base_prompt_id(prompt_id: str) -> str:
-    """
-    Extract the base prompt ID by stripping the version suffix if present.
-
-    Args:
-        prompt_id: Prompt ID that may include version suffix (e.g., "jack_success.v1" or "jack_success_v1")
-
-    Returns:
-        Base prompt ID without version suffix (e.g., "jack_success")
-
-    Examples:
-        >>> get_base_prompt_id("jack_success.v1")
-        "jack_success"
-        >>> get_base_prompt_id("jack_success_v1")
-        "jack_success"
-        >>> get_base_prompt_id("jack_success")
-        "jack_success"
-    """
-    # Try dot separator first (.v)
-    if ".v" in prompt_id:
-        return prompt_id.split(".v")[0]
-    # Try underscore separator (_v)
-    if "_v" in prompt_id:
-        return prompt_id.split("_v")[0]
-    return prompt_id
-
-
-def get_version_number(prompt_id: str) -> int:
-    """
-    Extract the version number from a versioned prompt ID.
-
-    Args:
-        prompt_id: Prompt ID that may include version suffix (e.g., "jack_success.v2" or "jack_success_v2")
-
-    Returns:
-        Version number (defaults to 1 if no version suffix or invalid format)
-
-    Examples:
-        >>> get_version_number("jack_success.v2")
-        2
-        >>> get_version_number("jack_success_v2")
-        2
-        >>> get_version_number("jack_success")
-        1
-    """
-    # Try dot separator first (.v)
-    if ".v" in prompt_id:
-        version_str = prompt_id.split(".v")[1]
-        try:
-            return int(version_str)
-        except ValueError:
-            pass
-
-    # Try underscore separator (_v)
-    if "_v" in prompt_id:
-        version_str = prompt_id.split("_v")[1]
-        try:
-            return int(version_str)
-        except ValueError:
-            pass
-
-    return 1
-
-
-def construct_versioned_prompt_id(prompt_id: str, version: int | None = None) -> str:
-    """
-    Construct a versioned prompt ID from a base prompt_id and version number.
-
-    Args:
-        prompt_id: Base prompt ID (e.g., "jack_success")
-        version: Version number (if None, returns the base prompt_id unchanged)
-
-    Returns:
-        Versioned prompt ID (e.g., "jack_success.v4")
-
-    Examples:
-        >>> construct_versioned_prompt_id("jack_success", 4)
-        "jack_success.v4"
-        >>> construct_versioned_prompt_id("jack_success", None)
-        "jack_success"
-        >>> construct_versioned_prompt_id("jack_success.v2", 4)
-        "jack_success.v4"
-    """
-    if version is None:
-        return prompt_id
-
-    # Strip any existing version suffix first
-    base_id: Final = get_base_prompt_id(prompt_id)
-    return f"{base_id}.v{version}"
-
-
-def get_latest_version_prompt_id(prompt_id: str, all_prompt_ids: Mapping[str, object]) -> str:
-    """
-    Find the latest version of a prompt from available prompt IDs.
-
-    Args:
-        prompt_id: Base prompt ID or versioned prompt ID (e.g., "jack_success" or "jack_success.v2")
-        all_prompt_ids: Dictionary of all available prompt IDs (keys are prompt IDs)
-
-    Returns:
-        The prompt ID with the highest version number, or the original prompt_id if no versions exist
-
-    Examples:
-        >>> all_ids = {"jack.v1": {}, "jack.v2": {}, "jack.v3": {}}
-        >>> get_latest_version_prompt_id("jack", all_ids)
-        "jack.v3"
-        >>> get_latest_version_prompt_id("jack.v1", all_ids)
-        "jack.v3"
-        >>> all_ids = {"simple": {}}
-        >>> get_latest_version_prompt_id("simple", all_ids)
-        "simple"
-    """
-    base_id: Final = get_base_prompt_id(prompt_id=prompt_id)
-
-    # Find all versions of this prompt
-    matching_versions: Final = []
-    for stored_prompt_id in all_prompt_ids:
-        if get_base_prompt_id(prompt_id=stored_prompt_id) == base_id:
-            version_num = get_version_number(prompt_id=stored_prompt_id)
-            matching_versions.append((version_num, stored_prompt_id))
-
-    # Use the highest version number
-    if matching_versions:
-        matching_versions.sort(reverse=True)
-        return matching_versions[0][1]
-    else:
-        # No versioned prompts found, use the base ID as-is
-        return prompt_id
-
-
 def get_latest_prompt_versions(prompts: list[PromptSpec]) -> list[PromptSpec]:
     """
-    Filter a list of prompts to return only the latest version of each unique prompt.
-
-    Args:
-        prompts: List of PromptSpec objects
-
-    Returns:
-        List of PromptSpec objects with only the latest version of each prompt
+    Filter prompts down to the latest version per (base prompt id, environment).
     """
-    latest_prompts: Final[dict[str, PromptSpec]] = {}
-
-    for prompt in prompts:
-        base_id = get_base_prompt_id(prompt_id=prompt.prompt_id)
-        version = get_version_number(prompt_id=prompt.prompt_id)
-
-        # Keep the prompt with the highest version number
-        if base_id not in latest_prompts:
-            latest_prompts[base_id] = prompt
-        else:
-            existing_version = get_version_number(prompt_id=latest_prompts[base_id].prompt_id)
-            if version > existing_version:
-                latest_prompts[base_id] = prompt
-
+    sorted_prompts: Final = sorted(prompts, key=lambda prompt: get_version_number(prompt_id=prompt.prompt_id))
+    latest_prompts: Final = {
+        (get_base_prompt_id(prompt_id=prompt.prompt_id), prompt_environment_or_default(prompt.environment)): prompt
+        for prompt in sorted_prompts
+    }
     return list(latest_prompts.values())
 
 
 async def get_next_version_for_prompt(
-    prisma_client: "PrismaClient", prompt_id: str, environment: str = "development"
+    prisma_client: "PrismaClient", prompt_id: str, environment: str = DEFAULT_PROMPT_ENVIRONMENT
 ) -> int:
     """
     Get the next version number for a prompt in a specific environment.
@@ -403,11 +264,14 @@ async def list_prompts(
     if key_metadata is not None:
         prompts: Final = cast(list[str] | None, key_metadata.get("prompts", None))
         if prompts is not None:
-            all_prompts = [
-                IN_MEMORY_PROMPT_REGISTRY.IN_MEMORY_PROMPTS[prompt_id]
-                for prompt_id in prompts
-                if prompt_id in IN_MEMORY_PROMPT_REGISTRY.IN_MEMORY_PROMPTS
+            allowed_prompt_ids: Final = frozenset(prompts)
+            allowed_prompts: Final = [
+                spec
+                for spec in IN_MEMORY_PROMPT_REGISTRY.IN_MEMORY_PROMPTS.values()
+                if spec.prompt_id in allowed_prompt_ids
+                or get_base_prompt_id(prompt_id=spec.prompt_id) in allowed_prompt_ids
             ]
+            all_prompts = get_latest_prompt_versions(prompts=allowed_prompts)
             if environment:
                 all_prompts = [p for p in all_prompts if p.environment == environment]
             prompt_list: Final = []
@@ -576,7 +440,7 @@ def _get_prompt_template(prompt_spec: PromptSpec, base_prompt_id: str) -> Prompt
                     metadata=parsed.get("metadata"),
                 )
         else:
-            prompt_callback: Final = IN_MEMORY_PROMPT_REGISTRY.get_prompt_callback_by_id(prompt_spec.prompt_id)
+            prompt_callback: Final = IN_MEMORY_PROMPT_REGISTRY.get_prompt_callback_for_prompt(prompt=prompt_spec)
             if prompt_callback is not None:
                 integration_name: Final = prompt_callback.integration_name
                 if integration_name == "dotprompt":
@@ -690,15 +554,10 @@ async def get_prompt_info(
         if env_prompts:
             prompt_spec = create_versioned_prompt_spec(db_prompt=env_prompts[0])
 
-    # Fallback: use in-memory registry (no environment filter)
-    if prompt_spec is None and environment is None:
-        prompt_spec = IN_MEMORY_PROMPT_REGISTRY.get_prompt_by_id(prompt_id)
-        if prompt_spec is None:
-            latest_prompt_id: Final = get_latest_version_prompt_id(
-                prompt_id=prompt_id,
-                all_prompt_ids=IN_MEMORY_PROMPT_REGISTRY.IN_MEMORY_PROMPTS,
-            )
-            prompt_spec = IN_MEMORY_PROMPT_REGISTRY.get_prompt_by_id(latest_prompt_id)
+    if prompt_spec is None:
+        prompt_spec = IN_MEMORY_PROMPT_REGISTRY.resolve_prompt_spec(
+            prompt_id, version=requested_version, environment=environment
+        )
 
     if prompt_spec is None:
         raise HTTPException(
@@ -785,7 +644,7 @@ async def create_prompt(
         environment: Final = (
             request.prompt_info.environment
             if request.prompt_info and request.prompt_info.environment
-            else "development"
+            else DEFAULT_PROMPT_ENVIRONMENT
         )
 
         # Get next version number
@@ -885,7 +744,7 @@ async def update_prompt(
         environment: Final = (
             request.prompt_info.environment
             if request.prompt_info and request.prompt_info.environment
-            else "development"
+            else DEFAULT_PROMPT_ENVIRONMENT
         )
 
         # Check if any version of this prompt exists (in any environment)
@@ -897,9 +756,7 @@ async def update_prompt(
                 detail=f"Prompt with ID {base_prompt_id} not found",
             )
 
-        # Check if it's a config prompt
-        existing_in_memory: Final = IN_MEMORY_PROMPT_REGISTRY.get_prompt_by_id(prompt_id)
-        if existing_in_memory and existing_in_memory.prompt_info.prompt_type == "config":
+        if IN_MEMORY_PROMPT_REGISTRY.has_config_prompt(base_prompt_id=base_prompt_id):
             raise HTTPException(
                 status_code=400,
                 detail="Cannot update config prompts.",
@@ -988,40 +845,26 @@ async def delete_prompt(
         raise HTTPException(status_code=500, detail=CommonProxyErrors.db_not_connected_error.value)
 
     try:
-        # Try to get prompt directly first
-        existing_prompt = IN_MEMORY_PROMPT_REGISTRY.get_prompt_by_id(prompt_id)
-
-        # If not found, try to find the latest version
-        if existing_prompt is None:
-            latest_prompt_id: Final = get_latest_version_prompt_id(
-                prompt_id=prompt_id,
-                all_prompt_ids=IN_MEMORY_PROMPT_REGISTRY.IN_MEMORY_PROMPTS,
-            )
-            existing_prompt = IN_MEMORY_PROMPT_REGISTRY.get_prompt_by_id(latest_prompt_id)
-            # Use the resolved prompt_id for deletion
-            prompt_id = latest_prompt_id
+        base_prompt_id: Final = get_base_prompt_id(prompt_id=prompt_id)
+        existing_prompt: Final = IN_MEMORY_PROMPT_REGISTRY.resolve_prompt_spec(prompt_id, environment=environment)
 
         if existing_prompt is None:
             raise HTTPException(status_code=404, detail=f"Prompt with ID {prompt_id} not found")
 
-        if existing_prompt.prompt_info.prompt_type == "config":
+        if IN_MEMORY_PROMPT_REGISTRY.has_config_prompt(base_prompt_id=base_prompt_id):
             raise HTTPException(
                 status_code=400,
                 detail="Cannot delete config prompts.",
             )
 
-        # Get the base prompt ID (without version suffix) for database deletion
-        base_prompt_id: Final = get_base_prompt_id(prompt_id=prompt_id)
-
-        # Build delete filter; scope to environment if provided
-        delete_where: Final[dict[str, str]] = {"prompt_id": base_prompt_id}
-        if environment:
-            delete_where["environment"] = environment
-
-        # Delete versions from the database (scoped to environment if provided)
+        delete_where: Final[dict[str, str]] = {
+            "prompt_id": base_prompt_id,
+            **({"environment": environment} if environment else {}),
+        }
         await _prompt_table(prisma_client).delete_many(where=delete_where)
-
-        IN_MEMORY_PROMPT_REGISTRY.delete_prompts_by_base_id(base_prompt_id, environment=environment or None)
+        IN_MEMORY_PROMPT_REGISTRY.delete_prompts_by_base_id(
+            base_prompt_id=base_prompt_id, environment=environment or None
+        )
 
         env_msg: Final = f" from {environment}" if environment else ""
         return {"message": f"Prompt {base_prompt_id} deleted successfully{env_msg}"}
@@ -1093,7 +936,7 @@ async def patch_prompt(
     try:
         # Resolve the target row: find the latest version in the given environment
         base_prompt_id: Final = get_base_prompt_id(prompt_id=prompt_id)
-        env: Final = environment or "development"
+        env: Final = prompt_environment_or_default(environment)
         requested_version: Final = get_version_number(prompt_id=prompt_id) if prompt_id != base_prompt_id else None
 
         # Build query to find the exact row by composite unique key
@@ -1117,11 +960,7 @@ async def patch_prompt(
 
         target_row: Final = db_rows[0]
 
-        # Check if prompt exists in memory
-        versioned_id: Final = f"{base_prompt_id}.v{target_row.version}"
-        existing_prompt: Final = IN_MEMORY_PROMPT_REGISTRY.get_prompt_by_id(versioned_id)
-
-        if existing_prompt and existing_prompt.prompt_info.prompt_type == "config":
+        if IN_MEMORY_PROMPT_REGISTRY.has_config_prompt(base_prompt_id=base_prompt_id):
             raise HTTPException(
                 status_code=400,
                 detail="Cannot update config prompts.",

litellm/proxy/prompts/prompt_registry.py

@@ -1,6 +1,6 @@
 import importlib
 import os
-from collections.abc import Callable
+from collections.abc import Callable, Sequence
 from pathlib import Path
 from typing import Final
 
@@ -14,6 +14,87 @@ from litellm.types.prompts.init_prompts import (
 
 prompt_initializer_registry = {}
 
+DEFAULT_PROMPT_ENVIRONMENT: Final = "development"
+PROMPT_ENVIRONMENT_SERVE_PRECEDENCE: Final = ("production", "staging", "development")
+
+
+def get_base_prompt_id(prompt_id: str) -> str:
+    """
+    Extract the base prompt ID by stripping the version suffix if present.
+
+    Examples:
+        >>> get_base_prompt_id("jack_success.v1")
+        "jack_success"
+        >>> get_base_prompt_id("jack_success_v1")
+        "jack_success"
+        >>> get_base_prompt_id("jack_success")
+        "jack_success"
+    """
+    if ".v" in prompt_id:
+        return prompt_id.split(".v")[0]
+    if "_v" in prompt_id:
+        return prompt_id.split("_v")[0]
+    return prompt_id
+
+
+def get_version_number(prompt_id: str) -> int:
+    """
+    Extract the version number from a versioned prompt ID (defaults to 1).
+
+    Examples:
+        >>> get_version_number("jack_success.v2")
+        2
+        >>> get_version_number("jack_success_v2")
+        2
+        >>> get_version_number("jack_success")
+        1
+    """
+    if ".v" in prompt_id:
+        version_str = prompt_id.split(".v")[1]
+        try:
+            return int(version_str)
+        except ValueError:
+            pass
+
+    if "_v" in prompt_id:
+        version_str = prompt_id.split("_v")[1]
+        try:
+            return int(version_str)
+        except ValueError:
+            pass
+
+    return 1
+
+
+def prompt_environment_or_default(environment: str | None) -> str:
+    return environment or DEFAULT_PROMPT_ENVIRONMENT
+
+
+def registry_key_for_prompt(prompt: PromptSpec) -> str:
+    return f"{prompt.prompt_id}::{prompt_environment_or_default(prompt.environment)}"
+
+
+def parse_prompt_version(raw_version: object) -> int | None:
+    if isinstance(raw_version, bool):
+        return None
+    if isinstance(raw_version, int):
+        return raw_version
+    if isinstance(raw_version, str) and raw_version.isdigit():
+        return int(raw_version)
+    return None
+
+
+def _spec_version(prompt: PromptSpec) -> int:
+    return prompt.version if prompt.version is not None else get_version_number(prompt_id=prompt.prompt_id)
+
+
+def _default_serve_environment(prompts: Sequence[PromptSpec]) -> str:
+    present: Final = frozenset(prompt_environment_or_default(prompt.environment) for prompt in prompts)
+    ladder_pick: Final = next((env for env in PROMPT_ENVIRONMENT_SERVE_PRECEDENCE if env in present), None)
+    if ladder_pick is not None:
+        return ladder_pick
+    return min(present) if present else DEFAULT_PROMPT_ENVIRONMENT
+
 
 def get_prompt_initializer_from_integrations():
     """
@@ -113,17 +194,16 @@ class InMemoryPromptRegistry:
         """
         import litellm
 
-        prompt_id: Final = prompt.prompt_id
-        if prompt_id in self.IN_MEMORY_PROMPTS:
-            verbose_proxy_logger.debug("prompt_id already exists in IN_MEMORY_PROMPTS")
-            return self.IN_MEMORY_PROMPTS[prompt_id]
+        registry_key: Final = registry_key_for_prompt(prompt)
+        if registry_key in self.IN_MEMORY_PROMPTS:
+            verbose_proxy_logger.debug("prompt already exists in IN_MEMORY_PROMPTS")
+            return self.IN_MEMORY_PROMPTS[registry_key]
 
         parsed_prompt, custom_prompt_callback = self._build_prompt_callback(prompt=prompt)
         litellm.logging_callback_manager.add_litellm_callback(custom_prompt_callback)
 
-        # store references to the prompt in memory
-        self.IN_MEMORY_PROMPTS[prompt_id] = parsed_prompt
-        self.prompt_id_to_custom_prompt[prompt_id] = custom_prompt_callback
+        self.IN_MEMORY_PROMPTS[registry_key] = parsed_prompt
+        self.prompt_id_to_custom_prompt[registry_key] = custom_prompt_callback
 
         return parsed_prompt
 
@@ -166,68 +246,93 @@ class InMemoryPromptRegistry:
         import litellm
 
         parsed_prompt, new_callback = self._build_prompt_callback(prompt=prompt)
-        stale_callback: Final = self.prompt_id_to_custom_prompt.pop(prompt.prompt_id, None)
-        self.IN_MEMORY_PROMPTS.pop(prompt.prompt_id, None)
+        registry_key: Final = registry_key_for_prompt(parsed_prompt)
+        stale_callback: Final = self.prompt_id_to_custom_prompt.pop(registry_key, None)
+        self.IN_MEMORY_PROMPTS.pop(registry_key, None)
         if stale_callback is not None:
             litellm.logging_callback_manager.remove_callback_from_all_lists(stale_callback)
         litellm.logging_callback_manager.add_litellm_callback(new_callback)
-        self.IN_MEMORY_PROMPTS[prompt.prompt_id] = parsed_prompt
-        self.prompt_id_to_custom_prompt[prompt.prompt_id] = new_callback
+        self.IN_MEMORY_PROMPTS[registry_key] = parsed_prompt
+        self.prompt_id_to_custom_prompt[registry_key] = new_callback
         return parsed_prompt
 
     def sync_prompt_from_db(self, prompt: PromptSpec) -> PromptSpec | None:
-        existing: Final = self.IN_MEMORY_PROMPTS.get(prompt.prompt_id)
+        existing: Final = self.IN_MEMORY_PROMPTS.get(registry_key_for_prompt(prompt))
         if existing is None:
             return self.initialize_prompt(prompt=prompt)
         if existing.litellm_params == prompt.litellm_params and existing.prompt_info == prompt.prompt_info:
             return existing
         return self.reload_prompt(prompt=prompt)
 
-    def get_prompt_by_id(self, prompt_id: str) -> PromptSpec | None:
+    def resolve_prompt_spec(
+        self,
+        prompt_id: str,
+        version: int | None = None,
+        environment: str | None = None,
+    ) -> PromptSpec | None:
         """
-        Get a prompt by its ID from memory
-        """
-        return self.IN_MEMORY_PROMPTS.get(prompt_id)
+        Resolve a prompt spec by base prompt id, optional version, and optional environment.
 
-    def get_prompt_callback_by_id(self, prompt_id: str) -> CustomPromptManagement | None:
+        With no environment, resolves within the default serve environment
+        (production > staging > development > alphabetical first present).
+        With no version, resolves to the highest version in the chosen environment.
         """
-        Get a prompt callback by its ID from memory
-        """
-        return self.prompt_id_to_custom_prompt.get(prompt_id)
+        base_prompt_id: Final = get_base_prompt_id(prompt_id=prompt_id)
+        base_matches: Final = tuple(
+            spec
+            for spec in self.IN_MEMORY_PROMPTS.values()
+            if get_base_prompt_id(prompt_id=spec.prompt_id) == base_prompt_id
+        )
+        if not base_matches:
+            return None
+        resolved_environment: Final = (
+            environment if environment is not None else _default_serve_environment(base_matches)
+        )
+        env_matches: Final = tuple(
+            spec for spec in base_matches if prompt_environment_or_default(spec.environment) == resolved_environment
+        )
+        if not env_matches:
+            return None
+        if version is not None:
+            return next((spec for spec in env_matches if _spec_version(spec) == version), None)
+        return max(env_matches, key=_spec_version)
 
-    def remove_prompt(self, prompt_id: str) -> None:
+    def get_prompt_callback_for_prompt(self, prompt: PromptSpec) -> CustomPromptManagement | None:
+        return self.prompt_id_to_custom_prompt.get(registry_key_for_prompt(prompt))
+
+    def has_config_prompt(self, base_prompt_id: str) -> bool:
+        return any(
+            spec.prompt_info.prompt_type == "config"
+            for spec in self.IN_MEMORY_PROMPTS.values()
+            if get_base_prompt_id(prompt_id=spec.prompt_id) == base_prompt_id
+        )
+
+    def remove_prompt(self, registry_key: str) -> None:
         import litellm
 
-        self.IN_MEMORY_PROMPTS.pop(prompt_id, None)
-        stale_callback: Final = self.prompt_id_to_custom_prompt.pop(prompt_id, None)
+        self.IN_MEMORY_PROMPTS.pop(registry_key, None)
+        stale_callback: Final = self.prompt_id_to_custom_prompt.pop(registry_key, None)
         if stale_callback is not None:
             litellm.logging_callback_manager.remove_callback_from_all_lists(stale_callback)
 
     def delete_prompts_by_base_id(self, base_prompt_id: str, environment: str | None = None) -> list[str]:
         """
-        Delete all prompts matching the given base prompt ID from memory, along with their
-        registered callbacks; scoped to one environment when given.
+        Delete matching prompts from memory, along with their registered callbacks,
+        scoped to one environment when given.
 
-        Args:
-            base_prompt_id: The base prompt ID (without version suffix)
-            environment: When set, only delete prompts deployed to this environment
-
-        Returns:
-            List of prompt IDs that were deleted
+        Returns the registry keys that were deleted.
         """
-        from litellm.proxy.prompts.prompt_endpoints import get_base_prompt_id
-
-        prompts_to_delete: Final = [
-            pid
-            for pid, prompt in self.IN_MEMORY_PROMPTS.items()
-            if get_base_prompt_id(prompt_id=pid) == base_prompt_id
-            and (environment is None or prompt.environment == environment)
+        keys_to_delete: Final = [
+            key
+            for key, spec in self.IN_MEMORY_PROMPTS.items()
+            if get_base_prompt_id(prompt_id=spec.prompt_id) == base_prompt_id
+            and (environment is None or prompt_environment_or_default(spec.environment) == environment)
         ]
 
-        for pid in prompts_to_delete:
-            self.remove_prompt(prompt_id=pid)
+        for key in keys_to_delete:
+            self.remove_prompt(registry_key=key)
 
-        return prompts_to_delete
+        return keys_to_delete
 
 
 IN_MEMORY_PROMPT_REGISTRY: Final = InMemoryPromptRegistry()

litellm/proxy/proxy_server.py

@@ -120,6 +120,8 @@ from litellm.router_utils.add_retry_fallback_headers import (
 from litellm.router_utils.auto_router_model_naming import (
     STRATEGY_ROUTER_PARAM_FIELDS,
     carries_complexity_router_settings,
+    count_heuristic_v2_routers,
+    heuristic_v2_limit_violation,
     validate_complexity_router_config_placement,
 )
 from litellm.types.utils import (
@@ -301,7 +303,7 @@ from litellm.proxy.auth.auth_utils import (
 )
 from litellm.proxy.auth.fallback_model_access import router_fallback_access_check
 from litellm.proxy.auth.handle_jwt import JWTHandler
-from litellm.proxy.auth.litellm_license import LicenseCheck
+from litellm.proxy.auth.litellm_license import HEURISTIC_V2_LICENSE_REMEDY, LicenseCheck
 from litellm.proxy.auth.model_checks import (
     expand_wildcard_deployments_for_model_info,
     get_all_fallbacks,
@@ -4316,6 +4318,19 @@ def validate_deployment_complexity_router_placement(model: Mapping[str, object])
         raise ValueError(f"model {model.get('model_name', '')!r}: {violation}")
 
 
+def validate_heuristic_v2_router_limit(model_list: Sequence[Mapping[str, object]], *, limit: int | None) -> None:
+    """
+    Refuse to start when config.yaml defines more heuristic_v2 auto-routers than the license allows.
+
+    Checked here rather than left to router registration for the same reason as the two
+    validators above: the proxy builds its router with `ignore_invalid_deployments=True`, so
+    the router's own refusal would turn the extra router into a silently missing model.
+    """
+    violation: Final = heuristic_v2_limit_violation(held=count_heuristic_v2_routers(model_list), limit=limit)
+    if violation is not None:
+        raise ValueError(f"config.yaml model_list: {violation} {HEURISTIC_V2_LICENSE_REMEDY}")
+
+
 def pin_complexity_router_model_id(model: dict) -> None:  # mutable-ok: out-param, model_info is stamped in place
     """
     Stamps `model_info.id` from the raw litellm_params before plugin resolution swaps
@@ -5721,6 +5736,7 @@ class ProxyConfig:
         model_list: Final = config.get("model_list", None)
         if model_list:
             router_params["model_list"] = model_list
+            validate_heuristic_v2_router_limit(model_list, limit=_license_check.heuristic_v2_router_limit())
             print(  # noqa: T201
                 "\033[32mLiteLLM: Proxy initialized with Config, Set models:\033[0m"
             )
@@ -5810,6 +5826,7 @@ class ProxyConfig:
             ),
             ignore_invalid_deployments=True,  # don't raise an error if a deployment is invalid
             fallback_access_check=router_fallback_access_check,
+            heuristic_v2_router_limit=_license_check.heuristic_v2_router_limit,
         )
 
         if redis_usage_cache is not None and router.cache.redis_cache is None:
@@ -6270,6 +6287,7 @@ class ProxyConfig:
                         search_tools=search_tools,
                         ignore_invalid_deployments=True,
                         fallback_access_check=router_fallback_access_check,
+                        heuristic_v2_router_limit=_license_check.heuristic_v2_router_limit,
                     )
                     verbose_proxy_logger.debug("updated llm_router: %s", llm_router)
             else:
@@ -7571,7 +7589,7 @@ class ProxyConfig:
         return create_versioned_prompt_spec(db_prompt=db_prompt)
 
     async def _init_prompts_in_db(self, prisma_client: PrismaClient):
-        from litellm.proxy.prompts.prompt_registry import IN_MEMORY_PROMPT_REGISTRY
+        from litellm.proxy.prompts.prompt_registry import IN_MEMORY_PROMPT_REGISTRY, registry_key_for_prompt
         from litellm.types.prompts.init_prompts import PromptSpec
 
         def parse_row(db_prompt: object) -> PromptSpec | None:
@@ -7586,21 +7604,12 @@ class ProxyConfig:
                 return None
 
         try:
-            prompt_ids_loaded_before_db_read: Final = frozenset(IN_MEMORY_PROMPT_REGISTRY.IN_MEMORY_PROMPTS)
+            registry_keys_loaded_before_db_read: Final = frozenset(IN_MEMORY_PROMPT_REGISTRY.IN_MEMORY_PROMPTS)
             prompts_in_db: Final[Sequence[object]] = await PromptRepository(prisma_client).table.find_many()
             parsed_specs: Final[tuple[PromptSpec, ...]] = tuple(
                 spec for row in prompts_in_db if (spec := parse_row(row)) is not None
             )
-            newest_spec_per_id: Final[Mapping[str, PromptSpec]] = MappingProxyType(
-                {
-                    spec.prompt_id: spec
-                    for spec in sorted(
-                        parsed_specs,
-                        key=lambda s: s.updated_at.timestamp() if s.updated_at else float("-inf"),
-                    )
-                }
-            )
-            for prompt_spec in newest_spec_per_id.values():
+            for prompt_spec in parsed_specs:
                 try:
                     IN_MEMORY_PROMPT_REGISTRY.sync_prompt_from_db(prompt=prompt_spec)
                 except Exception as prompt_sync_error:  # noqa: BLE001  # one poisoned row must not block syncing the remaining prompts
@@ -7612,15 +7621,16 @@ class ProxyConfig:
             # An unparsable row still exists in the DB, so skip the sweep rather than unload its in-memory copy
             every_row_parsed: Final = len(parsed_specs) == len(prompts_in_db)
             if every_row_parsed:
-                deleted_db_prompt_ids: Final = tuple(
-                    prompt_id
-                    for prompt_id in prompt_ids_loaded_before_db_read
-                    if (loaded_spec := IN_MEMORY_PROMPT_REGISTRY.IN_MEMORY_PROMPTS.get(prompt_id)) is not None
+                db_registry_keys: Final = frozenset(registry_key_for_prompt(spec) for spec in parsed_specs)
+                deleted_db_registry_keys: Final = tuple(
+                    registry_key
+                    for registry_key in registry_keys_loaded_before_db_read
+                    if (loaded_spec := IN_MEMORY_PROMPT_REGISTRY.IN_MEMORY_PROMPTS.get(registry_key)) is not None
                     and loaded_spec.prompt_info.prompt_type == "db"
-                    and prompt_id not in newest_spec_per_id
+                    and registry_key not in db_registry_keys
                 )
-                for deleted_prompt_id in deleted_db_prompt_ids:
-                    IN_MEMORY_PROMPT_REGISTRY.remove_prompt(prompt_id=deleted_prompt_id)
+                for deleted_registry_key in deleted_db_registry_keys:
+                    IN_MEMORY_PROMPT_REGISTRY.remove_prompt(registry_key=deleted_registry_key)
         except Exception as e:
             verbose_proxy_logger.debug("litellm.proxy.proxy_server.py::ProxyConfig:_init_prompts_in_db - %s", e)