Merge remote-tracking branch 'origin/litellm_internal_staging' into litellm_decrease_anys_fable6

# Conflicts:
#	basedpyright-code-budget.json
#	ruff-strict-budget.json
#	type-discipline-budget.json

.github/workflows/codspeed.yml

@@ -12,6 +12,7 @@ on:
       - "uv.lock"
       - ".github/workflows/codspeed.yml"
       - ".github/actions/setup-uv-with-retries/**"
+      - ".github/actions/cache-cargo-build/**"
   pull_request:
     branches:
       - main
@@ -23,6 +24,7 @@ on:
       - "uv.lock"
       - ".github/workflows/codspeed.yml"
       - ".github/actions/setup-uv-with-retries/**"
+      - ".github/actions/cache-cargo-build/**"
   # Allow CodSpeed to trigger backtest performance analysis
   # in order to generate initial data
   workflow_dispatch:
@@ -55,6 +57,26 @@ jobs:
         with:
           version: "0.10.9"
 
+      - name: Cache the Rust build
+        uses: ./.github/actions/cache-cargo-build
+
+      # Build the wheel and resolve every dependency outside the CodSpeed
+      # runner: the same maturin build took 42 minutes inside `codspeed run`
+      # versus under 3 minutes as a plain step (LIT-6183)
+      - name: Build environment
+        run: >
+          env PYTEST_DISABLE_PLUGIN_AUTOLOAD=1
+          uv run --frozen --no-default-groups
+          --with pytest==8.3.5
+          --with pytest-codspeed==4.3.0
+          --with "mcp>=1.26.0,<2.0"
+          --with "a2a-sdk>=1.1.0,<2.0"
+          pytest
+          -p pytest_codspeed.plugin
+          tests/benchmarks/
+          --codspeed
+          --collect-only -q
+
       - name: Run benchmarks
         uses: CodSpeedHQ/action@1c8ae4843586d3ba879736b7f6b7b0c990757fab # v4.12.1
         with:

basedpyright-code-budget.json

@@ -1,9 +1,9 @@
 {
   "reportAny": {
-    "limit": 16413
+    "limit": 16984
   },
   "reportArgumentType": {
-    "limit": 2530
+    "limit": 2535
   },
   "reportAssignmentType": {
     "limit": 319
@@ -18,13 +18,13 @@
     "limit": 40
   },
   "reportDeprecated": {
-    "limit": 209
+    "limit": 211
   },
   "reportDuplicateImport": {
     "limit": 19
   },
   "reportExplicitAny": {
-    "limit": 5357
+    "limit": 5442
   },
   "reportFunctionMemberAccess": {
     "limit": 7
@@ -54,10 +54,10 @@
     "limit": 0
   },
   "reportMissingParameterType": {
-    "limit": 5649
+    "limit": 5655
   },
   "reportMissingTypeArgument": {
-    "limit": 15407
+    "limit": 15419
   },
   "reportMissingTypeStubs": {
     "limit": 40
@@ -93,25 +93,25 @@
     "limit": 213
   },
   "reportTypedDictNotRequiredAccess": {
-    "limit": 22
+    "limit": 24
   },
   "reportUndefinedVariable": {
     "limit": 0
   },
   "reportUnknownArgumentType": {
-    "limit": 44463
+    "limit": 44505
   },
   "reportUnknownLambdaType": {
     "limit": 109
   },
   "reportUnknownMemberType": {
-    "limit": 38625
+    "limit": 38689
   },
   "reportUnknownParameterType": {
-    "limit": 19754
+    "limit": 19770
   },
   "reportUnknownVariableType": {
-    "limit": 30212
+    "limit": 30264
   },
   "reportUnnecessaryCast": {
     "limit": 117
@@ -123,7 +123,7 @@
     "limit": 5
   },
   "reportUnnecessaryIsInstance": {
-    "limit": 826
+    "limit": 828
   },
   "reportUntypedBaseClass": {
     "limit": 0

codecov.yaml

@@ -25,6 +25,8 @@ flag_management:
       carryforward: false
     - name: proxy-db-schema-migration
       carryforward: false
+    - name: circleci
+      carryforward: false
 
 component_management:
   individual_components:

enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py

@@ -2,9 +2,10 @@
 Polls LiteLLM_ManagedObjectTable to check if the batch job is complete, and if the cost has been tracked.
 """
 
+from dataclasses import replace as dataclasses_replace
 from datetime import datetime, timedelta, timezone
 from types import MappingProxyType
-from typing import TYPE_CHECKING, Any, Dict, Final, List, Optional, Tuple
+from typing import TYPE_CHECKING, Any, Dict, Final, List, Literal, Optional, Tuple, cast
 
 from litellm._logging import verbose_proxy_logger
 from litellm._uuid import uuid
@@ -626,6 +627,7 @@ class CheckBatchCost:
         later poll.
         """
         from litellm.batches.batch_utils import (
+            count_error_file_failed_requests,
             _get_file_content_as_dictionary,
             calculate_batch_cost_and_usage,
         )
@@ -761,16 +763,33 @@ class CheckBatchCost:
             model_id=model_id,
             deployment_model=litellm_model_name,
         )
-        batch_cost, batch_usage, batch_models = (
-            await calculate_batch_cost_and_usage(
-                file_content_dictionary=file_content_as_dict,
-                custom_llm_provider=llm_provider,  # type: ignore
-                model_name=model_name,
-                model_info=deployment_model_info,
+        batch_file_provider: Final = cast(
+            Literal["openai", "azure", "vertex_ai", "hosted_vllm", "anthropic"], llm_provider
+        )
+        output_file_result: Final = await calculate_batch_cost_and_usage(
+            file_content_dictionary=file_content_as_dict,
+            custom_llm_provider=batch_file_provider,
+            model_name=model_name,
+            model_info=deployment_model_info,
+        )
+        error_file_failed_requests: Final = await count_error_file_failed_requests(
+            response,
+            custom_llm_provider=batch_file_provider,
+            litellm_params={
+                **credentials,
+                "_litellm_internal_model_credentials": MappingProxyType(dict(credentials)),
+            },
+        )
+        batch_result: Final = (
+            output_file_result
+            if not error_file_failed_requests
+            else dataclasses_replace(
+                output_file_result,
+                failed_requests=output_file_result.failed_requests + error_file_failed_requests,
             )
         )
         logging_obj = LiteLLMLogging(
-            model=batch_models[0],
+            model=batch_result.models[0],
             messages=[{"role": "user", "content": "<retrieve_batch>"}],
             stream=False,
             call_type="aretrieve_batch",
@@ -802,9 +821,11 @@ class CheckBatchCost:
         try:
             await logging_obj.async_success_handler(
                 result=response,
-                batch_cost=batch_cost,
-                batch_usage=batch_usage,
-                batch_models=batch_models,
+                batch_cost=batch_result.cost,
+                batch_usage=batch_result.usage,
+                batch_models=batch_result.models,
+                batch_successful_requests=batch_result.successful_requests,
+                batch_failed_requests=batch_result.failed_requests,
             )
         except Exception:
             await self._release_job_claim(job)

enterprise/litellm_enterprise/proxy/hooks/managed_files.py

@@ -36,6 +36,7 @@ from litellm.llms.base_llm.managed_resources.isolation import (
     build_list_page,
     build_owner_filter,
     can_access_resource,
+    resolve_resource_owner_id,
 )
 from litellm.proxy._types import (
     CallTypes,
@@ -222,7 +223,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
                 file_object=file_object,
                 model_mappings=model_mappings,
                 flat_model_file_ids=list(model_mappings.values()),
-                created_by=user_api_key_dict.user_id,
+                created_by=resolve_resource_owner_id(user_api_key_dict),
                 team_id=user_api_key_dict.team_id,
                 updated_by=user_api_key_dict.user_id,
             )
@@ -238,7 +239,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
             "unified_file_id": file_id,
             "model_mappings": json.dumps(model_mappings),
             "flat_model_file_ids": list(model_mappings.values()),
-            "created_by": user_api_key_dict.user_id,
+            "created_by": resolve_resource_owner_id(user_api_key_dict),
             "team_id": user_api_key_dict.team_id,
             "updated_by": user_api_key_dict.user_id,
         }
@@ -342,7 +343,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
                     "file_object": file_object.model_dump_json(),
                     "model_object_id": model_object_id,
                     "file_purpose": file_purpose,
-                    "created_by": user_api_key_dict.user_id,
+                    "created_by": resolve_resource_owner_id(user_api_key_dict),
                     "team_id": user_api_key_dict.team_id,
                     "updated_by": user_api_key_dict.user_id,
                     "status": file_object.status,
@@ -473,19 +474,56 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
                 )
 
         page_size: Final = min(limit or 20, 100)
-        cursor_args: _CursorPageArgs = {"cursor": {"unified_object_id": after}, "skip": 1} if after else {}
-
-        batches = await _managed_object_table(self.prisma_client).find_many(
-            where=where_clause,
-            take=page_size + 1,
-            order=[{"created_at": "desc"}, {"unified_object_id": "desc"}],
-            **cursor_args,
+        matches: Final = await self._collect_listed_batches(
+            where_clause=where_clause,
+            after=after,
+            wanted=page_size + 1,
+            user_api_key_dict=user_api_key_dict,
         )
+        return build_list_page(list(matches[:page_size]), has_more=len(matches) > page_size)
 
-        has_more = len(batches) > page_size
+    async def _collect_listed_batches(
+        self,
+        where_clause: Mapping[str, object],
+        after: Optional[str],
+        wanted: int,
+        user_api_key_dict: UserAPIKeyAuth,
+    ) -> tuple[LiteLLMBatch, ...]:
+        """Read chunks newest-first until ``wanted`` batches survive parsing and
+        file-id resolution or the caller's rows run out, so a run of rows that will
+        not parse refills the page instead of emptying it. The first chunk is
+        ``wanted`` rows, so a healthy page still costs one query; a scan that has to
+        continue widens to ``FILE_LIST_CONTINUATION_CHUNK_SIZE`` like ``afile_list``,
+        and every chunk advances the keyset cursor, so the walk ends once the
+        caller's rows are exhausted."""
+        matches: tuple[LiteLLMBatch, ...] = ()  # rebind-ok: accumulates survivors across chunks
+        cursor_id: Optional[str] = after  # rebind-ok: keyset cursor advances to each chunk's last row
+        chunk_size: int = wanted  # rebind-ok: widens once a scan has to continue past the first chunk
+        while len(matches) < wanted:
+            cursor_args: _CursorPageArgs = {"cursor": {"unified_object_id": cursor_id}, "skip": 1} if cursor_id else {}
+            chunk = await _managed_object_table(self.prisma_client).find_many(
+                where=where_clause,
+                take=chunk_size,
+                order=[{"created_at": "desc"}, {"unified_object_id": "desc"}],
+                **cursor_args,
+            )
+            matches = matches + await self._resolve_listed_rows(
+                rows=chunk, wanted=wanted - len(matches), user_api_key_dict=user_api_key_dict
+            )
+            if len(chunk) < chunk_size:
+                break
+            cursor_id = chunk[-1].unified_object_id
+            chunk_size = max(chunk_size, FILE_LIST_CONTINUATION_CHUNK_SIZE)
+        return matches
 
+    async def _resolve_listed_rows(
+        self,
+        rows: "Sequence[PrismaManagedObjectRow]",
+        wanted: int,
+        user_api_key_dict: UserAPIKeyAuth,
+    ) -> tuple[LiteLLMBatch, ...]:
         parsed_rows: Final = tuple(
-            (row, batch_obj) for row in batches[:page_size] if (batch_obj := _parse_managed_batch_row(row)) is not None
+            (row, batch_obj) for row in rows if (batch_obj := _parse_managed_batch_row(row)) is not None
         )
         unified_id_by_raw_id: Final = await map_raw_file_ids_to_unified(
             raw_file_ids=frozenset(
@@ -496,19 +534,19 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
             ),
             prisma_client=self.prisma_client,
         )
-        resolved_batches: Final = [
-            await self._resolve_listed_batch(
+        resolved: Final[list[LiteLLMBatch]] = []  # mutable-ok: resolution stops as soon as the page is full
+        for row, batch_obj in parsed_rows:
+            if len(resolved) == wanted:
+                break
+            resolved_batch = await self._resolve_listed_batch(
                 row=row,
                 batch_obj=batch_obj,
                 unified_id_by_raw_id=unified_id_by_raw_id,
                 user_api_key_dict=user_api_key_dict,
             )
-            for row, batch_obj in parsed_rows
-        ]
-        return build_list_page(
-            [batch_obj for batch_obj in resolved_batches if batch_obj is not None],
-            has_more=has_more,
-        )
+            if resolved_batch is not None:
+                resolved.append(resolved_batch)
+        return tuple(resolved)
 
     async def _resolve_listed_batch(
         self,

enterprise/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "litellm-enterprise"
-version = "0.1.61"
+version = "0.1.62"
 description = "Package for LiteLLM Enterprise features"
 readme = "README.md"
 requires-python = ">=3.9"
@@ -26,7 +26,7 @@ required-version = ">=0.10.9"
 module-root = ""
 
 [tool.commitizen]
-version = "0.1.61"
+version = "0.1.62"
 version_files = [
     "pyproject.toml:^version",
     "../pyproject.toml:litellm-enterprise==",

helm/litellm/templates/ingress.yaml

@@ -5,6 +5,41 @@
 {{- $gatewayPort := .Values.gateway.service.port -}}
 {{- $backendPort := .Values.backend.service.port -}}
 {{- $uiPort := .Values.ui.service.port -}}
+{{/*
+  Backends addressable from ingress.extraPaths, keyed by the `service` field.
+*/}}
+{{- $extraPathBackends := dict
+    "gateway" (dict "name" $gatewayName "port" $gatewayPort)
+    "backend" (dict "name" $backendName "port" $backendPort)
+    "ui" (dict "name" $uiName "port" $uiPort)
+-}}
+{{/*
+  UI paths (Next.js static export).
+
+  /ui/* is where the SPA serves its login + dashboard routes (e.g. /ui/login).
+  Without it, /ui/* falls into the catch-all → backend → 404.
+
+  The App Router (output: "export", basePath: "") emits the RSC/flight payload
+  for every route as a ROOT-level <route>.txt (/index.txt, /teams.txt,
+  /__next._tree.txt, ...). The client router fetches these on every soft
+  navigation / prefetch as <route>.txt?_rsc=<hash> (the query string is
+  irrelevant to path matching). They are not under /ui, /_next, or
+  /litellm-asset-prefix, so without /*.txt they fall to the backend catch-all
+  → 404 → client-side navigation never settles and the login flow spins in an
+  infinite redirect loop (/ ⇄ /ui/login). ui/nginx.conf already serves *.txt
+  from the export; the rule only routes the request to it. Needs an ingress
+  controller whose ImplementationSpecific path is a wildcard pattern
+  (AWS ALB: `*` = 0+ chars); this chart targets the AWS Load Balancer
+  Controller.
+*/}}
+{{- $uiPaths := list
+    (dict "path" "/" "pathType" "Exact")
+    (dict "path" "/favicon.ico" "pathType" "Exact")
+    (dict "path" "/litellm-asset-prefix" "pathType" "Prefix")
+    (dict "path" "/_next" "pathType" "Prefix")
+    (dict "path" "/ui" "pathType" "Prefix")
+    (dict "path" "/*.txt" "pathType" "ImplementationSpecific")
+-}}
 {{/*
   Gateway data-plane prefixes — must mirror gateway/routes/allowlist.py.
   Versioned paths are listed explicitly to avoid routing management routes
@@ -39,6 +74,21 @@
   routes at startup -> 404. So /test is rendered as a standalone Exact path
   and /test/* falls through to the backend catch-all.
 */}}
+{{/*
+  Every "<path>|<pathType>" this template renders on its own. An
+  ingress.extraPaths entry that repeats one of these is rejected: duplicates
+  in a single rule are resolved by position or by controller-specific tie
+  breaking, so the operator entry could take over a built-in route (an entry
+  at "/" Prefix would swallow the whole backend management API) instead of
+  adding to it.
+*/}}
+{{- $builtinPathKeys := list "/test|Exact" "/|Prefix" -}}
+{{- range $uiPaths }}
+{{- $builtinPathKeys = append $builtinPathKeys (printf "%s|%s" .path .pathType) }}
+{{- end }}
+{{- range $gatewayPrefixes }}
+{{- $builtinPathKeys = append $builtinPathKeys (printf "%s|Prefix" .) }}
+{{- end }}
 apiVersion: networking.k8s.io/v1
 kind: Ingress
 metadata:
@@ -64,65 +114,15 @@ spec:
       http:
         paths:
           # --- UI (Next.js static export) ---
-          - path: /
-            pathType: Exact
-            backend:
-              service:
-                name: {{ $uiName }}
-                port:
-                  number: {{ $uiPort }}
-          - path: /favicon.ico
-            pathType: Exact
-            backend:
-              service:
-                name: {{ $uiName }}
-                port:
-                  number: {{ $uiPort }}
-          - path: /litellm-asset-prefix
-            pathType: Prefix
-            backend:
-              service:
-                name: {{ $uiName }}
-                port:
-                  number: {{ $uiPort }}
-          - path: /_next
-            pathType: Prefix
-            backend:
-              service:
-                name: {{ $uiName }}
-                port:
-                  number: {{ $uiPort }}
-          # /ui/* is where the Next.js SPA serves its login + dashboard
-          # routes (e.g. /ui/login). Without this, /ui/* falls into the
-          # catch-all → backend → 404.
-          - path: /ui
-            pathType: Prefix
-            backend:
-              service:
-                name: {{ $uiName }}
-                port:
-                  number: {{ $uiPort }}
-          # Next.js App Router (output: "export", basePath: "") emits the
-          # RSC/flight payload for every route as a ROOT-level <route>.txt
-          # (/index.txt, /teams.txt, /__next._tree.txt, ...). The client
-          # router fetches these on every soft navigation / prefetch as
-          # <route>.txt?_rsc=<hash> (the query string is irrelevant to path
-          # matching). They are not under /ui, /_next, or
-          # /litellm-asset-prefix, so without this rule they fall to the
-          # backend catch-all → 404 → client-side navigation never settles
-          # and the login flow spins in an infinite redirect loop
-          # (/ ⇄ /ui/login). ui/nginx.conf already serves *.txt from the
-          # export; this rule only routes the request to it. Needs an
-          # ingress controller whose ImplementationSpecific path is a
-          # wildcard pattern (AWS ALB: `*` = 0+ chars); this chart targets
-          # the AWS Load Balancer Controller.
-          - path: /*.txt
-            pathType: ImplementationSpecific
+          {{- range $uiPaths }}
+          - path: {{ .path }}
+            pathType: {{ .pathType }}
             backend:
               service:
                 name: {{ $uiName }}
                 port:
                   number: {{ $uiPort }}
+          {{- end }}
           # --- Gateway data plane ---
           # Exact /test only (see the $gatewayPrefixes comment above);
           # /test/* MCP management endpoints fall to the backend catch-all.
@@ -142,6 +142,46 @@ spec:
                 port:
                   number: {{ $gatewayPort }}
           {{- end }}
+          {{- /*
+            --- Operator-supplied extra paths (ingress.extraPaths) ---
+            Rendered after every built-in path so an entry can never take
+            precedence over a default, and before the backend catch-all.
+            Position only decides the match on controllers that honour manifest
+            order: the AWS Load Balancer Controller this chart targets sorts
+            Exact paths first and Prefix paths longest-first, but keeps
+            ImplementationSpecific paths in manifest order, which is what the
+            /*.txt rule above already depends on.
+          */}}
+          {{- range $idx, $extra := .Values.ingress.extraPaths }}
+          {{- if not (kindIs "map" $extra) }}
+          {{- fail (printf "ingress.extraPaths[%d]: each entry must be a mapping with a 'path' key" $idx) }}
+          {{- end }}
+          {{- if not $extra.path }}
+          {{- fail (printf "ingress.extraPaths[%d]: 'path' is required" $idx) }}
+          {{- end }}
+          {{- $service := $extra.service | default "gateway" }}
+          {{- $target := get $extraPathBackends $service }}
+          {{- if not $target }}
+          {{- fail (printf "ingress.extraPaths[%d] (path %s): unknown service %q, expected one of backend, gateway, ui" $idx $extra.path $service) }}
+          {{- end }}
+          {{- $pathType := $extra.pathType | default "Prefix" }}
+          {{- if not (has $pathType (list "Prefix" "Exact" "ImplementationSpecific")) }}
+          {{- fail (printf "ingress.extraPaths[%d] (path %s): unknown pathType %q, expected one of Exact, ImplementationSpecific, Prefix" $idx $extra.path $pathType) }}
+          {{- end }}
+          {{- if eq $extra.path "/" }}
+          {{- fail (printf "ingress.extraPaths[%d]: path / is already routed in both directions, Exact to ui and Prefix to backend, so no pathType leaves a request for an entry here to capture" $idx) }}
+          {{- end }}
+          {{- if has (printf "%s|%s" $extra.path $pathType) $builtinPathKeys }}
+          {{- fail (printf "ingress.extraPaths[%d]: path %s with pathType %s is already routed by this chart, and a duplicate would take it over rather than add to it" $idx $extra.path $pathType) }}
+          {{- end }}
+          - path: {{ $extra.path | quote }}
+            pathType: {{ $pathType }}
+            backend:
+              service:
+                name: {{ $target.name }}
+                port:
+                  number: {{ $target.port }}
+          {{- end }}
           # --- Catch-all → backend (management API: /key/*, /user/*, /team/*, ...) ---
           - path: /
             pathType: Prefix

helm/litellm/tests/ingress_extra_paths_tests.yaml

@@ -0,0 +1,317 @@
+suite: test ingress.extraPaths
+templates:
+  - ingress.yaml
+values:
+  - ./values/required.yaml
+tests:
+  - it: renders nothing extra between the built-in gateway prefixes and the backend catch-all when unset
+    set:
+      ingress.enabled: true
+    asserts:
+      - equal:
+          path: spec.rules[0].http.paths[-1]
+          value:
+            path: /
+            pathType: Prefix
+            backend:
+              service:
+                name: RELEASE-NAME-litellm-backend
+                port:
+                  number: 4001
+      - equal:
+          path: spec.rules[0].http.paths[-2]
+          value:
+            path: /metrics
+            pathType: Prefix
+            backend:
+              service:
+                name: RELEASE-NAME-litellm-gateway
+                port:
+                  number: 4000
+
+  - it: routes an extra path to the gateway by default, immediately before the backend catch-all
+    set:
+      ingress.enabled: true
+      ingress.extraPaths:
+        - path: /watsonx
+    asserts:
+      - equal:
+          path: spec.rules[0].http.paths[-2]
+          value:
+            path: /watsonx
+            pathType: Prefix
+            backend:
+              service:
+                name: RELEASE-NAME-litellm-gateway
+                port:
+                  number: 4000
+      - equal:
+          path: spec.rules[0].http.paths[-1]
+          value:
+            path: /
+            pathType: Prefix
+            backend:
+              service:
+                name: RELEASE-NAME-litellm-backend
+                port:
+                  number: 4001
+
+  - it: keeps every built-in path when extra paths are supplied
+    set:
+      ingress.enabled: true
+      ingress.extraPaths:
+        - path: /watsonx
+    asserts:
+      - contains:
+          path: spec.rules[0].http.paths
+          content:
+            path: /
+            pathType: Exact
+            backend:
+              service:
+                name: RELEASE-NAME-litellm-ui
+                port:
+                  number: 3000
+      - contains:
+          path: spec.rules[0].http.paths
+          content:
+            path: /ui
+            pathType: Prefix
+            backend:
+              service:
+                name: RELEASE-NAME-litellm-ui
+                port:
+                  number: 3000
+      - contains:
+          path: spec.rules[0].http.paths
+          content:
+            path: /test
+            pathType: Exact
+            backend:
+              service:
+                name: RELEASE-NAME-litellm-gateway
+                port:
+                  number: 4000
+      - contains:
+          path: spec.rules[0].http.paths
+          content:
+            path: /v1/chat
+            pathType: Prefix
+            backend:
+              service:
+                name: RELEASE-NAME-litellm-gateway
+                port:
+                  number: 4000
+      - contains:
+          path: spec.rules[0].http.paths
+          content:
+            path: /vertex_ai
+            pathType: Prefix
+            backend:
+              service:
+                name: RELEASE-NAME-litellm-gateway
+                port:
+                  number: 4000
+
+  - it: renders every entry in order and honours the service and pathType selectors
+    set:
+      ingress.enabled: true
+      ingress.extraPaths:
+        - path: /watsonx
+          service: gateway
+        - path: /my-passthrough
+          pathType: Exact
+          service: backend
+        - path: /brand.txt
+          pathType: ImplementationSpecific
+          service: ui
+    asserts:
+      - equal:
+          path: spec.rules[0].http.paths[-4]
+          value:
+            path: /watsonx
+            pathType: Prefix
+            backend:
+              service:
+                name: RELEASE-NAME-litellm-gateway
+                port:
+                  number: 4000
+      - equal:
+          path: spec.rules[0].http.paths[-3]
+          value:
+            path: /my-passthrough
+            pathType: Exact
+            backend:
+              service:
+                name: RELEASE-NAME-litellm-backend
+                port:
+                  number: 4001
+      - equal:
+          path: spec.rules[0].http.paths[-2]
+          value:
+            path: /brand.txt
+            pathType: ImplementationSpecific
+            backend:
+              service:
+                name: RELEASE-NAME-litellm-ui
+                port:
+                  number: 3000
+
+  - it: addresses the component services by their configured ports
+    set:
+      ingress.enabled: true
+      gateway.service.port: 8000
+      backend.service.port: 8001
+      ui.service.port: 8080
+      ingress.extraPaths:
+        - path: /watsonx
+        - path: /my-passthrough
+          service: backend
+        - path: /brand.txt
+          service: ui
+    asserts:
+      - equal:
+          path: spec.rules[0].http.paths[-4].backend.service.port.number
+          value: 8000
+      - equal:
+          path: spec.rules[0].http.paths[-3].backend.service.port.number
+          value: 8001
+      - equal:
+          path: spec.rules[0].http.paths[-2].backend.service.port.number
+          value: 8080
+
+  - it: rejects an entry naming a service the chart does not deploy
+    set:
+      ingress.enabled: true
+      ingress.extraPaths:
+        - path: /watsonx
+          service: proxy
+    asserts:
+      - failedTemplate:
+          errorMessage: 'ingress.extraPaths[0] (path /watsonx): unknown service "proxy", expected one of backend, gateway, ui'
+
+  - it: rejects an entry whose pathType is not a kubernetes pathType
+    set:
+      ingress.enabled: true
+      ingress.extraPaths:
+        - path: /watsonx
+          pathType: prefix
+    asserts:
+      - failedTemplate:
+          errorMessage: 'ingress.extraPaths[0] (path /watsonx): unknown pathType "prefix", expected one of Exact, ImplementationSpecific, Prefix'
+
+  - it: rejects an entry with no path
+    set:
+      ingress.enabled: true
+      ingress.extraPaths:
+        - service: gateway
+    asserts:
+      - failedTemplate:
+          errorMessage: "ingress.extraPaths[0]: 'path' is required"
+
+
+  - it: rejects a root entry that would take over the backend catch-all
+    set:
+      ingress.enabled: true
+      ingress.extraPaths:
+        - path: /
+          service: gateway
+    asserts:
+      - failedTemplate:
+          errorMessage: "ingress.extraPaths[0]: path / is already routed in both directions, Exact to ui and Prefix to backend, so no pathType leaves a request for an entry here to capture"
+
+  - it: rejects a root entry that would take over the UI root
+    set:
+      ingress.enabled: true
+      ingress.extraPaths:
+        - path: /
+          pathType: Exact
+          service: gateway
+    asserts:
+      - failedTemplate:
+          errorMessage: "ingress.extraPaths[0]: path / is already routed in both directions, Exact to ui and Prefix to backend, so no pathType leaves a request for an entry here to capture"
+
+  # A root ImplementationSpecific entry duplicates no built-in pair, so the
+  # duplicate check alone would admit it. It is still dead: the built-in
+  # Exact / sorts ahead of it on the AWS Load Balancer Controller and claims
+  # the only request its pattern matches, so it renders and never routes.
+  - it: rejects a root entry that would render but never match
+    set:
+      ingress.enabled: true
+      ingress.extraPaths:
+        - path: /
+          pathType: ImplementationSpecific
+          service: gateway
+    asserts:
+      - failedTemplate:
+          errorMessage: "ingress.extraPaths[0]: path / is already routed in both directions, Exact to ui and Prefix to backend, so no pathType leaves a request for an entry here to capture"
+
+  - it: rejects an entry that would take over a UI prefix
+    set:
+      ingress.enabled: true
+      ingress.extraPaths:
+        - path: /ui
+          service: gateway
+    asserts:
+      - failedTemplate:
+          errorMessage: "ingress.extraPaths[0]: path /ui with pathType Prefix is already routed by this chart, and a duplicate would take it over rather than add to it"
+
+  - it: rejects an entry that would take over the UI RSC payload rule
+    set:
+      ingress.enabled: true
+      ingress.extraPaths:
+        - path: /*.txt
+          pathType: ImplementationSpecific
+          service: backend
+    asserts:
+      - failedTemplate:
+          errorMessage: "ingress.extraPaths[0]: path /*.txt with pathType ImplementationSpecific is already routed by this chart, and a duplicate would take it over rather than add to it"
+
+  - it: rejects an entry that would take over a gateway data-plane prefix
+    set:
+      ingress.enabled: true
+      ingress.extraPaths:
+        - path: /v1/chat
+          service: backend
+    asserts:
+      - failedTemplate:
+          errorMessage: "ingress.extraPaths[0]: path /v1/chat with pathType Prefix is already routed by this chart, and a duplicate would take it over rather than add to it"
+
+  - it: rejects an entry that would take over the exact /test route
+    set:
+      ingress.enabled: true
+      ingress.extraPaths:
+        - path: /test
+          pathType: Exact
+          service: backend
+    asserts:
+      - failedTemplate:
+          errorMessage: "ingress.extraPaths[0]: path /test with pathType Exact is already routed by this chart, and a duplicate would take it over rather than add to it"
+
+  - it: allows a built-in path under a different pathType, which is a distinct rule
+    set:
+      ingress.enabled: true
+      ingress.extraPaths:
+        - path: /ui
+          pathType: Exact
+          service: ui
+    asserts:
+      - equal:
+          path: spec.rules[0].http.paths[-2]
+          value:
+            path: /ui
+            pathType: Exact
+            backend:
+              service:
+                name: RELEASE-NAME-litellm-ui
+                port:
+                  number: 3000
+
+  - it: rejects a bare string entry instead of failing on template internals
+    set:
+      ingress.enabled: true
+      ingress.extraPaths:
+        - /watsonx
+    asserts:
+      - failedTemplate:
+          errorMessage: "ingress.extraPaths[0]: each entry must be a mapping with a 'path' key"

helm/litellm/values.yaml

@@ -13,6 +13,27 @@ ingress:
   annotations: {}
   host: ""        # optional; if set, becomes the rule's host
   tls: []
+  # Extra HTTP paths appended to the ingress rule. Additive: every built-in
+  # UI / gateway / backend path is still rendered, these entries are placed
+  # after them and before the backend catch-all, and an entry that repeats a
+  # path the chart already routes is rejected at render time rather than
+  # silently taking it over.
+  #
+  # The chart's built-in gateway prefix list is a snapshot of the data-plane
+  # surface at release time. Use extraPaths for passthrough routes it does not
+  # cover: a provider prefix added upstream after this chart version, or a
+  # custom general_settings.pass_through_endpoints route.
+  #
+  #   path      required; the HTTP path to route
+  #   service   which component serves it: gateway (default), backend, or ui
+  #   pathType  Prefix (default), Exact, or ImplementationSpecific
+  #
+  # The target component only answers paths its own route allowlist keeps, so
+  # a path here still has to be one that component serves.
+  extraPaths: []
+  # - path: /watsonx
+  #   pathType: Prefix
+  #   service: gateway
 
 # Per-component ServiceAccounts for gateway, backend, and ui.
 #

litellm-proxy-extras/litellm_proxy_extras/migrations/20260804162853_add_budget_window_spend_table/migration.sql

@@ -0,0 +1,12 @@
+CREATE TABLE IF NOT EXISTS "LiteLLM_BudgetWindowSpend" (
+    "entity_type" TEXT NOT NULL,
+    "entity_id" TEXT NOT NULL,
+    "window_duration" TEXT NOT NULL,
+    "window_start" TIMESTAMP(3) NOT NULL,
+    "spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0,
+    "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updated_at" TIMESTAMP(3) NOT NULL,
+
+    CONSTRAINT "LiteLLM_BudgetWindowSpend_pkey" PRIMARY KEY ("entity_type","entity_id","window_duration")
+);
+

litellm-proxy-extras/litellm_proxy_extras/migrations/20260829000000_add_model_access_group_budget_table/migration.sql

@@ -0,0 +1,20 @@
+-- CreateTable
+CREATE TABLE IF NOT EXISTS "LiteLLM_ModelAccessGroupBudgetTable" (
+    "access_group_name" TEXT NOT NULL,
+    "spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0,
+    "budget_id" TEXT,
+    "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "created_by" TEXT,
+    "updated_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updated_by" TEXT,
+
+    CONSTRAINT "LiteLLM_ModelAccessGroupBudgetTable_pkey" PRIMARY KEY ("access_group_name")
+);
+
+-- AddForeignKey
+DO $$
+BEGIN
+    IF NOT EXISTS (SELECT 1 FROM pg_constraint WHERE conname = 'LiteLLM_ModelAccessGroupBudgetTable_budget_id_fkey') THEN
+        ALTER TABLE "LiteLLM_ModelAccessGroupBudgetTable" ADD CONSTRAINT "LiteLLM_ModelAccessGroupBudgetTable_budget_id_fkey" FOREIGN KEY ("budget_id") REFERENCES "LiteLLM_BudgetTable"("budget_id") ON DELETE SET NULL ON UPDATE CASCADE;
+    END IF;
+END $$;

litellm-proxy-extras/litellm_proxy_extras/schema.prisma

@@ -29,6 +29,7 @@ model LiteLLM_BudgetTable {
   keys LiteLLM_VerificationToken[] // multiple keys can have the same budget
   end_users LiteLLM_EndUserTable[] // multiple end-users can have the same budget
   tags LiteLLM_TagTable[] // multiple tags can have the same budget
+  model_access_groups LiteLLM_ModelAccessGroupBudgetTable[] // multiple model access groups can have the same budget
   team_membership LiteLLM_TeamMembership[] // budgets of Users within a Team 
   organization_membership LiteLLM_OrganizationMembership[] // budgets of Users within a Organization
 }
@@ -585,6 +586,20 @@ model LiteLLM_EndUserTable {
   blocked Boolean @default(false)
 }
 
+// Budget and shared spend for a model access group. The groups themselves are not rows anywhere:
+// they are free-text strings in LiteLLM_ProxyModelTable.model_info.access_groups, so a row here
+// exists only once someone gives that group a budget.
+model LiteLLM_ModelAccessGroupBudgetTable {
+  access_group_name String @id
+  spend Float @default(0.0)
+  budget_id String?
+  litellm_budget_table LiteLLM_BudgetTable? @relation(fields: [budget_id], references: [budget_id])
+  created_at DateTime @default(now()) @map("created_at")
+  created_by String?
+  updated_at DateTime @default(now()) @updatedAt
+  updated_by String?
+}
+
 // Track tags with budgets and spend
 model LiteLLM_TagTable {
   tag_name String @id
@@ -649,6 +664,18 @@ model LiteLLM_SpendLogs {
   @@index([session_id])
 }
 
+model LiteLLM_BudgetWindowSpend {
+  entity_type     String
+  entity_id       String
+  window_duration String
+  window_start    DateTime
+  spend           Float    @default(0.0)
+  created_at      DateTime @default(now())
+  updated_at      DateTime @updatedAt
+
+  @@id([entity_type, entity_id, window_duration])
+}
+
 // View spend, model, api_key per request
 model LiteLLM_ErrorLogs {
   request_id          String   @id @default(uuid())

litellm-proxy-extras/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "litellm-proxy-extras"
-version = "0.4.90"
+version = "0.4.91"
 description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package."
 readme = "README.md"
 requires-python = ">=3.9"
@@ -26,7 +26,7 @@ required-version = ">=0.10.9"
 module-root = ""
 
 [tool.commitizen]
-version = "0.4.90"
+version = "0.4.91"
 version_files = [
     "pyproject.toml:^version",
     "../pyproject.toml:litellm-proxy-extras==",

litellm-rust/Cargo.lock

@@ -2,6 +2,36 @@
 # It is not intended for manual editing.
 version = 4
 
+[[package]]
+name = "aho-corasick"
+version = "1.1.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba"
+dependencies = [
+ "memchr",
+]
+
+[[package]]
+name = "alloca"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e5a7d05ea6aea7e9e64d25b9156ba2fee3fdd659e34e41063cd2fc7cd020d7f4"
+dependencies = [
+ "cc",
+]
+
+[[package]]
+name = "anes"
+version = "0.1.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299"
+
+[[package]]
+name = "anstyle"
+version = "1.0.14"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000"
+
 [[package]]
 name = "arc-swap"
 version = "1.9.2"
@@ -506,6 +536,12 @@ dependencies = [
  "either",
 ]
 
+[[package]]
+name = "cast"
+version = "0.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5"
+
 [[package]]
 name = "cc"
 version = "1.3.0"
@@ -541,6 +577,58 @@ dependencies = [
  "rand_core 0.10.1",
 ]
 
+[[package]]
+name = "ciborium"
+version = "0.2.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "42e69ffd6f0917f5c029256a24d0161db17cea3997d185db0d35926308770f0e"
+dependencies = [
+ "ciborium-io",
+ "ciborium-ll",
+ "serde",
+]
+
+[[package]]
+name = "ciborium-io"
+version = "0.2.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "05afea1e0a06c9be33d539b876f1ce3692f4afea2cb41f740e7743225ed1c757"
+
+[[package]]
+name = "ciborium-ll"
+version = "0.2.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "57663b653d948a338bfb3eeba9bb2fd5fcfaecb9e199e87e1eda4d9e8b240fd9"
+dependencies = [
+ "ciborium-io",
+ "half",
+]
+
+[[package]]
+name = "clap"
+version = "4.6.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "473c7e07f409a8d772161724aa8db6a765a2532a70f9667eeb7b49d3d02fbdca"
+dependencies = [
+ "clap_builder",
+]
+
+[[package]]
+name = "clap_builder"
+version = "4.6.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7b48fea5a88e9ae728a2dcbedbfc0e730f7d60da42e1cb049a83c9fb8b789889"
+dependencies = [
+ "anstyle",
+ "clap_lex",
+]
+
+[[package]]
+name = "clap_lex"
+version = "1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9"
+
 [[package]]
 name = "cmake"
 version = "0.1.58"
@@ -596,6 +684,72 @@ dependencies = [
  "libc",
 ]
 
+[[package]]
+name = "criterion"
+version = "0.8.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "950046b2aa2492f9a536f5f4f9a3de7b9e2476e575e05bd6c333371add4d98f3"
+dependencies = [
+ "alloca",
+ "anes",
+ "cast",
+ "ciborium",
+ "clap",
+ "criterion-plot",
+ "itertools",
+ "num-traits",
+ "oorandom",
+ "page_size",
+ "plotters",
+ "rayon",
+ "regex",
+ "serde",
+ "serde_json",
+ "tinytemplate",
+ "walkdir",
+]
+
+[[package]]
+name = "criterion-plot"
+version = "0.8.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d8d80a2f4f5b554395e47b5d8305bc3d27813bacb73493eb1001e8f76dae29ea"
+dependencies = [
+ "cast",
+ "itertools",
+]
+
+[[package]]
+name = "crossbeam-deque"
+version = "0.8.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5181e0de7b61eb03a81e347d6dd8797bae9da5146707b51077e2d71a54ec0ceb"
+dependencies = [
+ "crossbeam-epoch",
+ "crossbeam-utils",
+]
+
+[[package]]
+name = "crossbeam-epoch"
+version = "0.9.20"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2d6914041f254d6e9176c01941b21115dcfb7089e55135a35411081bd106ef3f"
+dependencies = [
+ "crossbeam-utils",
+]
+
+[[package]]
+name = "crossbeam-utils"
+version = "0.8.22"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "61803da095bee82a81bb1a452ecc25d3b2f1416d1897eb86430c6159ef717c17"
+
+[[package]]
+name = "crunchy"
+version = "0.2.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5"
+
 [[package]]
 name = "crypto-common"
 version = "0.1.7"
@@ -856,6 +1010,17 @@ dependencies = [
  "tracing",
 ]
 
+[[package]]
+name = "half"
+version = "2.7.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b"
+dependencies = [
+ "cfg-if",
+ "crunchy",
+ "zerocopy",
+]
+
 [[package]]
 name = "hashbrown"
 version = "0.17.1"
@@ -1179,6 +1344,15 @@ version = "2.12.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d98f6fed1fde3f8c21bc40a1abb88dd75e67924f9cffc3ef95607bad8017f8e2"
 
+[[package]]
+name = "itertools"
+version = "0.13.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186"
+dependencies = [
+ "either",
+]
+
 [[package]]
 name = "itoa"
 version = "1.0.18"
@@ -1255,10 +1429,13 @@ dependencies = [
 name = "litellm-python-bridge"
 version = "0.1.0"
 dependencies = [
+ "criterion",
  "litellm-ai-gateway",
  "litellm-core",
  "pyo3",
  "pyo3-async-runtimes",
+ "pythonize",
+ "serde",
  "serde_json",
  "tokio",
 ]
@@ -1340,6 +1517,12 @@ version = "1.21.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50"
 
+[[package]]
+name = "oorandom"
+version = "11.1.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e"
+
 [[package]]
 name = "openssl-probe"
 version = "0.2.1"
@@ -1352,6 +1535,16 @@ version = "0.5.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "1a80800c0488c3a21695ea981a54918fbb37abf04f4d0720c453632255e2ff0e"
 
+[[package]]
+name = "page_size"
+version = "0.6.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "30d5b2194ed13191c1999ae0704b7839fb18384fa22e49b57eeaa97d79ce40da"
+dependencies = [
+ "libc",
+ "winapi",
+]
+
 [[package]]
 name = "percent-encoding"
 version = "2.3.2"
@@ -1376,6 +1569,34 @@ version = "0.3.33"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e"
 
+[[package]]
+name = "plotters"
+version = "0.3.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5aeb6f403d7a4911efb1e33402027fc44f29b5bf6def3effcc22d7bb75f2b747"
+dependencies = [
+ "num-traits",
+ "plotters-backend",
+ "plotters-svg",
+ "wasm-bindgen",
+ "web-sys",
+]
+
+[[package]]
+name = "plotters-backend"
+version = "0.3.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "df42e13c12958a16b3f7f4386b9ab1f3e7933914ecea48da7139435263a4172a"
+
+[[package]]
+name = "plotters-svg"
+version = "0.3.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "51bae2ac328883f7acdfea3d66a7c35751187f870bc81f94563733a154d7a670"
+dependencies = [
+ "plotters-backend",
+]
+
 [[package]]
 name = "portable-atomic"
 version = "1.14.0"
@@ -1486,6 +1707,16 @@ dependencies = [
  "syn 2.0.119",
 ]
 
+[[package]]
+name = "pythonize"
+version = "0.29.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6ec376e1216e0c929a74964ce2020012a1a39f32d80e78aa688721219ea7fb89"
+dependencies = [
+ "pyo3",
+ "serde",
+]
+
 [[package]]
 name = "quinn"
 version = "0.11.11"
@@ -1613,12 +1844,61 @@ dependencies = [
  "rand_core 0.10.1",
 ]
 
+[[package]]
+name = "rayon"
+version = "1.12.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fb39b166781f92d482534ef4b4b1b2568f42613b53e5b6c160e24cfbfa30926d"
+dependencies = [
+ "either",
+ "rayon-core",
+]
+
+[[package]]
+name = "rayon-core"
+version = "1.13.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91"
+dependencies = [
+ "crossbeam-deque",
+ "crossbeam-utils",
+]
+
+[[package]]
+name = "regex"
+version = "1.13.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d"
+dependencies = [
+ "aho-corasick",
+ "memchr",
+ "regex-automata",
+ "regex-syntax",
+]
+
+[[package]]
+name = "regex-automata"
+version = "0.4.18"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2"
+dependencies = [
+ "aho-corasick",
+ "memchr",
+ "regex-syntax",
+]
+
 [[package]]
 name = "regex-lite"
 version = "0.1.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "cab834c73d247e67f4fae452806d17d3c7501756d98c8808d7c9c7aa7d18f973"
 
+[[package]]
+name = "regex-syntax"
+version = "0.8.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4"
+
 [[package]]
 name = "reqwest"
 version = "0.12.28"
@@ -1774,6 +2054,15 @@ version = "1.0.23"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f"
 
+[[package]]
+name = "same-file"
+version = "1.0.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502"
+dependencies = [
+ "winapi-util",
+]
+
 [[package]]
 name = "schannel"
 version = "0.1.29"
@@ -2099,6 +2388,16 @@ dependencies = [
  "zerovec",
 ]
 
+[[package]]
+name = "tinytemplate"
+version = "1.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "be4d6b5f19ff7664e8c98d03e2139cb510db9b0a60b55f8e8709b689d939b6bc"
+dependencies = [
+ "serde",
+ "serde_json",
+]
+
 [[package]]
 name = "tinyvec"
 version = "1.12.0"
@@ -2363,6 +2662,16 @@ version = "0.8.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "5c3082ca00d5a5ef149bb8b555a72ae84c9c59f7250f013ac822ac2e49b19c64"
 
+[[package]]
+name = "walkdir"
+version = "2.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b"
+dependencies = [
+ "same-file",
+ "winapi-util",
+]
+
 [[package]]
 name = "want"
 version = "0.3.1"
@@ -2475,6 +2784,37 @@ dependencies = [
  "rustls-pki-types",
 ]
 
+[[package]]
+name = "winapi"
+version = "0.3.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419"
+dependencies = [
+ "winapi-i686-pc-windows-gnu",
+ "winapi-x86_64-pc-windows-gnu",
+]
+
+[[package]]
+name = "winapi-i686-pc-windows-gnu"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6"
+
+[[package]]
+name = "winapi-util"
+version = "0.1.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22"
+dependencies = [
+ "windows-sys 0.61.2",
+]
+
+[[package]]
+name = "winapi-x86_64-pc-windows-gnu"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
+
 [[package]]
 name = "windows-link"
 version = "0.2.1"

litellm-rust/Cargo.toml

@@ -18,6 +18,7 @@ litellm-ai-gateway = { path = "crates/ai-gateway", default-features = false }
 axum = "0.7"
 pyo3 = "0.29.0"
 pyo3-async-runtimes = { version = "0.29.0", features = ["tokio-runtime"] }
+pythonize = "0.29.0"
 rand = "0.8"
 reqwest = { version = "0.12", default-features = false, features = ["blocking", "json", "rustls-tls", "http2", "stream"] }
 serde = { version = "1.0", features = ["derive"] }

litellm-rust/crates/python-bridge/Cargo.toml

@@ -9,10 +9,23 @@ repository.workspace = true
 name = "_native"
 crate-type = ["cdylib"]
 
+[features]
+default = ["extension-module"]
+extension-module = ["pyo3/extension-module"]
+
 [dependencies]
 litellm-core = { workspace = true, features = ["bedrock-auth"] }
 litellm-ai-gateway = { workspace = true, default-features = false }
-pyo3 = { workspace = true, features = ["extension-module"] }
+pyo3.workspace = true
 pyo3-async-runtimes.workspace = true
+pythonize.workspace = true
+serde.workspace = true
 serde_json.workspace = true
 tokio.workspace = true
+
+[dev-dependencies]
+criterion = "0.8.2"
+
+[[bench]]
+name = "serialization"
+harness = false

litellm-rust/crates/python-bridge/benches/serialization.rs

@@ -0,0 +1,103 @@
+use std::hint::black_box;
+use std::time::Duration;
+
+use criterion::{BenchmarkId, Criterion, criterion_group, criterion_main};
+use pyo3::prelude::*;
+use pyo3::types::PyDict;
+use serde_json::{Value, json};
+
+const PAYLOAD_SIZES: &[(&str, usize)] = &[
+    ("1_KiB", 1024),
+    ("64_KiB", 64 * 1024),
+    ("1_MiB", 1024 * 1024),
+    ("4_MiB", 4 * 1024 * 1024),
+    ("16_MiB", 16 * 1024 * 1024),
+];
+
+fn former_json_roundtrip_from_py(py: Python<'_>, value: &Bound<'_, PyAny>) -> Value {
+    let json = py.import("json").expect("Python json module should import");
+    let encoded: String = json
+        .call_method1("dumps", (value,))
+        .expect("payload should serialize")
+        .extract()
+        .expect("json.dumps should return a string");
+    serde_json::from_str(&encoded).expect("serialized JSON should parse")
+}
+
+fn pythonize_from_py(value: &Bound<'_, PyAny>) -> Value {
+    pythonize::depythonize(value).expect("payload should depythonize")
+}
+
+fn former_json_roundtrip_to_py(py: Python<'_>, value: &Value) -> Py<PyAny> {
+    let json = py.import("json").expect("Python json module should import");
+    let encoded = serde_json::to_string(value).expect("response should serialize");
+    json.call_method1("loads", (encoded,))
+        .expect("serialized response should parse in Python")
+        .unbind()
+}
+
+fn pythonize_to_py(py: Python<'_>, value: &Value) -> Py<PyAny> {
+    pythonize::pythonize(py, value)
+        .expect("response should pythonize")
+        .unbind()
+}
+
+fn serialization(c: &mut Criterion) {
+    Python::initialize();
+    Python::attach(|py| {
+        for &(label, payload_bytes) in PAYLOAD_SIZES {
+            let data_uri = format!("data:image/png;base64,{}", "A".repeat(payload_bytes));
+            let document = PyDict::new(py);
+            document
+                .set_item("type", "image_url")
+                .expect("document type should be set");
+            document
+                .set_item("image_url", &data_uri)
+                .expect("document URL should be set");
+            let response = json!({
+                "pages": [{
+                    "index": 0,
+                    "markdown": "OCR text",
+                    "images": [{"image_base64": data_uri}],
+                }],
+                "model": "mistral-ocr-latest",
+                "document_annotation": null,
+                "usage_info": {"pages_processed": 1},
+                "object": "ocr",
+            });
+
+            c.bench_with_input(
+                BenchmarkId::new("python_to_rust_json", label),
+                &document,
+                |b, document| {
+                    b.iter(|| former_json_roundtrip_from_py(py, black_box(document.as_any())))
+                },
+            );
+            c.bench_with_input(
+                BenchmarkId::new("python_to_rust_pythonize", label),
+                &document,
+                |b, document| b.iter(|| pythonize_from_py(black_box(document.as_any()))),
+            );
+            c.bench_with_input(
+                BenchmarkId::new("rust_to_python_json", label),
+                &response,
+                |b, response| b.iter(|| former_json_roundtrip_to_py(py, black_box(response))),
+            );
+            c.bench_with_input(
+                BenchmarkId::new("rust_to_python_pythonize", label),
+                &response,
+                |b, response| b.iter(|| pythonize_to_py(py, black_box(response))),
+            );
+        }
+    });
+}
+
+criterion_group! {
+    name = benches;
+    config = Criterion::default()
+        .sample_size(20)
+        .warm_up_time(Duration::from_secs(1))
+        .measurement_time(Duration::from_secs(4));
+    targets = serialization
+}
+criterion_main!(benches);

litellm-rust/crates/python-bridge/src/lib.rs

@@ -19,6 +19,9 @@ use pyo3::types::{PyAny, PyDict};
 use serde_json::{Map, Value};
 
 mod gil;
+mod marshal;
+
+use marshal::{from_py, to_py};
 
 pyo3::create_exception!(
     _native,
@@ -41,35 +44,18 @@ type MarshaledOcrInputs = (
     Option<Duration>,
 );
 
-fn py_to_json(py: Python<'_>, value: &Bound<'_, PyAny>) -> PyResult<Value> {
-    let json = py.import("json")?;
-    let encoded: String = json.call_method1("dumps", (value,))?.extract()?;
-    serde_json::from_str(&encoded).map_err(|err| PyValueError::new_err(err.to_string()))
-}
-
-fn json_to_py(py: Python<'_>, value: Value) -> PyResult<Py<PyAny>> {
-    let json = py.import("json")?;
-    let encoded =
-        serde_json::to_string(&value).map_err(|err| PyValueError::new_err(err.to_string()))?;
-    Ok(json.call_method1("loads", (encoded,))?.unbind())
-}
-
 fn messages_response_to_py(
     py: Python<'_>,
     response: AnthropicMessagesResponse,
 ) -> PyResult<Py<PyAny>> {
-    let value =
-        serde_json::to_value(response).map_err(|err| PyValueError::new_err(err.to_string()))?;
-    json_to_py(py, value)
+    to_py(py, &response)
 }
 
 fn chat_completions_response_to_py(
     py: Python<'_>,
     response: ChatCompletionsResponse,
 ) -> PyResult<Py<PyAny>> {
-    let value =
-        serde_json::to_value(response).map_err(|err| PyValueError::new_err(err.to_string()))?;
-    json_to_py(py, value)
+    to_py(py, &response)
 }
 
 fn core_error_to_pyerr(err: CoreError) -> PyErr {
@@ -116,7 +102,7 @@ fn optional_object_to_map(
     value: Option<Py<PyAny>>,
 ) -> PyResult<Map<String, Value>> {
     match value {
-        Some(value) => match py_to_json(py, value.bind(py))? {
+        Some(value) => match from_py(value.bind(py))? {
             Value::Object(map) => Ok(map),
             _ => Err(PyValueError::new_err(format!("{name} must be a dict"))),
         },
@@ -139,7 +125,7 @@ fn marshal_headers(
     headers: Option<Py<PyAny>>,
 ) -> PyResult<HashMap<String, String>> {
     let value = match headers {
-        Some(headers) => py_to_json(py, headers.bind(py))?,
+        Some(headers) => from_py(headers.bind(py))?,
         None => Value::Object(Map::new()),
     };
     let Value::Object(headers) = value else {
@@ -211,7 +197,7 @@ fn marshal_inputs(
     optional_params: Option<Py<PyAny>>,
     timeout_seconds: Option<f64>,
 ) -> PyResult<MarshaledOcrInputs> {
-    let document = py_to_json(py, document.bind(py))?;
+    let document = from_py(document.bind(py))?;
     let extra_headers = match extra_headers {
         Some(headers) => Some(optional_object_to_map(py, "extra_headers", Some(headers))?),
         None => None,
@@ -262,7 +248,7 @@ fn ocr(
     });
 
     match result {
-        Ok(value) => json_to_py(py, value),
+        Ok(value) => to_py(py, &value),
         Err(err) => Err(core_error_to_pyerr(err)),
     }
 }
@@ -307,7 +293,7 @@ fn aocr(
         .await
         .map_err(core_error_to_pyerr)?;
 
-        Python::attach(|py| json_to_py(py, value))
+        Python::attach(|py| to_py(py, &value))
     })
 }
 
@@ -325,7 +311,7 @@ fn transcription(
     optional_params: Option<Py<PyAny>>,
     timeout_seconds: Option<f64>,
 ) -> PyResult<Py<PyAny>> {
-    let audio = py_to_json(py, audio.bind(py))?;
+    let audio = from_py(audio.bind(py))?;
     let extra_headers = match extra_headers {
         Some(headers) => Some(optional_object_to_map(py, "extra_headers", Some(headers))?),
         None => None,
@@ -351,7 +337,7 @@ fn transcription(
         ))
     });
     match result {
-        Ok(value) => json_to_py(py, value),
+        Ok(value) => to_py(py, &value),
         Err(err) => Err(core_error_to_pyerr(err)),
     }
 }
@@ -370,7 +356,7 @@ fn atranscription(
     optional_params: Option<Py<PyAny>>,
     timeout_seconds: Option<f64>,
 ) -> PyResult<Bound<'_, PyAny>> {
-    let audio = py_to_json(py, audio.bind(py))?;
+    let audio = from_py(audio.bind(py))?;
     let extra_headers = match extra_headers {
         Some(headers) => Some(optional_object_to_map(py, "extra_headers", Some(headers))?),
         None => None,
@@ -394,7 +380,7 @@ fn atranscription(
         })
         .await
         .map_err(core_error_to_pyerr)?;
-        Python::attach(|py| json_to_py(py, value))
+        Python::attach(|py| to_py(py, &value))
     })
 }
 
@@ -406,7 +392,7 @@ fn marshal_messages_inputs(
     extra_headers: Option<Py<PyAny>>,
     timeout_seconds: Option<f64>,
 ) -> PyResult<MarshaledMessagesInputs> {
-    let body = py_to_json(py, body.bind(py))?;
+    let body: Value = from_py(body.bind(py))?;
     if !body.is_object() {
         return Err(PyValueError::new_err("body must be a dict"));
     }
@@ -498,7 +484,7 @@ fn marshal_chat_completions_inputs(
     extra_headers: Option<Py<PyAny>>,
     timeout_seconds: Option<f64>,
 ) -> PyResult<MarshaledChatCompletionsInputs> {
-    let messages = py_to_json(py, messages.bind(py))?;
+    let messages: Value = from_py(messages.bind(py))?;
     if !messages.is_array() {
         return Err(PyValueError::new_err("messages must be a list"));
     }
@@ -527,7 +513,7 @@ fn chat_completions_decline(
     optional_params: Option<Py<PyAny>>,
     custom_llm_provider: Option<String>,
 ) -> PyResult<Option<String>> {
-    let messages = py_to_json(py, messages.bind(py))?;
+    let messages = from_py(messages.bind(py))?;
     let optional_params = optional_object_to_map(py, "optional_params", optional_params)?;
     Ok(chat_completions_decline_reason(
         &model,

litellm-rust/crates/python-bridge/src/marshal.rs

@@ -0,0 +1,20 @@
+use pyo3::exceptions::PyValueError;
+use pyo3::prelude::*;
+use serde::Serialize;
+use serde::de::DeserializeOwned;
+
+pub fn from_py<T>(value: &Bound<'_, PyAny>) -> PyResult<T>
+where
+    T: DeserializeOwned,
+{
+    pythonize::depythonize(value).map_err(|error| PyValueError::new_err(error.to_string()))
+}
+
+pub fn to_py<T>(py: Python<'_>, value: &T) -> PyResult<Py<PyAny>>
+where
+    T: Serialize + ?Sized,
+{
+    pythonize::pythonize(py, value)
+        .map(Bound::unbind)
+        .map_err(|error| PyValueError::new_err(error.to_string()))
+}

litellm-rust/crates/python-bridge/tests/marshal_boundary.rs

@@ -0,0 +1,52 @@
+use std::fs;
+use std::path::{Path, PathBuf};
+
+const DISALLOWED_OUTSIDE_MARSHAL: &[&str] = &[
+    "py.import(\"json\")",
+    "pythonize::",
+    "serde_json::to_string",
+    "serde_json::from_str",
+];
+
+fn source_root() -> PathBuf {
+    Path::new(env!("CARGO_MANIFEST_DIR")).join("src")
+}
+
+fn rust_sources(directory: &Path) -> Vec<PathBuf> {
+    fs::read_dir(directory)
+        .expect("bridge source directory should be readable")
+        .map(|entry| {
+            entry
+                .expect("bridge source entry should be readable")
+                .path()
+        })
+        .flat_map(|path| {
+            if path.is_dir() {
+                rust_sources(&path)
+            } else if path.extension().is_some_and(|extension| extension == "rs") {
+                vec![path]
+            } else {
+                Vec::new()
+            }
+        })
+        .collect()
+}
+
+#[test]
+fn serialization_is_centralized_in_marshal_module() {
+    let root = source_root();
+
+    for path in rust_sources(&root) {
+        if path == root.join("marshal.rs") {
+            continue;
+        }
+        let source = fs::read_to_string(&path).expect("bridge source should be readable");
+        for disallowed in DISALLOWED_OUTSIDE_MARSHAL {
+            assert!(
+                !source.contains(disallowed),
+                "{} bypasses the typed marshal module with `{disallowed}`",
+                path.display()
+            );
+        }
+    }
+}

litellm/batches/batch_utils.py

@@ -1,6 +1,8 @@
 import json
 from collections.abc import Iterable, Iterator, Mapping
 from dataclasses import dataclass
+from dataclasses import replace as dataclasses_replace
+from enum import Enum
 from typing import Any, Final, Literal
 
 import litellm
@@ -12,12 +14,23 @@ from litellm.types.utils import CallTypes, ModelInfo, Usage
 from litellm.utils import token_counter
 
 
+@dataclass(frozen=True, slots=True)
+class BatchCostUsageResult:
+    """Aggregate cost, usage, and per-line pass/fail counts for a completed batch."""
+
+    cost: float
+    usage: Usage
+    models: list[str]
+    successful_requests: int
+    failed_requests: int
+
+
 async def calculate_batch_cost_and_usage(
     file_content_dictionary: list[dict],
     custom_llm_provider: Literal["openai", "azure", "vertex_ai", "hosted_vllm", "anthropic"],
     model_name: str | None = None,
     model_info: ModelInfo | None = None,
-) -> tuple[float, Usage, list[str]]:
+) -> BatchCostUsageResult:
     """
     Calculate the cost and usage of a batch.
 
@@ -32,8 +45,7 @@ async def calculate_batch_cost_and_usage(
         and model_name
         and getattr(litellm, "disable_vertex_batch_output_transformation", False)
     ):
-        batch_cost, batch_usage = calculate_vertex_ai_batch_cost_and_usage(file_content_dictionary, model_name)
-        return batch_cost, batch_usage, [model_name]
+        return calculate_vertex_ai_batch_cost_and_usage(file_content_dictionary, model_name)
 
     return _aggregate_batch_cost_usage_models(
         entries=file_content_dictionary,
@@ -49,7 +61,7 @@ async def _handle_completed_batch(
     model_name: str | None = None,
     litellm_params: dict | None = None,
     model_info: ModelInfo | None = None,
-) -> tuple[float, Usage, list[str]]:
+) -> BatchCostUsageResult:
     """Fetch a completed batch's output file and aggregate its cost, usage, and
     models in a single pass over the JSONL lines, so the parsed file content is
     never materialized in memory.
@@ -72,27 +84,49 @@ async def _handle_completed_batch(
     # The generic retrieval helper keeps raising for callers that explicitly ask
     # for a missing output file.
     if batch.output_file_id is None:
-        return 0.0, Usage(prompt_tokens=0, completion_tokens=0, total_tokens=0), []
+        return BatchCostUsageResult(
+            cost=0.0,
+            usage=Usage(prompt_tokens=0, completion_tokens=0, total_tokens=0),
+            models=[],  # mutable-ok: no output file means no model was ever priced; BatchCostUsageResult.models requires list[str]
+            successful_requests=0,
+            failed_requests=await count_error_file_failed_requests(
+                batch, custom_llm_provider=custom_llm_provider, litellm_params=litellm_params
+            ),
+        )
 
     file_content = await _fetch_batch_output_file_content(batch, custom_llm_provider, litellm_params=litellm_params)
-
-    if (
-        custom_llm_provider == "vertex_ai"
-        and model_name
-        and getattr(litellm, "disable_vertex_batch_output_transformation", False)
-    ):
-        batch_cost, batch_usage = calculate_vertex_ai_batch_cost_and_usage(
-            _get_file_content_as_dictionary(file_content), model_name
-        )
-        return batch_cost, batch_usage, [model_name]
-
-    return _aggregate_batch_cost_usage_models(
-        entries=_iter_batch_output_entries(file_content),
-        custom_llm_provider=custom_llm_provider,
-        model_name=model_name,
-        model_info=model_info,
+    error_file_failed_requests: Final = await count_error_file_failed_requests(
+        batch, custom_llm_provider=custom_llm_provider, litellm_params=litellm_params
     )
 
+    output_file_result: Final = (
+        calculate_vertex_ai_batch_cost_and_usage(_get_file_content_as_dictionary(file_content), model_name)
+        if (
+            custom_llm_provider == "vertex_ai"
+            and model_name
+            and getattr(litellm, "disable_vertex_batch_output_transformation", False)
+        )
+        else _aggregate_batch_cost_usage_models(
+            entries=_iter_batch_output_entries(file_content),
+            custom_llm_provider=custom_llm_provider,
+            model_name=model_name,
+            model_info=model_info,
+        )
+    )
+
+    if not error_file_failed_requests:
+        return output_file_result
+    return dataclasses_replace(
+        output_file_result, failed_requests=output_file_result.failed_requests + error_file_failed_requests
+    )
+
+
+class _LineOutcome(Enum):
+    """A batch output line that yielded no billable stats."""
+
+    PROVIDER_FAILED = "provider_failed"
+    UNCOSTABLE = "uncostable"
+
 
 @dataclass(frozen=True, slots=True)
 class _BatchOutputLineStats:
@@ -102,19 +136,27 @@ class _BatchOutputLineStats:
     total_tokens: int
     cache_read_tokens: int
     cache_creation_tokens: int
+    reasoning_tokens: int
     model: str | None
 
 
-def _iter_successful_output_line_stats(
+def _classify_output_line_stats(
     entries: Iterable[dict],
     custom_llm_provider: Literal["openai", "azure", "vertex_ai", "hosted_vllm", "anthropic", "bedrock"],
     model_name: str | None,
     model_info: ModelInfo | None,
-) -> Iterator[_BatchOutputLineStats]:
+) -> Iterator[_BatchOutputLineStats | _LineOutcome]:
+    """Classify every output line in a single pass, so counting failures never needs
+    a second read of a potentially huge output file. A line the provider reported as
+    failed yields ``PROVIDER_FAILED``; a successful line litellm could not price
+    yields ``UNCOSTABLE`` and still counts as a successful request billed at $0, so
+    the counts stay reconcilable with the provider's own ``request_counts``."""
     for entry in entries:
+        if not _batch_response_was_successful(entry, custom_llm_provider):
+            yield _LineOutcome.PROVIDER_FAILED
+            continue
         stats = _safe_output_line_stats(entry, custom_llm_provider, model_name, model_info)
-        if stats is not None:
-            yield stats
+        yield stats if stats is not None else _LineOutcome.UNCOSTABLE
 
 
 def _safe_output_line_stats(
@@ -123,13 +165,11 @@ def _safe_output_line_stats(
     model_name: str | None,
     model_info: ModelInfo | None,
 ) -> _BatchOutputLineStats | None:
-    """Return the stats for one batch output line, or None for a line that is
-    unsuccessful or cannot be costed, so a single bad line never aborts the
-    whole batch's cost accounting."""
+    """Return the stats for one provider-successful batch output line, or None when
+    it cannot be costed, so a single bad line never aborts the whole batch's cost
+    accounting."""
     custom_id: Final = entry.get("custom_id") if isinstance(entry, dict) else None
     try:
-        if not _batch_response_was_successful(entry, custom_llm_provider):
-            return None
         return _compute_output_line_stats(entry, custom_llm_provider, model_name, model_info)
     except Exception as e:  # noqa: BLE001  # any single line's costing failure must not abort the whole batch
         verbose_logger.warning(
@@ -152,6 +192,7 @@ def _compute_output_line_stats(
     prompt_details: Final = parse_prompt_tokens_details(usage)
     raw_model: Final = response_body.get("model")
     response_model: Final = raw_model if isinstance(raw_model, str) and raw_model else None
+    completion_details: Final = usage.completion_tokens_details
     return _BatchOutputLineStats(
         cost=_output_line_cost(
             response_body=response_body,
@@ -166,6 +207,7 @@ def _compute_output_line_stats(
         total_tokens=usage.total_tokens,
         cache_read_tokens=prompt_details["cache_hit_tokens"],
         cache_creation_tokens=prompt_details["cache_creation_tokens"],
+        reasoning_tokens=(completion_details.reasoning_tokens if completion_details else None) or 0,
         model=response_model,
     )
 
@@ -203,10 +245,14 @@ def _aggregate_batch_cost_usage_models(
     custom_llm_provider: Literal["openai", "azure", "vertex_ai", "hosted_vllm", "anthropic", "bedrock"],
     model_name: str | None = None,
     model_info: ModelInfo | None = None,
-) -> tuple[float, Usage, list[str]]:
-    """Aggregate cost, usage, and models from batch output entries in a single
-    pass, holding one small stats record per line instead of the parsed file."""
-    line_stats: Final = tuple(_iter_successful_output_line_stats(entries, custom_llm_provider, model_name, model_info))
+) -> BatchCostUsageResult:
+    """Aggregate cost, usage, models, and pass/fail counts from batch output
+    entries in a single pass, holding one small stats record per line instead
+    of the parsed file."""
+    all_results: Final = tuple(_classify_output_line_stats(entries, custom_llm_provider, model_name, model_info))
+    line_stats: Final = tuple(result for result in all_results if isinstance(result, _BatchOutputLineStats))
+    failed_requests: Final = sum(1 for result in all_results if result is _LineOutcome.PROVIDER_FAILED)
+    successful_requests: Final = len(all_results) - failed_requests
 
     cache_token_params: Final = {
         key: tokens
@@ -220,18 +266,32 @@ def _aggregate_batch_cost_usage_models(
         total_tokens=sum(stats.total_tokens for stats in line_stats),
         prompt_tokens=sum(stats.prompt_tokens for stats in line_stats),
         completion_tokens=sum(stats.completion_tokens for stats in line_stats),
+        reasoning_tokens=sum(stats.reasoning_tokens for stats in line_stats),
         **cache_token_params,
     )
     batch_models: Final = [model_name] if model_name else [stats.model for stats in line_stats if stats.model]
     total_cost: Final = sum((stats.cost for stats in line_stats), 0.0)
-    verbose_logger.debug("batch output aggregate: cost=%s usage=%s models=%s", total_cost, batch_usage, batch_models)
-    return total_cost, batch_usage, batch_models
+    verbose_logger.debug(
+        "batch output aggregate: cost=%s usage=%s models=%s successful=%d failed=%d",
+        total_cost,
+        batch_usage,
+        batch_models,
+        successful_requests,
+        failed_requests,
+    )
+    return BatchCostUsageResult(
+        cost=total_cost,
+        usage=batch_usage,
+        models=batch_models,
+        successful_requests=successful_requests,
+        failed_requests=failed_requests,
+    )
 
 
 def calculate_vertex_ai_batch_cost_and_usage(
     vertex_ai_batch_responses: list[dict],
     model_name: str | None = None,
-) -> tuple[float, Usage]:
+) -> BatchCostUsageResult:
     """
     Calculate both cost and usage from raw Vertex AI batch responses.
 
@@ -242,6 +302,10 @@ def calculate_vertex_ai_batch_cost_and_usage(
       {"request": ..., "response": {"candidates": [...], "usageMetadata": {...}}}
 
     usageMetadata contains promptTokenCount, candidatesTokenCount, totalTokenCount.
+
+    A row with no ``response`` is counted as failed - the same signal already
+    used to skip it from cost/usage aggregation, since Vertex batch prediction
+    output doesn't establish a distinct error shape in this (non-default) path.
     """
     from litellm.cost_calculator import batch_cost_calculator
 
@@ -249,12 +313,16 @@ def calculate_vertex_ai_batch_cost_and_usage(
     total_tokens = 0
     prompt_tokens = 0
     completion_tokens = 0
+    successful_requests = 0  # rebind-ok: loop accumulator, matches total_cost/total_tokens above
+    failed_requests = 0  # rebind-ok: loop accumulator, matches total_cost/total_tokens above
     actual_model_name: Final = model_name or "gemini-2.0-flash-001"
 
     for response in vertex_ai_batch_responses:
         response_body = response.get("response")
         if response_body is None:
+            failed_requests += 1
             continue
+        successful_requests += 1
 
         usage_metadata = response_body.get("usageMetadata", {})
         _prompt = usage_metadata.get("promptTokenCount", 0) or 0
@@ -282,17 +350,25 @@ def calculate_vertex_ai_batch_cost_and_usage(
         total_tokens += _total
 
     verbose_logger.info(
-        "vertex_ai batch cost: cost=%s, prompt=%d, completion=%d, total=%d",
+        "vertex_ai batch cost: cost=%s, prompt=%d, completion=%d, total=%d, successful=%d, failed=%d",
         total_cost,
         prompt_tokens,
         completion_tokens,
         total_tokens,
+        successful_requests,
+        failed_requests,
     )
 
-    return total_cost, Usage(
-        total_tokens=total_tokens,
-        prompt_tokens=prompt_tokens,
-        completion_tokens=completion_tokens,
+    return BatchCostUsageResult(
+        cost=total_cost,
+        usage=Usage(
+            total_tokens=total_tokens,
+            prompt_tokens=prompt_tokens,
+            completion_tokens=completion_tokens,
+        ),
+        models=[actual_model_name],
+        successful_requests=successful_requests,
+        failed_requests=failed_requests,
     )
 
 
@@ -322,6 +398,36 @@ def _provider_output_file_id(output_file_id: str) -> str:
     return extracted
 
 
+async def _fetch_batch_managed_file_content(
+    file_id: str,
+    custom_llm_provider: Literal["openai", "azure", "vertex_ai", "hosted_vllm", "anthropic"] = "openai",
+    litellm_params: dict | None = None,
+) -> bytes:
+    """
+    Fetch a batch's output or error file and return its raw JSONL bytes.
+
+    Args:
+        file_id: The provider or unified (litellm-managed) file id to fetch
+        custom_llm_provider: The LLM provider
+        litellm_params: Optional litellm parameters containing credentials (api_key, api_base, etc.)
+                       Required for Azure and other providers that need authentication
+    """
+    from litellm.files.main import afile_content
+
+    # Build kwargs for afile_content with credentials from litellm_params
+    file_content_kwargs: Final = {
+        "file_id": _provider_output_file_id(file_id),
+        "custom_llm_provider": custom_llm_provider,
+    }
+
+    # Extract and add credentials for file access
+    credentials: Final = _extract_file_access_credentials(litellm_params)
+    file_content_kwargs.update(credentials)
+
+    _file_content: Final = await afile_content(**file_content_kwargs)
+    return _file_content.content
+
+
 async def _fetch_batch_output_file_content(
     batch: Batch,
     custom_llm_provider: Literal["openai", "azure", "vertex_ai", "hosted_vllm", "anthropic"] = "openai",
@@ -336,25 +442,36 @@ async def _fetch_batch_output_file_content(
         litellm_params: Optional litellm parameters containing credentials (api_key, api_base, etc.)
                        Required for Azure and other providers that need authentication
     """
-    from litellm.files.main import afile_content
-
     if batch.output_file_id is None:
         raise ValueError("Output file id is None cannot retrieve file content")
 
-    file_id: Final = _provider_output_file_id(batch.output_file_id)
+    return await _fetch_batch_managed_file_content(
+        batch.output_file_id, custom_llm_provider=custom_llm_provider, litellm_params=litellm_params
+    )
 
-    # Build kwargs for afile_content with credentials from litellm_params
-    file_content_kwargs: Final = {
-        "file_id": file_id,
-        "custom_llm_provider": custom_llm_provider,
-    }
 
-    # Extract and add credentials for file access
-    credentials: Final = _extract_file_access_credentials(litellm_params)
-    file_content_kwargs.update(credentials)
+async def count_error_file_failed_requests(
+    batch: Batch,
+    custom_llm_provider: Literal["openai", "azure", "vertex_ai", "hosted_vllm", "anthropic"],
+    litellm_params: dict | None,
+) -> int:
+    """Count failed requests reported only in the batch's separate error file.
 
-    _file_content: Final = await afile_content(**file_content_kwargs)
-    return _file_content.content
+    OpenAI-shaped batch providers write successful lines to ``output_file_id``
+    and per-request failures (e.g. a rejected param) to a distinct
+    ``error_file_id`` - they never appear in the output file at all, so
+    counting failures from the output file alone silently undercounts them.
+    """
+    if batch.error_file_id is None:
+        return 0
+    try:
+        error_file_content = await _fetch_batch_managed_file_content(
+            batch.error_file_id, custom_llm_provider=custom_llm_provider, litellm_params=litellm_params
+        )
+    except Exception as e:  # noqa: BLE001  # a failed/missing error file must not abort cost tracking for the batch
+        verbose_logger.debug("Failed to fetch batch error file %s: %s", batch.error_file_id, e)
+        return 0
+    return sum(1 for _ in _iter_batch_input_lines(error_file_content))
 
 
 def _extract_file_access_credentials(litellm_params: dict | None) -> dict:

litellm/constants.py

@@ -35,6 +35,7 @@ DEFAULT_COOLDOWN_TIME_SECONDS: Final = int(os.getenv("DEFAULT_COOLDOWN_TIME_SECO
 DEFAULT_REPLICATE_POLLING_RETRIES: Final = int(os.getenv("DEFAULT_REPLICATE_POLLING_RETRIES", 5))
 DEFAULT_REPLICATE_POLLING_DELAY_SECONDS: Final = int(os.getenv("DEFAULT_REPLICATE_POLLING_DELAY_SECONDS", 1))
 DEFAULT_IMAGE_TOKEN_COUNT: Final = int(os.getenv("DEFAULT_IMAGE_TOKEN_COUNT", 250))
+HF_CONFIG_FETCH_TIMEOUT_SECONDS: Final = 10.0
 
 # Maximum wall-clock seconds a streaming response is allowed to run.
 # Streams exceeding this duration are terminated with a Timeout error.
@@ -288,6 +289,7 @@ REDIS_DAILY_ORG_SPEND_UPDATE_BUFFER_KEY: Final = "litellm_daily_org_spend_update
 REDIS_DAILY_END_USER_SPEND_UPDATE_BUFFER_KEY: Final = "litellm_daily_end_user_spend_update_buffer"
 REDIS_DAILY_AGENT_SPEND_UPDATE_BUFFER_KEY: Final = "litellm_daily_agent_spend_update_buffer"
 REDIS_DAILY_TAG_SPEND_UPDATE_BUFFER_KEY: Final = "litellm_daily_tag_spend_update_buffer"
+REDIS_WINDOW_SPEND_UPDATE_BUFFER_KEY: Final = "litellm_window_spend_update_buffer"
 MAX_REDIS_BUFFER_DEQUEUE_COUNT: Final = int(os.getenv("MAX_REDIS_BUFFER_DEQUEUE_COUNT", 100))
 # Bounds asyncio.Queue() instances (log queues, spend update queues, etc.) to prevent unbounded memory growth
 LITELLM_ASYNCIO_QUEUE_MAXSIZE: Final = int(os.getenv("LITELLM_ASYNCIO_QUEUE_MAXSIZE", 1000))
@@ -1681,6 +1683,7 @@ DEFAULT_MCP_NAMESPACE_CSV_MAX_TOKENS: Final = 16
 # Ceilings on the cached auth registries; larger tables fall back to per-row lookups
 # instead of holding an unbounded id set in every worker.
 TAG_REGISTRY_MAX_SIZE: Final = 5000
+MODEL_ACCESS_GROUP_REGISTRY_MAX_SIZE: Final = 5000
 END_USER_RESTRICTED_REGISTRY_MAX_SIZE: Final = 5000
 # How long a failed registry load is remembered as "unusable", so a degraded Postgres
 # is not re-scanned on every request on top of the per-id lookups it falls back to.

litellm/integrations/otel/plumbing/providers.py

@@ -1,7 +1,7 @@
 """Provider / exporter factory + the Baggage span processor."""
 
 from collections.abc import Callable, Iterable
-from typing import TYPE_CHECKING, Any, Final
+from typing import TYPE_CHECKING, Any, Final, Literal
 
 from opentelemetry import _logs, baggage, metrics
 from opentelemetry._events import EventLogger
@@ -135,14 +135,36 @@ def parse_headers(raw: str | None) -> dict[str, str]:
     return dict(parse_env_headers(raw, liberal=True))
 
 
+_IN_MEMORY_KINDS: Final = ("in_memory", "inmemory", "memory")
+_OTLP_HTTP_KINDS: Final = ("otlp_http", "http", "http/protobuf", "http/json")
+_OTLP_GRPC_KINDS: Final = ("otlp_grpc", "grpc")
+
+
+def exporter_transport(kind: str) -> Literal["http", "grpc", "headerless"]:
+    """How an exporter of this ``kind`` carries credentials, per ``_exporter_from_spec``.
+
+    ``http``/``grpc`` exporters (and any registered factory, which builds an
+    OTLP exporter) stamp ``spec.headers``; ``console``, ``in_memory``, and any
+    unrecognized kind (which falls back to a header-ignoring console exporter)
+    are ``headerless``. Routability decisions must read this rather than a
+    denylist, so a typo'd or unavailable kind is not mistaken for OTLP.
+    """
+    resolved: Final = kind.lower()
+    if resolved in _OTLP_HTTP_KINDS or resolved in _EXPORTER_FACTORIES:
+        return "http"
+    if resolved in _OTLP_GRPC_KINDS:
+        return "grpc"
+    return "headerless"
+
+
 def _exporter_from_spec(spec: ExporterSpec) -> SpanExporter:
     kind: Final = (spec.kind or "console").lower()
     factory: Final = _EXPORTER_FACTORIES.get(kind)
     if factory is not None:
         return factory(spec)
-    if kind in ("in_memory", "inmemory", "memory"):
+    if kind in _IN_MEMORY_KINDS:
         return InMemorySpanExporter()
-    if kind in ("otlp_http", "http", "http/protobuf", "http/json"):
+    if kind in _OTLP_HTTP_KINDS:
         from opentelemetry.exporter.otlp.proto.http.trace_exporter import (
             OTLPSpanExporter as HTTPExporter,
         )
@@ -151,7 +173,7 @@ def _exporter_from_spec(spec: ExporterSpec) -> SpanExporter:
             endpoint=_otlp_traces_endpoint(spec.endpoint),
             headers=parse_headers(spec.headers),
         )
-    if kind in ("otlp_grpc", "grpc"):
+    if kind in _OTLP_GRPC_KINDS:
         from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import (
             OTLPSpanExporter as GRPCExporter,
         )

litellm/integrations/otel/plumbing/routing.py

@@ -27,6 +27,7 @@ from litellm.constants import OTEL_SERVICE_NAME_METADATA_KEYS
 from litellm.integrations.otel.model.config import ExporterSpec, OpenTelemetryV2Config
 from litellm.integrations.otel.plumbing.providers import (
     build_tracer_provider,
+    exporter_transport,
     get_tracer,
 )
 from litellm.integrations.otel.presets import (
@@ -121,13 +122,27 @@ def _encoded_header_string(headers: Mapping[str, str]) -> str:
 class TenantRoute:
     """The tracer to create a span on, plus whether it must root its own trace.
 
-    ``detached`` is True when project routing engaged. Phoenix assigns a whole
+    ``detached`` is True when the routed span exports to a DIFFERENT backend
+    than the request's root span, which always exports through the default
+    tracer. A detached span roots a fresh trace with a link back to the request
+    trace for correlation, so the destination account is not left holding a
+    child whose parent it never received. It is driven by whether routing
+    headers were actually applied to an owned exporter, not merely requested:
+    a credential or project route whose callback owns no exporter those headers
+    can reach exports through the default backend unchanged, so it stays
+    parented like an unrouted span.
+
+    Credential routing (a team/key's own vendor account) is one detaching case:
+    the root, auth, and db spans stay on the operator's default backend while
+    the LLM-call span exports to the tenant's account, so parenting it into the
+    request trace makes the tenant account show a fragmented span with a missing
+    parent. Project routing (Phoenix) is the other: Phoenix assigns a whole
     trace to one project by whichever of its spans arrives first, so a
     project-routed span parented into the request trace gets dragged into the
     project of the default-exported request spans and the header does nothing.
-    The span must therefore start a fresh trace (with a link back to the
-    request trace for correlation) — which is also how the v1 Phoenix logger
-    behaved, exporting each request under its own Phoenix-local parent span.
+    Both mirror the v1 loggers, which exported each request under its own
+    backend-local root. Service-name routing does NOT detach: it relabels
+    ``service.name`` on the SAME operator backend, where the parent is present.
     """
 
     tracer: Tracer
@@ -161,11 +176,20 @@ class TenantTracerCache:
         self._open_span_counts: dict[TracerProvider, int] = {}  # mutable-ok: live refcount state
         # Oldest-first so an overflow of draining providers sheds the stalest.
         self._retired: OrderedDict[TracerProvider, None] = OrderedDict()  # mutable-ok: draining evicted providers
-        self._project_routable = any(
-            spec.owner == callback_name and spec.kind.lower() not in (*_NON_OTLP_KINDS, *_GRPC_KINDS)
-            for spec in config.exporters
+        # An owned exporter is routable only when its kind actually resolves to a
+        # header-carrying OTLP exporter. A denylist would accept a typo'd or
+        # unavailable kind, which ``_exporter_from_spec`` falls back to a
+        # header-ignoring console exporter: detaching such a span would strand it
+        # on the operator's console, never reaching the tenant backend. Project
+        # headers are HTTP-only; credentials ride gRPC metadata too (Arize's
+        # default exporter is gRPC), so they accept either OTLP transport.
+        owned_transports: Final = tuple(
+            exporter_transport(spec.kind) for spec in config.exporters if spec.owner == callback_name
         )
+        self._project_routable = "http" in owned_transports
+        self._credential_routable = "http" in owned_transports or "grpc" in owned_transports
         self._warned_project_unroutable = False
+        self._warned_credential_unroutable = False
 
     def release(self, provider: TracerProvider | None) -> None:
         """Drop one open-span count; shut a retired provider down once drained.
@@ -207,7 +231,7 @@ class TenantTracerCache:
         concurrent overflow eviction can't shut it down between selection and
         the caller's span start. The caller must ``release`` it exactly once.
         """
-        credential_headers: Final = dynamic_otlp_headers(self._callback_name, dynamic_params) or _NO_HEADERS
+        credential_headers: Final = self._credential_headers(dynamic_params)
         project_headers: Final = self._project_headers(auth_metadata)
         service_name: Final = tenant_service_name(auth_metadata)
         if not credential_headers and not project_headers and service_name is None:
@@ -231,7 +255,7 @@ class TenantTracerCache:
             _shutdown_provider(evicted)
         return TenantRoute(
             tracer=get_tracer(provider, self._tracer_name),
-            detached=bool(project_headers),
+            detached=bool(project_headers) or bool(credential_headers),
             provider=provider,
         )
 
@@ -275,6 +299,26 @@ class TenantTracerCache:
         self._open_span_counts.pop(overflowed, None)
         return overflowed
 
+    def _credential_headers(self, dynamic_params: StandardCallbackDynamicParams | None) -> Mapping[str, str]:
+        """The per-request dynamic OTLP credentials, if this cache can apply them.
+
+        A callback owning only a console/in_memory exporter has nowhere to stamp
+        them, so the span would export to the operator's default backend
+        unchanged; routing there and detaching would orphan it on the very
+        backend that holds its parent. Warn once and keep the default tracer.
+        """
+        requested: Final = dynamic_otlp_headers(self._callback_name, dynamic_params) or _NO_HEADERS
+        if not requested or self._credential_routable:
+            return requested
+        if not self._warned_credential_unroutable:
+            self._warned_credential_unroutable = True
+            verbose_logger.warning(
+                "OTel V2: %s request carries dynamic credentials, but the callback owns no "
+                "OTLP exporter to stamp them onto; spans export to the default backend.",
+                self._callback_name,
+            )
+        return _NO_HEADERS
+
     def _project_headers(self, auth_metadata: Mapping[str, str] | None) -> Mapping[str, str]:
         """The per-request project-routing headers, if this cache can apply them.
 

litellm/litellm_core_utils/audio_utils/subtitle_utils.py

@@ -1,19 +1,36 @@
 """Provider-agnostic SRT/WebVTT subtitle synthesis from timestamped transcription tokens."""
 
+import unicodedata
 from collections.abc import Sequence
 from dataclasses import dataclass
-from itertools import accumulate, chain
+from itertools import accumulate, groupby
 from typing import Final
 
 from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError
 
-CUE_MAX_TOKENS: Final = 15
-CUE_MAX_DURATION_MS: Final = 5000
+CUE_MAX_CHARS: Final = 84
+CUE_MAX_DURATION_MS: Final = 7000
+CUE_GAP_MS: Final = 700
 
 SRT_RESPONSE_FORMAT: Final = "srt"
 VTT_RESPONSE_FORMAT: Final = "vtt"
 SUBTITLE_RESPONSE_FORMATS: Final = frozenset((SRT_RESPONSE_FORMAT, VTT_RESPONSE_FORMAT))
 
+_SENTENCE_END_CHARS: Final = (".", "!", "?", "。", "！", "？", "؟", "۔", "।", "॥", "։", "።")
+
+_CJK_RANGES: Final = (
+    (0x3400, 0x4DBF),
+    (0x4E00, 0x9FFF),
+    (0xF900, 0xFAFF),
+    (0x3040, 0x309F),
+    (0x30A0, 0x30FF),
+    (0x31F0, 0x31FF),
+)
+
+_CJK_NO_BREAK_BEFORE: Final = "、。，．！？：；・ー…」』）〉》】〕"
+
+_CJK_NO_BREAK_AFTER: Final = "「『（〈《【〔"
+
 
 @dataclass(frozen=True, slots=True)
 class SubtitleToken:
@@ -31,69 +48,138 @@ class SubtitleCue:
 
 
 @dataclass(frozen=True, slots=True)
-class _CueAccumulator:
-    texts: tuple[str, ...] = ()
-    start_ms: int | None = None
-    end_ms: int | None = None
-    speaker: str | int | None = None
+class _Word:
+    text: str
+    start_ms: int | None
+    end_ms: int | None
+    speaker: str | int | None
 
 
-def _completed_cue(accumulator: _CueAccumulator) -> tuple[SubtitleCue, ...]:
-    if not accumulator.texts or accumulator.start_ms is None:
-        return ()
-    text: Final = "".join(accumulator.texts).strip()
-    if not text:
-        return ()
-    end_ms: Final = accumulator.end_ms if accumulator.end_ms is not None else accumulator.start_ms
-    return (SubtitleCue(start_ms=accumulator.start_ms, end_ms=end_ms, text=text),)
+def _is_cjk(ch: str) -> bool:
+    cp: Final = ord(ch)
+    return any(lo <= cp <= hi for lo, hi in _CJK_RANGES)
 
 
-def _cue_break_reached(accumulator: _CueAccumulator, token: SubtitleToken) -> bool:
-    if len(accumulator.texts) >= CUE_MAX_TOKENS:
-        return True
+def _is_cjk_word_boundary(prev_ch: str, next_ch: str) -> bool:
+    if not (_is_cjk(prev_ch) or _is_cjk(next_ch)):
+        return False
+    return next_ch not in _CJK_NO_BREAK_BEFORE and prev_ch not in _CJK_NO_BREAK_AFTER
+
+
+def _text_width(text: str) -> int:
+    return sum(2 if unicodedata.east_asian_width(ch) in ("W", "F") else 1 for ch in text)
+
+
+def _starts_new_word(prev: SubtitleToken, token: SubtitleToken) -> bool:
+    prev_last: Final = prev.text[-1:]
+    first: Final = token.text[0]
     return (
-        accumulator.start_ms is not None
-        and token.start_ms is not None
-        and token.start_ms - accumulator.start_ms >= CUE_MAX_DURATION_MS
+        first.isspace()
+        or prev_last.isspace()
+        or token.speaker != prev.speaker
+        or _is_cjk_word_boundary(prev_last, first)
     )
 
 
-_AbsorbStep = tuple[tuple[SubtitleCue, ...], _CueAccumulator]
-
-
-def _absorb_token(accumulator: _CueAccumulator, token: SubtitleToken) -> _AbsorbStep:
-    if token.start_ms is None and accumulator.start_ms is None:
-        return (), accumulator
-    if token.speaker is not None and token.speaker != accumulator.speaker:
-        return _completed_cue(accumulator), _CueAccumulator(
-            texts=(token.text,),
-            start_ms=token.start_ms,
-            end_ms=token.end_ms,
-            speaker=token.speaker,
-        )
-    if _cue_break_reached(accumulator, token):
-        return _completed_cue(accumulator), _CueAccumulator(
-            texts=(token.text,),
-            start_ms=token.start_ms,
-            end_ms=token.end_ms,
-            speaker=accumulator.speaker,
-        )
-    return (), _CueAccumulator(
-        texts=(*accumulator.texts, token.text),
-        start_ms=accumulator.start_ms if accumulator.start_ms is not None else token.start_ms,
-        end_ms=token.end_ms if token.end_ms is not None else accumulator.end_ms,
-        speaker=accumulator.speaker,
+def _build_word(group: Sequence[SubtitleToken]) -> _Word:
+    return _Word(
+        text="".join(t.text for t in group),
+        start_ms=next((t.start_ms for t in group if t.start_ms is not None), None),
+        end_ms=next((t.end_ms for t in reversed(group) if t.end_ms is not None), None),
+        speaker=group[0].speaker,
     )
 
 
-def _absorb_step(carry: _AbsorbStep, token: SubtitleToken) -> _AbsorbStep:
-    return _absorb_token(carry[1], token)
+def _merge_tokens_into_words(tokens: Sequence[SubtitleToken]) -> tuple[_Word, ...]:
+    """
+    Merge subword tokens (e.g. ``"Hel"``, ``"lo"``) into whole words.
+
+    A token starts a new word when its text begins with whitespace, when the
+    previous token's text ends with whitespace, when the speaker changes, or
+    at a CJK character boundary (CJK scripts carry no spaces, so without this
+    an entire utterance would fuse into a single unbreakable "word"; CJK
+    punctuation stays attached to the preceding character per kinsoku rules).
+    Each word carries the first/last available timestamps of its tokens.
+    """
+    kept: Final = tuple(t for t in tokens if t.text != "")
+    starts: Final = tuple(i for i, t in enumerate(kept) if i == 0 or _starts_new_word(kept[i - 1], t))
+    return tuple(_build_word(kept[begin:end]) for begin, end in zip(starts, (*starts[1:], len(kept))))
+
+
+def _cue_start(ws: Sequence[_Word]) -> int | None:
+    return next((w.start_ms for w in ws if w.start_ms is not None), None)
+
+
+def _cue_end(ws: Sequence[_Word]) -> int | None:
+    return next((w.end_ms for w in reversed(ws) if w.end_ms is not None), _cue_start(ws))
+
+
+def _cue_text(ws: Sequence[_Word]) -> str:
+    return "".join(w.text for w in ws).strip()
+
+
+def _should_break(cue: Sequence[_Word], word: _Word) -> bool:
+    speaker_changed: Final = word.speaker is not None and any(
+        w.speaker is not None and w.speaker != word.speaker for w in cue
+    )
+    cue_start: Final = _cue_start(cue)
+    cue_end: Final = _cue_end(cue)
+    gap_exceeded: Final = word.start_ms is not None and cue_end is not None and (word.start_ms - cue_end) >= CUE_GAP_MS
+    chars_exceeded: Final = _text_width(_cue_text(cue)) + _text_width(word.text) > CUE_MAX_CHARS
+    word_end: Final = word.end_ms if word.end_ms is not None else word.start_ms
+    duration_exceeded: Final = (
+        word_end is not None and cue_start is not None and (word_end - cue_start) > CUE_MAX_DURATION_MS
+    )
+    return speaker_changed or gap_exceeded or chars_exceeded or duration_exceeded
+
+
+def _cue_start_indices(words: Sequence[_Word]) -> tuple[int, ...]:
+    def next_start(start: int, index: int) -> int:
+        if words[index - 1].text.rstrip().endswith(_SENTENCE_END_CHARS):
+            return index
+        if _should_break(words[start:index], words[index]):
+            return index
+        return start
+
+    if not words:
+        return ()
+    return tuple(start for start, _ in groupby(accumulate(range(1, len(words)), next_start, initial=0)))
+
+
+def _build_cue(ws: Sequence[_Word]) -> SubtitleCue | None:
+    text: Final = _cue_text(ws)
+    start: Final = _cue_start(ws)
+    if not text or start is None:
+        return None
+    end: Final = _cue_end(ws)
+    return SubtitleCue(start_ms=start, end_ms=end if end is not None else start, text=text)
 
 
 def group_subtitle_tokens_into_cues(tokens: Sequence[SubtitleToken]) -> tuple[SubtitleCue, ...]:
-    steps: Final = tuple(accumulate(tokens, _absorb_step, initial=((), _CueAccumulator())))
-    completed: Final = chain.from_iterable(emitted for emitted, _ in steps)
-    return (*completed, *_completed_cue(steps[-1][1]))
+    """
+    Group transcription tokens into subtitle cues aligned to the actual speech.
+
+    Cues only ever break at word boundaries (tokens may be subwords, so they
+    are first merged into words). A new cue starts when:
+      - the speaker changes (if diarization is on),
+      - a silence gap of at least CUE_GAP_MS separates two words, so
+        subtitles never bridge pauses in speech,
+      - adding the next word would exceed CUE_MAX_CHARS of display width
+        (~two subtitle lines; East-Asian wide characters count double), or
+      - adding the next word would make the cue span more than
+        CUE_MAX_DURATION_MS.
+    A cue also ends after sentence-final punctuation, which keeps cue breaks
+    at natural seams. Cue timestamps come straight from token timestamps;
+    words without timestamps stay attached to the surrounding cue, and a cue
+    whose words carry no timestamps at all is dropped.
+    """
+    words: Final = _merge_tokens_into_words(tokens)
+    starts: Final = _cue_start_indices(words)
+    return tuple(
+        cue
+        for begin, end in zip(starts, (*starts[1:], len(words)))
+        if (cue := _build_cue(words[begin:end])) is not None
+    )
 
 
 def _format_timestamp(total_ms: int, millis_separator: str) -> str:

litellm/litellm_core_utils/chat_completion_agentic_loop.py

@@ -1,6 +1,7 @@
 # this is a patch to allow for agentic loops covering llm_http_handler.py and openai sdk based calling flows for the .completion() api
 
 import json
+from collections.abc import Mapping
 from typing import Final, cast
 
 from litellm._logging import verbose_logger
@@ -9,8 +10,11 @@ from litellm.litellm_core_utils.agentic_loop_settings import (
     DEFAULT_MAX_AGENTIC_LOOPS,
     validated_max_agentic_loops,
 )
+from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObject
+from litellm.llms.base_llm.base_model_iterator import MockResponseIterator
 from litellm.types.integrations.custom_logger import (
     CHAT_COMPLETION_AGENTIC_SURFACE,
+    HEADROOM_CONVERTED_STREAM_KEY,
     NON_CODE_INTERPRETER_INTERCEPTION_INTERNAL_PREFIXES,
     AgenticLoopPlan,
     AgenticLoopRequestPatch,
@@ -50,6 +54,12 @@ def _post_hook_overridden(callback: CustomLogger) -> bool:
     return getattr(func, "__func__", func) is not getattr(base, "__func__", base)
 
 
+def _converted_stream_requested(kwargs: Mapping[str, object]) -> bool:
+    return bool(
+        kwargs.get("_code_interpreter_interception_converted_stream") or kwargs.get(HEADROOM_CONVERTED_STREAM_KEY)
+    )
+
+
 def _coerce_int(value: object, default: int) -> int:
     return int(value) if isinstance(value, (int, str)) else default
 
@@ -87,16 +97,24 @@ def _check_agentic_loop_safety(
     return fingerprint
 
 
-def _wrap_response_as_fake_stream(response: object) -> object:
-    if getattr(response, "object", None) == "chat.completion.chunk":
+def _wrap_response_as_fake_stream(
+    response: object,
+    *,
+    model: str,
+    custom_llm_provider: str,
+    logging_obj: object,
+) -> object:
+    if isinstance(response, CustomStreamWrapper):
         return response
-    if not hasattr(response, "choices"):
+    if not isinstance(response, ModelResponse) or not isinstance(logging_obj, LiteLLMLoggingObject):
         return response
-    from litellm.llms.base_llm.base_model_iterator import (
-        convert_model_response_to_streaming,
-    )
 
-    return convert_model_response_to_streaming(cast(ModelResponse, response))
+    return CustomStreamWrapper(
+        completion_stream=MockResponseIterator(model_response=response),
+        model=model,
+        custom_llm_provider=custom_llm_provider,
+        logging_obj=logging_obj,
+    )
 
 
 def _add_agentic_loop_metadata(kwargs_for_followup: dict[str, object]) -> None:
@@ -177,8 +195,13 @@ async def _execute_chat_completion_agentic_plan(
                     model,
                     str(e),
                 )
-        if kwargs.get("_code_interpreter_interception_converted_stream") and not depth:
-            return _wrap_response_as_fake_stream(response_followup)
+        if _converted_stream_requested(kwargs) and not depth:
+            return _wrap_response_as_fake_stream(
+                response_followup,
+                model=model,
+                custom_llm_provider=custom_llm_provider,
+                logging_obj=logging_obj,
+            )
         return response_followup
     finally:
         try:
@@ -302,9 +325,14 @@ async def maybe_run_chat_completion_agentic_loop(
                 str(e),
             )
 
-    if kwargs.get("_code_interpreter_interception_converted_stream") and not depth and hasattr(response, "choices"):
+    if _converted_stream_requested(kwargs) and not depth:
         return cast(
             "ModelResponse | CustomStreamWrapper",
-            _wrap_response_as_fake_stream(response),
+            _wrap_response_as_fake_stream(
+                response,
+                model=model,
+                custom_llm_provider=custom_llm_provider,
+                logging_obj=logging_obj,
+            ),
         )
     return None

litellm/litellm_core_utils/internal_call_metadata.py

@@ -25,6 +25,13 @@ from litellm.types.utils import BACKGROUND_RESPONSE_COST_POLL_CALL_ORIGIN, Inter
 
 BUDGET_RESERVATION_METADATA_KEYS: Final = frozenset({"user_api_key_budget_reservation"})
 
+MODEL_ACCESS_GROUP_METADATA_KEY: Final = "user_api_key_matched_model_access_groups"
+"""Where auth records the model access groups that authorized the request, for the spend writer.
+
+The ``user_api_key`` prefix is load-bearing, not cosmetic: when a request carries both
+``metadata`` and ``litellm_metadata``, ``get_litellm_metadata_from_kwargs`` returns the latter and
+copies a key across only when ``user_api_key`` appears in its name."""
+
 _USER_API_KEY_AUTH_KEY: Final = "user_api_key_auth"
 
 FORWARDABLE_IDENTITY_METADATA_KEYS: Final = frozenset(

litellm/litellm_core_utils/litellm_logging.py

@@ -64,7 +64,10 @@ from litellm.integrations.mlflow import MlflowLogger
 from litellm.integrations.sqs import SQSLogger
 from litellm.litellm_core_utils.core_helpers import is_expected_client_error, reconstruct_model_name
 from litellm.litellm_core_utils.get_litellm_params import get_litellm_params
-from litellm.litellm_core_utils.internal_call_metadata import is_unbilled_non_inference_call
+from litellm.litellm_core_utils.internal_call_metadata import (
+    MODEL_ACCESS_GROUP_METADATA_KEY,
+    is_unbilled_non_inference_call,
+)
 from litellm.litellm_core_utils.llm_cost_calc.guardrail_cost import (
     cost_breakdown_with_guardrail,
     guardrail_information_cost,
@@ -544,6 +547,9 @@ class Logging(LiteLLMLoggingBaseClass):
 
         # Init Caching related details
         self.caching_details: CachingDetails | None = None
+        # Timing for results that cannot carry ``_hidden_params`` (plain-dict /v1/messages
+        # responses and the bridge stream wrappers); see ``update_response_metadata``.
+        self.response_timing_metrics: Mapping[str, float] = {}  # mutable-ok: kept deep-copyable
 
         # Passthrough endpoint guardrails config for field targeting
         self.passthrough_guardrails_config: dict[str, Any] | None = None
@@ -563,6 +569,10 @@ class Logging(LiteLLMLoggingBaseClass):
         self._defer_async_logging: bool = False
         self._enqueue_deferred_logging: Callable[[], None] | None = None
 
+    def set_response_timing_metrics(self, timing_metrics: Mapping[str, float]) -> None:
+        """Keep ``_response_ms`` / ``litellm_overhead_time_ms`` for a result that has no ``_hidden_params``."""
+        self.response_timing_metrics = dict(timing_metrics)  # mutable-ok: kept deep-copyable
+
     def process_dynamic_callbacks(self):
         """
         Initializes CustomLogger compatible callbacks in self.dynamic_* callbacks
@@ -2872,6 +2882,8 @@ class Logging(LiteLLMLoggingBaseClass):
             batch_cost: Final = kwargs.get("batch_cost", None)
             batch_usage = kwargs.get("batch_usage", None)
             batch_models = kwargs.get("batch_models", None)
+            batch_successful_requests: Final = kwargs.get("batch_successful_requests", None)
+            batch_failed_requests: Final = kwargs.get("batch_failed_requests", None)
             has_explicit_batch_data: Final = all(x is not None for x in (batch_cost, batch_usage, batch_models))
 
             should_compute_batch_data: Final = (
@@ -2880,14 +2892,12 @@ class Logging(LiteLLMLoggingBaseClass):
             if has_explicit_batch_data:
                 result._hidden_params["response_cost"] = batch_cost
                 result._hidden_params["batch_models"] = batch_models
+                result._hidden_params["batch_successful_requests"] = batch_successful_requests  # pyright: ignore[reportPrivateUsage]  # rebind-ok: same result._hidden_params pattern as response_cost/batch_models above
+                result._hidden_params["batch_failed_requests"] = batch_failed_requests  # pyright: ignore[reportPrivateUsage]  # rebind-ok: same pattern as above
                 result.usage = batch_usage
 
             elif should_compute_batch_data:
-                (
-                    response_cost,
-                    batch_usage,
-                    batch_models,
-                ) = await _handle_completed_batch(
+                batch_result: Final = await _handle_completed_batch(
                     batch=result,
                     custom_llm_provider=self.custom_llm_provider,
                     model_name=self.get_deployment_model_for_cost(),
@@ -2895,9 +2905,11 @@ class Logging(LiteLLMLoggingBaseClass):
                     model_info=self.get_router_deployment_model_info(),
                 )
 
-                result._hidden_params["response_cost"] = response_cost
-                result._hidden_params["batch_models"] = batch_models
-                result.usage = batch_usage
+                result._hidden_params["response_cost"] = batch_result.cost
+                result._hidden_params["batch_models"] = batch_result.models
+                result._hidden_params["batch_successful_requests"] = batch_result.successful_requests  # pyright: ignore[reportPrivateUsage]  # rebind-ok: same pattern as above
+                result._hidden_params["batch_failed_requests"] = batch_result.failed_requests  # pyright: ignore[reportPrivateUsage]  # rebind-ok: same pattern as above
+                result.usage = batch_result.usage
 
         start_time, end_time, result = self._success_handler_helper_fn(
             start_time=start_time,
@@ -5049,6 +5061,42 @@ def is_valid_sha256_hash(value: str) -> bool:
     return bool(re.fullmatch(r"[a-fA-F0-9]{64}", value))
 
 
+def coerce_model_access_groups(value: object) -> tuple[str, ...]:
+    """Model access group names out of untrusted request metadata, deduped and order preserving."""
+    if not isinstance(value, (list, tuple)):
+        return ()
+    return tuple(dict.fromkeys(group for group in value if isinstance(group, str) and group))
+
+
+def _model_access_groups_on_auth_object(user_api_key_auth: object) -> object:
+    if isinstance(user_api_key_auth, Mapping):
+        return user_api_key_auth.get("matched_model_access_groups")
+    return getattr(user_api_key_auth, "matched_model_access_groups", None)
+
+
+def _model_access_groups_from_metadata(metadata: Mapping[str, object]) -> tuple[str, ...]:
+    stamped: Final = coerce_model_access_groups(metadata.get(MODEL_ACCESS_GROUP_METADATA_KEY))
+    if stamped:
+        return stamped
+    return coerce_model_access_groups(_model_access_groups_on_auth_object(metadata.get("user_api_key_auth")))
+
+
+def request_model_access_groups_from_litellm_params(litellm_params: Mapping[str, object]) -> tuple[str, ...]:
+    """Access groups the auth layer stamped onto this request, from whichever metadata field carries them.
+
+    Detached internal sub-calls only inherit the identity keys, so the auth object is the
+    fallback there, exactly as _get_budget_reservation_from_metadata does for reservations.
+    """
+    for metadata_variable_name in ("metadata", "litellm_metadata"):
+        metadata = litellm_params.get(metadata_variable_name)
+        if not isinstance(metadata, Mapping):
+            continue
+        model_access_groups = _model_access_groups_from_metadata(metadata)
+        if model_access_groups:
+            return model_access_groups
+    return ()
+
+
 class StandardLoggingPayloadSetup:
     @staticmethod
     def cleanup_timestamps(
@@ -5422,6 +5470,8 @@ class StandardLoggingPayloadSetup:
             additional_headers=None,
             litellm_overhead_time_ms=None,
             batch_models=None,
+            batch_successful_requests=None,
+            batch_failed_requests=None,
             litellm_model_name=None,
             usage_object=None,
         )
@@ -5812,6 +5862,8 @@ def _extract_response_obj_and_hidden_params(
                     response_cost=None,
                     litellm_overhead_time_ms=None,
                     batch_models=None,
+                    batch_successful_requests=None,
+                    batch_failed_requests=None,
                     litellm_model_name=None,
                     usage_object=None,
                 )
@@ -5896,6 +5948,7 @@ def get_standard_logging_object_payload(
         request_tags: Final = StandardLoggingPayloadSetup._get_request_tags(
             litellm_params=litellm_params, proxy_server_request=proxy_server_request
         )
+        request_model_access_groups: Final = request_model_access_groups_from_litellm_params(litellm_params)
 
         # cleanup timestamps
         (
@@ -5959,6 +6012,13 @@ def get_standard_logging_object_payload(
         clean_hidden_params: Final = StandardLoggingPayloadSetup.get_hidden_params(hidden_params)
         if clean_hidden_params["response_cost"] is None and raw_response_cost is not None:
             clean_hidden_params["response_cost"] = llm_response_cost
+        if clean_hidden_params["litellm_overhead_time_ms"] is None and status == "success":
+            # /v1/messages dict results and the bridge stream wrappers keep it on the logging object;
+            # failure payloads stay None like every response type that carries its own _hidden_params
+            timing_metrics: Final = (
+                getattr(logging_obj, "response_timing_metrics", None) or {}  # mutable-ok: empty fallback
+            )
+            clean_hidden_params["litellm_overhead_time_ms"] = timing_metrics.get("litellm_overhead_time_ms")
 
         model_cost_information: Final = StandardLoggingPayloadSetup.get_model_cost_information(
             base_model=base_model,
@@ -6058,6 +6118,7 @@ def get_standard_logging_object_payload(
             prompt_tokens=usage_dict.get("prompt_tokens", 0),
             completion_tokens=usage_dict.get("completion_tokens", 0),
             request_tags=request_tags,
+            request_model_access_groups=request_model_access_groups,
             end_user=end_user_id,
             api_base=StandardLoggingPayloadSetup.strip_trailing_slash(litellm_params.get("api_base", "")) or "",
             model_group=_model_group,
@@ -6228,6 +6289,8 @@ def create_dummy_standard_logging_payload() -> StandardLoggingPayload:
         additional_headers=None,
         litellm_overhead_time_ms=None,
         batch_models=None,
+        batch_successful_requests=None,
+        batch_failed_requests=None,
         litellm_model_name=None,
         usage_object=None,
     )
@@ -6269,6 +6332,7 @@ def create_dummy_standard_logging_payload() -> StandardLoggingPayload:
         cache_key=None,
         saved_cache_cost=saved_cache_cost,
         request_tags=[],
+        request_model_access_groups=(),
         end_user=None,
         requester_ip_address="127.0.0.1",
         messages=messages,

litellm/litellm_core_utils/llm_response_utils/response_metadata.py

@@ -1,4 +1,5 @@
 import datetime
+from collections.abc import Mapping
 from typing import Any, Final
 
 from litellm.constants import LITELLM_DETAILED_TIMING
@@ -13,6 +14,39 @@ from litellm.types.utils import (
 )
 
 
+def response_timing_metrics(
+    start_time: datetime.datetime,
+    end_time: datetime.datetime,
+    logging_obj: LiteLLMLoggingObject,
+    include_overhead: bool = True,
+) -> Mapping[str, float]:
+    """``_response_ms`` for the whole call, plus ``litellm_overhead_time_ms`` when it can be derived.
+
+    On a cache hit the overhead is the total minus the cache read; otherwise it is the total minus
+    the provider call (``llm_api_duration_ms``). It is omitted when neither duration was recorded,
+    and when ``include_overhead`` is False because the two durations cover different windows.
+    """
+    total_response_time_ms: Final = (end_time - start_time).total_seconds() * 1000
+    if not include_overhead:
+        return {"_response_ms": total_response_time_ms}  # mutable-ok: read-only timing result
+    caching_details: Final = logging_obj.caching_details
+    cache_duration_ms: Final = (
+        caching_details.get("cache_duration_ms")
+        if caching_details is not None and caching_details.get("cache_hit") is True
+        else None
+    )
+    llm_api_duration_ms: Final = logging_obj.model_call_details.get("llm_api_duration_ms")
+    if cache_duration_ms is not None:
+        overhead_ms: float | None = total_response_time_ms - cache_duration_ms
+    elif llm_api_duration_ms is not None:
+        overhead_ms = round(total_response_time_ms - llm_api_duration_ms, 4)
+    else:
+        overhead_ms = None
+    if overhead_ms is None:
+        return {"_response_ms": total_response_time_ms}
+    return {"_response_ms": total_response_time_ms, "litellm_overhead_time_ms": overhead_ms}
+
+
 class ResponseMetadata:
     """
     Handles setting and managing `_hidden_params`, `response_time_ms`, and `litellm_overhead_time_ms` for LiteLLM responses
@@ -52,7 +86,7 @@ class ResponseMetadata:
         }
         self._update_hidden_params(new_params)
 
-    def _update_hidden_params(self, new_params: dict) -> None:
+    def _update_hidden_params(self, new_params: Mapping[str, object]) -> None:
         """
         Update hidden params - handles when self._hidden_params is a dict or HiddenParams object
         """
@@ -76,37 +110,24 @@ class ResponseMetadata:
         start_time: datetime.datetime,
         end_time: datetime.datetime,
         logging_obj: LiteLLMLoggingObject,
+        include_overhead: bool = True,
     ) -> None:
         """Set response timing metrics"""
-        total_response_time_ms: Final = (end_time - start_time).total_seconds() * 1000
+        timing_metrics: Final = response_timing_metrics(start_time, end_time, logging_obj, include_overhead)
+        total_response_time_ms: Final = timing_metrics["_response_ms"]
 
         # Set total response time if supported
         if self.supports_response_time:
             self.result._response_ms = total_response_time_ms
 
         #########################################################
-        # 1. Add _response_ms total duration
+        # 1. Add _response_ms total duration and the LiteLLM overhead within it
+        #    (total minus the cache read on a cache hit, else total minus the provider call)
         #########################################################
-        self._update_hidden_params(
-            {
-                "_response_ms": total_response_time_ms,
-            }
-        )
+        self._update_hidden_params(timing_metrics)
 
         #########################################################
-        # 2. Add LiteLLM overhead duration
-        #########################################################
-        llm_api_duration_ms: Final = logging_obj.model_call_details.get("llm_api_duration_ms")
-        if llm_api_duration_ms is not None:
-            overhead_ms = round(total_response_time_ms - llm_api_duration_ms, 4)
-            self._update_hidden_params(
-                {
-                    "litellm_overhead_time_ms": overhead_ms,
-                }
-            )
-
-        #########################################################
-        # 3. Add callback processing duration
+        # 2. Add callback processing duration
         #########################################################
         callback_duration_ms: Final = getattr(logging_obj, "callback_duration_ms", None)
         if callback_duration_ms is not None:
@@ -117,24 +138,9 @@ class ResponseMetadata:
             )
 
         #########################################################
-        # 4. Add duration for reading from cache
-        # In this case overhead from litellm is the difference between the cache read duration and the total response time
-        #########################################################
-        if (
-            logging_obj.caching_details is not None
-            and logging_obj.caching_details.get("cache_hit") is True
-            and (cache_duration_ms := logging_obj.caching_details.get("cache_duration_ms")) is not None
-        ):
-            overhead_ms = total_response_time_ms - cache_duration_ms
-            self._update_hidden_params(
-                {
-                    "litellm_overhead_time_ms": overhead_ms,
-                }
-            )
-
-        #########################################################
-        # 5. Detailed per-phase timing (opt-in via env var)
+        # 3. Detailed per-phase timing (opt-in via env var)
         #########################################################
+        llm_api_duration_ms: Final = logging_obj.model_call_details.get("llm_api_duration_ms")
         if LITELLM_DETAILED_TIMING and llm_api_duration_ms is not None:
             detailed: Final[dict] = {
                 "timing_llm_api_ms": round(llm_api_duration_ms, 4),
@@ -170,6 +176,7 @@ def update_response_metadata(
     kwargs: dict,
     start_time: datetime.datetime,
     end_time: datetime.datetime,
+    include_overhead: bool = True,
 ) -> None:
     """
     Updates response metadata including hidden params and timing metrics
@@ -177,11 +184,22 @@ def update_response_metadata(
         - response._hidden_params
         - response._hidden_params["litellm_overhead_time_ms"]
         - response.response_time_ms
+    A result that cannot hold ``_hidden_params`` gets its timing on ``logging_obj`` instead.
+    Callers whose ``end_time`` covers more than the recorded provider call (a stream read to
+    completion) pass ``include_overhead=False``, since the overhead cannot be derived there.
     """
-    if result is None or not hasattr(result, "_hidden_params"):
+    if result is None:
+        return
+    if not hasattr(result, "_hidden_params"):
+        # /v1/messages returns a plain dict and the Anthropic / Responses bridge stream wrappers
+        # cannot hold ``_hidden_params``: keep only the timing on the logging object (no cost
+        # recompute) so the proxy headers and the standard logging payload can still read it.
+        logging_obj.set_response_timing_metrics(
+            response_timing_metrics(start_time, end_time, logging_obj, include_overhead)
+        )
         return
 
     metadata: Final = ResponseMetadata(result)
     metadata.set_hidden_params(logging_obj, model, kwargs)
-    metadata.set_timing_metrics(start_time, end_time, logging_obj)
+    metadata.set_timing_metrics(start_time, end_time, logging_obj, include_overhead)
     metadata.apply()

litellm/litellm_core_utils/prompt_templates/common_utils.py

@@ -10,6 +10,7 @@ from collections.abc import Iterable, Mapping, Sequence
 from itertools import groupby
 from os import PathLike
 from pathlib import Path
+from types import MappingProxyType
 from typing import TYPE_CHECKING, Any, Final, Literal, TypeVar, cast
 
 from openai.types.chat.chat_completion_custom_tool_param import (
@@ -1089,6 +1090,162 @@ def sanitize_input_schema_for_anthropic(input_schema: dict) -> "AnthropicInputSc
     return AnthropicInputSchema(**filtered)
 
 
+_TOP_LEVEL_SCHEMA_COMBINATORS: Final = ("allOf", "anyOf", "oneOf")
+_OPENAI_REJECTED_TOP_LEVEL_SCHEMA_KEYS: Final = ("enum", "const", "not")
+_LOCAL_SCHEMA_REF_PREFIXES: Final = (("#/$defs/", "$defs"), ("#/definitions/", "definitions"))
+_MAX_SCHEMA_FLATTEN_DEPTH: Final = 32
+_EMPTY_SCHEMA: Final[Mapping[str, object]] = MappingProxyType({})
+
+
+def _schema_properties(schema: Mapping[str, object]) -> Mapping[str, object]:
+    properties: Final = schema.get("properties")
+    return properties if isinstance(properties, dict) else _EMPTY_SCHEMA
+
+
+def _schema_branches(schema: Mapping[str, object], combinator: str) -> tuple[object, ...]:
+    branches: Final = schema.get(combinator)
+    return tuple(branches) if isinstance(branches, list) else ()
+
+
+def _schema_required_names(schema: Mapping[str, object]) -> frozenset[str]:
+    required: Final = schema.get("required")
+    if not isinstance(required, list):
+        return frozenset()
+    return frozenset(name for name in required if isinstance(name, str))
+
+
+def _combinator_required_names(combinator: str, branches: tuple[Mapping[str, object], ...]) -> frozenset[str]:
+    branch_names: Final = tuple(_schema_required_names(branch) for branch in branches)
+    if not branch_names:
+        return frozenset()
+    if combinator == "allOf":
+        return branch_names[0].union(*branch_names[1:])
+    return branch_names[0].intersection(*branch_names[1:])
+
+
+def _resolve_local_schema_ref(root: Mapping[str, object], ref: str) -> Mapping[str, object] | None:
+    matched: Final = next(
+        ((prefix, container) for prefix, container in _LOCAL_SCHEMA_REF_PREFIXES if ref.startswith(prefix)),
+        None,
+    )
+    if matched is None:
+        return None
+    prefix, container = matched
+    definitions: Final = root.get(container)
+    if not isinstance(definitions, dict):
+        return None
+    target: Final = definitions.get(ref[len(prefix) :])
+    return target if isinstance(target, dict) else None
+
+
+def _mergeable_branch(
+    root: Mapping[str, object],
+    branch: object,
+    seen_refs: frozenset[str],
+    depth: int,
+    expanded_refs: dict[str, Mapping[str, object] | None],  # mutable-ok: per-call memo bounding repeated $ref work
+) -> Mapping[str, object] | None:
+    if not isinstance(branch, dict) or depth > _MAX_SCHEMA_FLATTEN_DEPTH:
+        return None
+    ref: Final = branch.get("$ref")
+    if not isinstance(ref, str):
+        flattened: Final = _flatten_schema_against_root(branch, root, seen_refs, depth, expanded_refs)
+        if any(combinator in flattened for combinator in _TOP_LEVEL_SCHEMA_COMBINATORS):
+            return None
+        return flattened
+    if ref in expanded_refs:
+        return expanded_refs[ref]
+    if ref in seen_refs:
+        return None
+    target: Final = _resolve_local_schema_ref(root, ref)
+    expanded: Final = (
+        None
+        if target is None
+        else _mergeable_branch(root, target, seen_refs | frozenset((ref,)), depth + 1, expanded_refs)
+    )
+    expanded_refs[ref] = expanded
+    return expanded
+
+
+def _is_object_schema(schema: Mapping[str, object]) -> bool:
+    return schema.get("type") == "object" or ("type" not in schema and "properties" in schema)
+
+
+def _flatten_schema_against_root(
+    schema: Mapping[str, object],
+    root: Mapping[str, object],
+    seen_refs: frozenset[str],
+    depth: int,
+    expanded_refs: dict[str, Mapping[str, object] | None],  # mutable-ok: per-call memo bounding repeated $ref work
+) -> Mapping[str, object]:
+    raw_branch_groups: Final = tuple(
+        (
+            combinator,
+            tuple(
+                _mergeable_branch(root, branch, seen_refs, depth + 1, expanded_refs)
+                for branch in _schema_branches(schema, combinator)
+            ),
+        )
+        for combinator in _TOP_LEVEL_SCHEMA_COMBINATORS
+        if isinstance(schema.get(combinator), list)
+    )
+    dropped: Final = (
+        *(combinator for combinator, _ in raw_branch_groups),
+        *(key for key in _OPENAI_REJECTED_TOP_LEVEL_SCHEMA_KEYS if key in schema),
+    )
+    if not dropped:
+        return schema
+
+    if any(branch is None for _, group in raw_branch_groups for branch in group):
+        return schema
+    branch_groups: Final = tuple(
+        (combinator, tuple(branch for branch in group if branch is not None)) for combinator, group in raw_branch_groups
+    )
+    branches: Final = tuple(branch for _, group in branch_groups for branch in group)
+    is_object_schema: Final = _is_object_schema(schema) or (
+        "type" not in schema and branches != () and all(_is_object_schema(branch) for branch in branches)
+    )
+    if not is_object_schema:
+        return schema
+
+    merged_properties: Final = {  # mutable-ok: tool parameters are JSON dicts
+        name: value for source in (*reversed(branches), schema) for name, value in _schema_properties(source).items()
+    }
+    required_names: Final = _schema_required_names(schema).union(
+        *(_combinator_required_names(combinator, group) for combinator, group in branch_groups)
+    )
+    kept: Final = MappingProxyType({key: value for key, value in schema.items() if key not in dropped})
+    required_update: Final = MappingProxyType({"required": sorted(required_names)}) if required_names else _EMPTY_SCHEMA
+    return {  # mutable-ok: tool parameters are JSON dicts
+        **kept,
+        "type": "object",
+        "properties": merged_properties,
+        **required_update,
+    }
+
+
+def flatten_top_level_schema_combinators(schema: Mapping[str, object]) -> Mapping[str, object]:
+    """Merge top-level ``allOf``/``anyOf``/``oneOf`` branches into an object tool schema.
+
+    OpenAI's function-calling validator rejects tool ``parameters`` carrying
+    'oneOf'/'anyOf'/'allOf'/'enum'/'const'/'not' at the top level (nested uses
+    are accepted), while lenient backends such as the ChatGPT backend Codex
+    talks to natively accept them, so an MCP tool declaring a top-level union
+    400s through LiteLLM. Branch properties merge without clobbering (the
+    top-level schema wins, then earlier branches); ``required`` becomes the
+    top-level list plus the intersection of the branch lists for anyOf/oneOf
+    or their union for allOf. Branches that are local ``$ref``s
+    (``#/$defs/...`` or ``#/definitions/...``) are resolved first, each ref
+    at most once per call, and branches that are themselves combinators are
+    flattened recursively up to a fixed depth; a branch that cannot be fully
+    merged (a boolean schema, an external or cyclic ``$ref``, a non-object
+    union, or nesting past the depth cap) leaves the whole schema untouched so
+    OpenAI's own validation still applies. Non-object schemas pass through
+    unchanged and the input is never mutated.
+    """
+    return _flatten_schema_against_root(schema, schema, frozenset(), 0, {})  # mutable-ok: fresh per-call $ref memo
+
+
 def _get_image_mime_type_from_url(url: str) -> str | None:
     """
     Get mime type for common image URLs

litellm/llms/anthropic/chat/transformation.py

@@ -1268,7 +1268,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
             )
 
     @staticmethod
-    def _cap_thinking_budget_to_max_tokens(
+    def cap_thinking_budget_to_max_tokens(
         thinking: AnthropicThinkingParam, max_tokens: int | None
     ) -> AnthropicThinkingParam | None:
         """Cap a legacy ``thinking.budget_tokens`` below ``max_tokens`` (Anthropic
@@ -1530,7 +1530,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
                         llm_provider=self._resolved_provider,
                     )
                     capped_thinking = (
-                        AnthropicConfig._cap_thinking_budget_to_max_tokens(legacy_thinking, max_tokens)
+                        AnthropicConfig.cap_thinking_budget_to_max_tokens(legacy_thinking, max_tokens)
                         if legacy_thinking is not None
                         else None
                     )

litellm/llms/anthropic/common_utils.py

@@ -1099,6 +1099,25 @@ def is_empty_thinking_block(block: object) -> bool:
     return not isinstance(thinking, str) or not thinking.strip()
 
 
+def is_empty_unsigned_thinking_block(block: object) -> bool:
+    """
+    True for an empty ``{"type": "thinking"}`` block carrying no signature.
+
+    The emit-side predicate: response paths drop a thinking block only when it
+    holds nothing the client could need.  A signature-only block is a real
+    provider response (Bedrock Converse under adaptive thinking emits a
+    reasoning block with empty text and only a signature) and the client needs
+    the signature to replay reasoning across tool-use turns, so it must be
+    emitted.  Request paths keep using :func:`is_empty_thinking_block`:
+    Anthropic rejects empty thinking blocks in request history regardless of
+    signature, and the inbound strip self-heals a replayed signature-only
+    block.
+    """
+    if not isinstance(block, dict) or not is_empty_thinking_block(block):
+        return False
+    return not block.get("signature")
+
+
 def normalize_anthropic_tool_use_id(raw_id: str) -> str:
     """
     Normalize a tool_use / tool_result id for Anthropic's ``^[a-zA-Z0-9_-]+$``

litellm/llms/anthropic/experimental_pass_through/adapters/streaming_iterator.py

@@ -1029,7 +1029,7 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper):
 
     @staticmethod
     def _is_blank_delta(chunk: "ModelResponseStream") -> bool:
-        from litellm.llms.anthropic.common_utils import is_empty_thinking_block
+        from litellm.llms.anthropic.common_utils import is_empty_unsigned_thinking_block
 
         choice: Final = chunk.choices[0]
         if choice.finish_reason is not None:
@@ -1041,11 +1041,14 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper):
             return False
         if getattr(delta, "reasoning_content", None):
             return False
-        # thinking_blocks whose entries are all empty (even if signed) must not
+        # thinking_blocks whose entries are all empty AND unsigned must not
         # open a block: the emitted {"type": "thinking", "thinking": ""} gets
-        # replayed as history and Anthropic rejects it (LIT-6357).
+        # replayed as history and Anthropic rejects it (LIT-6357). A signed
+        # entry opens the block so the client receives the replay signature.
         thinking_blocks: Final = getattr(delta, "thinking_blocks", None)
-        if thinking_blocks and any(isinstance(b, dict) and not is_empty_thinking_block(b) for b in thinking_blocks):
+        if thinking_blocks and any(
+            isinstance(b, dict) and not is_empty_unsigned_thinking_block(b) for b in thinking_blocks
+        ):
             return False
         return True
 

litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py

@@ -90,7 +90,7 @@ from litellm.litellm_core_utils.reasoning_effort_utils import (
     reasoning_effort_from_thinking_budget,
 )
 from litellm.llms.anthropic.common_utils import (
-    is_empty_thinking_block,
+    is_empty_unsigned_thinking_block,
     normalize_anthropic_tool_use_id,
 )
 from litellm.llms.anthropic.experimental_pass_through.context_management import (
@@ -1267,7 +1267,7 @@ class LiteLLMAnthropicMessagesAdapter:
             if hasattr(choice.message, "thinking_blocks") and choice.message.thinking_blocks:
                 for thinking_block in choice.message.thinking_blocks:
                     if thinking_block.get("type") == "thinking":
-                        if is_empty_thinking_block(thinking_block):
+                        if is_empty_unsigned_thinking_block(thinking_block):
                             continue
                         thinking_value = thinking_block.get("thinking", "")
                         signature_value = thinking_block.get("signature", "")

litellm/llms/anthropic/experimental_pass_through/messages/transformation.py

@@ -40,6 +40,11 @@ DROP_UNSUPPORTED_ADAPTIVE_EFFORT_WARNING: Final = (
     "minimum thinking budget."
 )
 
+DROP_UNFITTING_REASONING_EFFORT_WARNING: Final = (
+    "Dropping `thinking` mapped from reasoning_effort=%s for model=%s: max_tokens=%s "
+    "is too small to fit the minimum thinking budget."
+)
+
 
 class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
     @property
@@ -335,11 +340,15 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
         return headers, api_base
 
     @staticmethod
-    def _translate_reasoning_effort_to_anthropic(model: str, optional_params: dict, custom_llm_provider: str) -> None:
+    def _translate_reasoning_effort_to_anthropic(
+        model: str, optional_params: dict, max_tokens: int | None, custom_llm_provider: str
+    ) -> None:
         """Map OpenAI-style ``reasoning_effort`` to native Anthropic params.
 
         Caller-supplied ``thinking`` / ``output_config`` win over the alias.
-        ``effort='none'`` clears both. Invalid efforts raise a 400.
+        ``effort='none'`` clears both. Invalid efforts raise a 400. A mapped
+        thinking budget is capped below ``max_tokens`` and dropped when even
+        the minimum budget cannot fit.
         """
         from litellm.exceptions import BadRequestError as _BadRequestError
         from litellm.llms.anthropic.chat.transformation import (
@@ -365,7 +374,12 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
             optional_params.pop("output_config", None)
             return
 
-        optional_params.setdefault("thinking", mapped_thinking)
+        fitted_thinking: Final = AnthropicConfig.cap_thinking_budget_to_max_tokens(mapped_thinking, max_tokens)
+        if fitted_thinking is None:
+            verbose_logger.warning(DROP_UNFITTING_REASONING_EFFORT_WARNING, reasoning_effort, model, max_tokens)
+            return
+
+        optional_params.setdefault("thinking", fitted_thinking)
         if AnthropicModelInfo._is_adaptive_thinking_model(model, custom_llm_provider):
             mapped_effort: Final = REASONING_EFFORT_TO_OUTPUT_CONFIG_EFFORT.get(reasoning_effort)
             if mapped_effort is None:
@@ -510,7 +524,7 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
         except _BadRequestError as e:
             raise AnthropicError(message=str(e.message), status_code=400)
         capped_thinking: Final = (
-            AnthropicConfig._cap_thinking_budget_to_max_tokens(legacy_thinking, max_tokens)
+            AnthropicConfig.cap_thinking_budget_to_max_tokens(legacy_thinking, max_tokens)
             if legacy_thinking is not None
             else None
         )
@@ -582,6 +596,7 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
         self._translate_reasoning_effort_to_anthropic(
             model=model,
             optional_params=anthropic_messages_optional_request_params,
+            max_tokens=max_tokens,
             custom_llm_provider=self._resolved_provider,
         )
 

litellm/llms/azure/azure.py

@@ -846,6 +846,7 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM):
         api_key: str,
         data: dict,
         headers: dict,
+        deployment_name: str | None = None,
     ) -> httpx.Response:
         """
         Implemented for azure dall-e-2 image gen calls
@@ -957,7 +958,7 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM):
                 content=json.dumps(result).encode("utf-8"),
                 request=httpx.Request(method="POST", url="https://api.openai.com/v1"),
             )
-        request_json: Final = azure_deployment_image_generation_json_body(api_base, data)
+        request_json: Final = azure_deployment_image_generation_json_body(api_base, data, deployment_name)
         return await async_handler.post(
             url=api_base,
             json=request_json,
@@ -973,6 +974,7 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM):
         api_key: str,
         data: dict,
         headers: dict,
+        deployment_name: str | None = None,
     ) -> httpx.Response:
         """
         Implemented for azure dall-e-2 image gen calls
@@ -1073,7 +1075,7 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM):
                 content=json.dumps(result).encode("utf-8"),
                 request=httpx.Request(method="POST", url="https://api.openai.com/v1"),
             )
-        request_json: Final = azure_deployment_image_generation_json_body(api_base, data)
+        request_json: Final = azure_deployment_image_generation_json_body(api_base, data, deployment_name)
         return sync_handler.post(
             url=api_base,
             json=request_json,
@@ -1091,9 +1093,10 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM):
             AzureFoundryMAIImageGenerationConfig,
         )
 
-        api_base: str = azure_client_params.get("azure_endpoint", "")  # "https://example-endpoint.openai.azure.com"
-        if api_base.endswith("/"):
-            api_base = api_base.rstrip("/")
+        # deployment-scoped endpoints are moved to "base_url" by select_azure_base_url_or_endpoint
+        api_base: str = (azure_client_params.get("azure_endpoint") or azure_client_params.get("base_url") or "").rstrip(
+            "/"
+        )
         api_version: Final[str] = azure_client_params.get("api_version", "")
         if model is None:
             model = ""
@@ -1113,6 +1116,14 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM):
                 api_version=api_version,
             )
 
+        v1_url: Final = BaseAzureLLM.get_azure_v1_image_url(
+            api_base=api_base,
+            api_version=api_version,
+            route="/openai/images/generations",
+        )
+        if v1_url is not None:
+            return v1_url
+
         if "/openai/deployments/" in api_base:
             base_url_with_deployment = api_base
         else:
@@ -1167,6 +1178,7 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM):
                 api_key=api_key,
                 data=data,
                 headers=headers,
+                deployment_name=model,
             )
 
             provider_config: Final = get_azure_image_generation_config(data.get("model", "dall-e-2"))
@@ -1302,6 +1314,7 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM):
                 api_key=api_key or "",
                 data=data,
                 headers=headers,
+                deployment_name=model,
             )
             provider_config: Final = get_azure_image_generation_config(data.get("model", "dall-e-2"))
             if isinstance(provider_config, AzureFoundryMAIImageGenerationConfig):

litellm/llms/azure/common_utils.py

@@ -4,6 +4,7 @@ import json
 import os
 from collections.abc import Callable, Mapping
 from functools import lru_cache
+from types import MappingProxyType
 from typing import Any, Final, Literal, NamedTuple, cast
 
 import httpx
@@ -789,6 +790,32 @@ class BaseAzureLLM(BaseOpenAILLM):
 
         return str(final_url)
 
+    @staticmethod
+    def get_azure_v1_image_url(api_base: str, api_version: str | None, route: str) -> str | None:
+        """
+        Azure's v1 surface serves images at ``/openai/v1/images/{generations,edits}`` and routes by
+        ``model`` in the request body, so any deployment path and stale ``api-version`` in
+        ``api_base`` have to be dropped.
+
+        Returns None when ``api_version`` is a dated one, which still uses the deployment route.
+        """
+        if not BaseAzureLLM._is_azure_v1_api_version(api_version):
+            return None
+
+        base_url: Final = httpx.URL(api_base)
+        openai_path_start: Final = base_url.path.find("/openai")
+        resource_base: Final = str(
+            base_url.copy_with(
+                path=base_url.path if openai_path_start == -1 else base_url.path[:openai_path_start],
+                params=httpx.QueryParams(tuple((k, v) for k, v in base_url.params.multi_items() if k != "api-version")),
+            )
+        )
+        return BaseAzureLLM._get_base_azure_url(
+            api_base=resource_base,
+            litellm_params=MappingProxyType({"api_version": api_version}),
+            route=route,
+        )
+
     @staticmethod
     def _is_azure_v1_api_version(api_version: str | None) -> bool:
         if api_version is None:

litellm/llms/azure/image_edit/transformation.py

@@ -93,8 +93,6 @@ class AzureImageEditConfig(OpenAIImageEditConfig):
             raise ValueError(
                 f"api_base is required for Azure AI Studio. Please set the api_base parameter. Passed `api_base={api_base}`"
             )
-        original_url: Final = httpx.URL(api_base)
-
         # Resolve api_version: litellm_params > litellm.api_version > AZURE_API_VERSION env > default.
         # Mirrors the fallback chain used by the Azure chat path in common_utils.py,
         # so callers that set a global / env api_version don't get an unversioned URL.
@@ -105,6 +103,16 @@ class AzureImageEditConfig(OpenAIImageEditConfig):
             or litellm.AZURE_DEFAULT_API_VERSION
         )
 
+        v1_url: Final = BaseAzureLLM.get_azure_v1_image_url(
+            api_base=api_base,
+            api_version=api_version,
+            route="/openai/images/edits",
+        )
+        if v1_url is not None:
+            return v1_url
+
+        original_url: Final = httpx.URL(api_base)
+
         # Create a new dictionary with existing params
         query_params: Final = dict(original_url.params)
 

litellm/llms/azure/image_generation/http_utils.py

@@ -1,7 +1,9 @@
 """HTTP helpers for Azure OpenAI image generation (REST, not SDK)."""
 
+from typing import Final
 
-def azure_deployment_image_generation_json_body(api_base: str, data: dict) -> dict:
+
+def azure_deployment_image_generation_json_body(api_base: str, data: dict, deployment_name: str | None = None) -> dict:
     """
     Build the JSON body for Azure OpenAI image generation POSTs.
 
@@ -9,9 +11,20 @@ def azure_deployment_image_generation_json_body(api_base: str, data: dict) -> di
     deployment in the URL only; sending ``model`` in the body (especially the deployment
     name) breaks some models (e.g. gpt-image-2). See LiteLLM #26316.
 
+    For the v1 surface (``.../openai/v1/images/...``), Azure routes by the deployment
+    name in the body ``model`` field, so the deployment name must replace any base
+    model name there or Azure answers 404 DeploymentNotFound.
+
     Provider-style URLs (e.g. ``/providers/...`` for FLUX on Azure AI) keep all keys
     so non–OpenAI-deployment payloads still work.
     """
-    if "images/generations" in api_base and "/openai/deployments/" in api_base:
-        return {k: v for k, v in data.items() if k != "model"}
-    return data
+    drop_model: Final = "images/generations" in api_base and "/openai/deployments/" in api_base
+    v1_route: Final = "/openai/v1/images/" in api_base and bool(deployment_name)
+    if not drop_model and not v1_route:
+        return data
+    entries: Final = (
+        tuple((k, v) for k, v in data.items() if k != "model")
+        if drop_model
+        else (*data.items(), ("model", deployment_name))
+    )
+    return {k: v for k, v in entries}

litellm/llms/base_llm/managed_resources/base_managed_resource.py

@@ -12,6 +12,7 @@ from litellm.llms.base_llm.managed_resources.isolation import (
     build_list_page,
     build_owner_filter,
     can_access_resource,
+    resolve_resource_owner_id,
 )
 from litellm.proxy._types import UserAPIKeyAuth
 from litellm.types.utils import SpecialEnums
@@ -157,7 +158,7 @@ class BaseManagedResource(ABC, Generic[ResourceObjectType]):
             "resource_object": resource_object,
             "model_mappings": model_mappings,
             "flat_model_resource_ids": list(model_mappings.values()),
-            "created_by": user_api_key_dict.user_id,
+            "created_by": resolve_resource_owner_id(user_api_key_dict),
             "team_id": user_api_key_dict.team_id,
             "updated_by": user_api_key_dict.user_id,
         }
@@ -179,7 +180,7 @@ class BaseManagedResource(ABC, Generic[ResourceObjectType]):
             "unified_resource_id": unified_resource_id,
             "model_mappings": json.dumps(model_mappings),
             "flat_model_resource_ids": list(model_mappings.values()),
-            "created_by": user_api_key_dict.user_id,
+            "created_by": resolve_resource_owner_id(user_api_key_dict),
             "team_id": user_api_key_dict.team_id,
             "updated_by": user_api_key_dict.user_id,
         }

litellm/llms/base_llm/managed_resources/isolation.py

@@ -3,10 +3,11 @@ Tenant-isolation helpers for managed file/batch/vector-store resources.
 
 Returns a Prisma filter and an ownership check that scope managed resources
 to the caller's identity: proxy admins see everything, user-keyed callers
-see records they created, and service-account keys (no user_id) fall back
-to the resource's owning team. Callers with no admin role and no
-identifying ids are denied so an empty user_id can never select an
-unscoped query.
+see records they created, service-account keys (no user_id) fall back to
+the resource's owning team, and keys with neither a user_id nor a team_id
+fall back to their own hashed token so they can still reach the resources
+they created. Callers with no admin role and no identifying ids at all
+are denied so an empty user_id can never select an unscoped query.
 """
 
 from typing import Any, Final
@@ -19,6 +20,32 @@ from litellm.proxy._types import (
 )
 
 
+def resolve_resource_owner_id(
+    user_api_key_dict: UserAPIKeyAuth,
+) -> str | None:
+    """Return the identity to stamp on (and match against) a managed
+    resource's ``created_by``.
+
+    A key with neither a user_id nor a team_id would otherwise stamp
+    ``created_by=None`` and be locked out of its own resources, so it owns
+    them under its hashed token instead, using the ``key:`` scope prefix
+    already used by ``proxy/common_utils/resource_ownership.py``. ``None``
+    means the caller has no usable identity of its own and must fall back
+    to team scoping, or be denied.
+    """
+    if user_api_key_dict.user_id is not None:
+        return user_api_key_dict.user_id
+
+    if user_api_key_dict.team_id is not None:
+        return None
+
+    token: Final = user_api_key_dict.token or user_api_key_dict.api_key
+    if token:
+        return f"key:{token}"
+
+    return None
+
+
 def build_list_page(items: list[Any], has_more: bool = False) -> dict[str, Any]:
     """Build the OpenAI-style paginated list response shape used by managed
     file/batch/vector-store listings. ``first_id`` and ``last_id`` are
@@ -39,7 +66,8 @@ def build_owner_filter(
     to records the caller is allowed to see.
 
     - ``{}`` means no scoping (proxy admins).
-    - ``{"created_by": <user_id>}`` for user-keyed callers.
+    - ``{"created_by": <owner_id>}`` for user-keyed callers, and for keys
+      with no user_id and no team_id (owner id is their hashed token).
     - ``{"team_id": <team_id>}`` for service-account callers
       that have a team but no user_id.
     - ``{"OR": [...]}`` when the caller has both — listing must include
@@ -62,12 +90,13 @@ def build_owner_filter(
             ]
         }
 
-    if user_id is not None:
-        return {"created_by": user_id}
-
     if team_id is not None:
         return {"team_id": team_id}
 
+    owner_id: Final = resolve_resource_owner_id(user_api_key_dict)
+    if owner_id is not None:
+        return {"created_by": owner_id}
+
     return None
 
 
@@ -86,8 +115,8 @@ def can_access_resource(
     if _user_has_admin_view(user_api_key_dict):
         return True
 
-    user_id: Final = user_api_key_dict.user_id
-    if user_id is not None and created_by is not None and created_by == user_id:
+    owner_id: Final = resolve_resource_owner_id(user_api_key_dict)
+    if owner_id is not None and created_by is not None and created_by == owner_id:
         return True
 
     team_id: Final = user_api_key_dict.team_id

litellm/llms/bedrock/chat/converse_transformation.py

@@ -924,7 +924,7 @@ class AmazonConverseConfig(BaseConfig):
                         custom_llm_provider="bedrock",
                     )
                     capped = (
-                        AnthropicConfig._cap_thinking_budget_to_max_tokens(legacy_thinking, max_tokens)
+                        AnthropicConfig.cap_thinking_budget_to_max_tokens(legacy_thinking, max_tokens)
                         if legacy_thinking is not None
                         else None
                     )

litellm/llms/bedrock/embed/cohere_transformation.py

@@ -20,7 +20,9 @@ class BedrockCohereEmbeddingConfig:
     def map_openai_params(self, non_default_params: dict, optional_params: dict) -> dict:
         for k, v in non_default_params.items():
             if k == "encoding_format":
-                optional_params["embedding_types"] = v if isinstance(v, list) else [v]
+                optional_params["embedding_types"] = [
+                    "float" if fmt == "base64" else fmt for fmt in (tuple(v) if isinstance(v, list) else (v,))
+                ]
             elif k == "dimensions":
                 optional_params["output_dimension"] = v
         return optional_params

litellm/llms/custom_httpx/llm_http_handler.py

@@ -2872,6 +2872,7 @@ class BaseLLMHTTPHandler:
                     headers=headers,
                     timeout=timeout or float(response_api_optional_request_params.get("timeout", 0)),
                     stream=stream,
+                    logging_obj=logging_obj,
                     **body_kwargs,
                 )
 
@@ -2903,6 +2904,7 @@ class BaseLLMHTTPHandler:
                     url=api_base,
                     headers=headers,
                     timeout=timeout or float(response_api_optional_request_params.get("timeout", 0)),
+                    logging_obj=logging_obj,
                     **body_kwargs,
                 )
 

litellm/llms/databricks/common_utils.py

@@ -13,6 +13,7 @@ Authentication priority:
 import os
 import re
 from typing import Any, Final, Literal
+from urllib.parse import urlsplit, urlunsplit
 
 from litellm.llms.base_llm.chat.transformation import BaseLLMException
 
@@ -224,11 +225,8 @@ class DatabricksBase:
         """
         import requests
 
-        # Extract workspace URL from api_base
-        workspace_url = api_base.rstrip("/")
-        if "/serving-endpoints" in workspace_url:
-            workspace_url = workspace_url.replace("/serving-endpoints", "")
-
+        api_base_parts: Final = urlsplit(api_base)
+        workspace_url: Final = urlunsplit((api_base_parts.scheme, api_base_parts.netloc, "", "", ""))
         token_url: Final = f"{workspace_url}/oidc/v1/token"
 
         try:

litellm/llms/openai/responses/transformation.py

@@ -1,3 +1,5 @@
+from collections.abc import Mapping, Sequence
+from types import MappingProxyType
 from typing import TYPE_CHECKING, Any, Final, cast, get_type_hints
 
 import httpx
@@ -29,6 +31,10 @@ if TYPE_CHECKING:
 else:
     LiteLLMLoggingObj = Any
 
+_NO_TOOL_UPDATE: Final[Mapping[str, object]] = MappingProxyType({})
+_MODEL_FAMILIES_REJECTING_TOP_LEVEL_SCHEMA_COMBINATORS: Final = ("gpt-4", "gpt-3.5", "chatgpt-4o", "o1", "o3", "o4")
+_PROVIDERS_WITH_COMBINATOR_REJECTING_VALIDATOR: Final = frozenset({LlmProviders.AZURE, LlmProviders.OPENAI})
+
 
 class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig):
     @property
@@ -167,8 +173,11 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig):
         input = self._validate_input_param(input)
         tools = response_api_optional_request_params.get("tools")
         input, tools = self.remove_cache_control_flag_from_input_and_tools(model=model, input=input, tools=tools)
-        if tools is not None:
-            response_api_optional_request_params["tools"] = tools
+        sanitized_tools: Final = self._flatten_tool_schema_combinators_for_openai(
+            model=model, tools=tools, litellm_params=litellm_params
+        )
+        if sanitized_tools is not None:
+            response_api_optional_request_params["tools"] = sanitized_tools
         final_request_params: Final = dict(
             ResponsesAPIRequestParams(model=model, input=input, **response_api_optional_request_params)
         )
@@ -207,6 +216,79 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig):
 
         return input, tools
 
+    def _flatten_tool_schema_combinators_for_openai(
+        self,
+        model: str,
+        tools: list[ALL_RESPONSES_API_TOOL_PARAMS] | None,  # mutable-ok: request tools are a JSON list
+        litellm_params: GenericLiteLLMParams,
+    ) -> list[ALL_RESPONSES_API_TOOL_PARAMS] | None:  # mutable-ok: request tools are a JSON list
+        """Flatten top-level schema combinators only where OpenAI's validator rejects them.
+
+        OpenAI-compatible backends reusing this config (and the ChatGPT backend
+        Codex talks to natively) accept them, and so do GPT-5 and later models,
+        which also call tools better with the union intact. Codex wraps MCP tools
+        inside namespace entries, so nested ``tools`` arrays are walked too.
+        Azure OpenAI shares the validator but names deployments arbitrarily, so
+        the router's declared ``model_info.base_model`` wins over the deployment
+        name and an unrecognized name without one is left untouched.
+        """
+        if tools is None or self.custom_llm_provider not in _PROVIDERS_WITH_COMBINATOR_REJECTING_VALIDATOR:
+            return tools
+        gate_model: Final = self._combinator_gate_model(model=model, litellm_params=litellm_params)
+        if not self._rejects_top_level_schema_combinators(gate_model):
+            return tools
+        flattened: Final = [  # mutable-ok: request tools are a JSON list
+            self._flattened_tool_or_passthrough(tool) for tool in tools
+        ]
+        return cast("list[ALL_RESPONSES_API_TOOL_PARAMS]", flattened)  # cast-ok: dict spread keeps each tool's shape
+
+    @staticmethod
+    def _flattened_tool_or_passthrough(tool: object) -> object:
+        return OpenAIResponsesAPIConfig._flattened_tool_entry(tool) if isinstance(tool, dict) else tool
+
+    @staticmethod
+    def _rejects_top_level_schema_combinators(model: str) -> bool:
+        bare_model: Final = model.split("/")[-1]
+        base_model: Final = bare_model.split(":")[1] if bare_model.startswith("ft:") else bare_model
+        return base_model.startswith(_MODEL_FAMILIES_REJECTING_TOP_LEVEL_SCHEMA_COMBINATORS)
+
+    @staticmethod
+    def _combinator_gate_model(model: str, litellm_params: GenericLiteLLMParams) -> str:
+        model_info: Final[object] = getattr(litellm_params, "model_info", None)
+        base_model: Final[object] = model_info.get("base_model") if isinstance(model_info, dict) else None
+        return base_model if isinstance(base_model, str) and base_model else model
+
+    @staticmethod
+    def _flattened_tool_entry(
+        entry: Mapping[str, object],
+    ) -> dict[str, object]:  # mutable-ok: request tools are JSON dicts
+        from litellm.litellm_core_utils.prompt_templates.common_utils import (
+            flatten_top_level_schema_combinators,
+        )
+
+        parameters: Final = entry.get("parameters")
+        nested_tools: Final = entry.get("tools")
+        parameters_update: Final = (
+            MappingProxyType({"parameters": flatten_top_level_schema_combinators(parameters)})
+            if isinstance(parameters, dict)
+            else _NO_TOOL_UPDATE
+        )
+        tools_update: Final = (
+            MappingProxyType({"tools": OpenAIResponsesAPIConfig._flattened_nested_tools(nested_tools)})
+            if isinstance(nested_tools, list)
+            else _NO_TOOL_UPDATE
+        )
+        return {**entry, **parameters_update, **tools_update}  # mutable-ok: request tools are JSON dicts
+
+    @staticmethod
+    def _flattened_nested_tools(
+        nested_tools: Sequence[object],
+    ) -> list[object]:  # mutable-ok: namespace tools are a JSON list
+        return [  # mutable-ok: namespace tools are a JSON list
+            OpenAIResponsesAPIConfig._flattened_tool_entry(item) if isinstance(item, dict) else item
+            for item in nested_tools
+        ]
+
     def _validate_input_param(self, input: str | ResponseInputParam) -> str | ResponseInputParam:
         """
         Ensure all input fields if pydantic are converted to dict
@@ -646,8 +728,11 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig):
         input = self._validate_input_param(input)
         tools = response_api_optional_request_params.get("tools")
         input, tools = self.remove_cache_control_flag_from_input_and_tools(model=model, input=input, tools=tools)
-        if tools is not None:
-            response_api_optional_request_params["tools"] = tools
+        sanitized_tools: Final = self._flatten_tool_schema_combinators_for_openai(
+            model=model, tools=tools, litellm_params=litellm_params
+        )
+        if sanitized_tools is not None:
+            response_api_optional_request_params["tools"] = sanitized_tools
         data: Final = dict(ResponsesAPIRequestParams(model=model, input=input, **response_api_optional_request_params))
 
         return url, data

litellm/llms/soniox/common_utils.py

@@ -138,13 +138,25 @@ def _soniox_token_to_subtitle_token(token: SonioxToken) -> SubtitleToken:
     )
 
 
+def _subtitle_tokens(tokens: Sequence[SonioxToken]) -> tuple[SubtitleToken, ...]:
+    """
+    Convert Soniox tokens for subtitle rendering, excluding translation tokens
+    (``translation_status == "translation"``): Soniox does not timestamp them,
+    so they cannot be aligned to the audio and would otherwise mix translated
+    text into original-language cues.
+    """
+    return tuple(
+        _soniox_token_to_subtitle_token(token) for token in tokens if token.get("translation_status") != "translation"
+    )
+
+
 def render_soniox_tokens_as_srt(tokens: Sequence[SonioxToken]) -> str:
     """
     Render Soniox tokens as SRT (SubRip) subtitle format.
 
     Returns an empty string if no tokens have timestamp data.
     """
-    return render_subtitle_tokens_as_srt(tuple(_soniox_token_to_subtitle_token(token) for token in tokens))
+    return render_subtitle_tokens_as_srt(_subtitle_tokens(tokens))
 
 
 def render_soniox_tokens_as_vtt(tokens: Sequence[SonioxToken]) -> str:
@@ -153,4 +165,4 @@ def render_soniox_tokens_as_vtt(tokens: Sequence[SonioxToken]) -> str:
 
     Returns the VTT header even if no cues are present.
     """
-    return render_subtitle_tokens_as_vtt(tuple(_soniox_token_to_subtitle_token(token) for token in tokens))
+    return render_subtitle_tokens_as_vtt(_subtitle_tokens(tokens))

litellm/llms/vertex_ai/audio_transcription/gemini_transcribe_transformation.py

@@ -0,0 +1,216 @@
+import base64
+from collections.abc import Mapping, Sequence
+from typing import Final
+
+from httpx import Headers, Response
+
+import litellm
+from litellm.exceptions import UnsupportedParamsError
+from litellm.litellm_core_utils.audio_utils.utils import (
+    normalize_transcription_language_to_bcp47,
+    process_audio_file,
+)
+from litellm.llms.base_llm.audio_transcription.transformation import (
+    AudioTranscriptionRequestData,
+    BaseAudioTranscriptionConfig,
+)
+from litellm.llms.base_llm.chat.transformation import BaseLLMException
+from litellm.llms.vertex_ai.audio_transcription.transformation import (
+    SUPPORTED_RESPONSE_FORMATS,
+    validate_vertex_transcription_location,
+    validate_vertex_transcription_project_id,
+)
+from litellm.llms.vertex_ai.common_utils import VertexAIError, get_vertex_base_url
+from litellm.llms.vertex_ai.vertex_llm_base import VertexBase
+from litellm.types.llms.openai import (
+    AllMessageValues,
+    OpenAIAudioTranscriptionOptionalParams,
+)
+from litellm.types.llms.vertex_ai_gemini_transcription import (
+    VertexGeminiTranscriptionAudioConfig,
+    VertexGeminiTranscriptionContent,
+    VertexGeminiTranscriptionGenerationConfig,
+    VertexGeminiTranscriptionInlineData,
+    VertexGeminiTranscriptionPart,
+    VertexGeminiTranscriptionRequest,
+    VertexGeminiTranscriptionResponse,
+)
+from litellm.types.utils import (
+    FileTypes,
+    TranscriptionResponse,
+    TranscriptionUsageInputTokenDetailsObject,
+    TranscriptionUsageTokensObject,
+)
+
+DEFAULT_GEMINI_TRANSCRIBE_LOCATION: Final = "global"
+AUDIO_MODALITY: Final = "AUDIO"
+
+
+class VertexGeminiAudioTranscriptionConfig(BaseAudioTranscriptionConfig, VertexBase):
+    def __init__(self) -> None:
+        BaseAudioTranscriptionConfig.__init__(self)
+        VertexBase.__init__(self)
+
+    def get_supported_openai_params(
+        self, model: str
+    ) -> list[OpenAIAudioTranscriptionOptionalParams]:  # mutable-ok: BaseAudioTranscriptionConfig signature
+        return ["language", "response_format"]
+
+    def map_openai_params(
+        self,
+        non_default_params: Mapping[str, object],
+        optional_params: Mapping[str, object],
+        model: str,
+        drop_params: bool,
+    ) -> dict[str, object]:  # mutable-ok: BaseAudioTranscriptionConfig signature
+        supported_params: Final = frozenset(self.get_supported_openai_params(model))
+        mapped: Final = {
+            **optional_params,
+            **{k: v for k, v in non_default_params.items() if k in supported_params},
+        }
+        response_format: Final = mapped.get("response_format")
+        if response_format is None or response_format in SUPPORTED_RESPONSE_FORMATS:
+            return mapped
+        if drop_params or litellm.drop_params:
+            return {k: v for k, v in mapped.items() if k != "response_format"}
+        raise UnsupportedParamsError(
+            status_code=400,
+            message=(
+                f"Vertex AI Gemini transcription does not support response_format={response_format!r}. "
+                f"Supported values: {', '.join(SUPPORTED_RESPONSE_FORMATS)}. "
+                "To drop unsupported openai params from the call, set `litellm.drop_params = True`"
+            ),
+        )
+
+    def get_error_class(
+        self,
+        error_message: str,
+        status_code: int,
+        headers: dict | Headers,  # mutable-ok: base signature and VertexAIError take dict | Headers
+    ) -> BaseLLMException:
+        return VertexAIError(status_code=status_code, message=error_message, headers=headers)
+
+    def validate_environment(
+        self,
+        headers: Mapping[str, str],
+        model: str,
+        messages: Sequence[AllMessageValues],
+        optional_params: Mapping[str, object],
+        litellm_params: Mapping[str, object],
+        api_key: str | None = None,
+        api_base: str | None = None,
+    ) -> dict[str, str]:  # mutable-ok: BaseAudioTranscriptionConfig signature
+        vertex_params: Final = dict(litellm_params)
+        access_token, project_id = self._ensure_access_token(
+            credentials=self.safe_get_vertex_ai_credentials(vertex_params),
+            project_id=self.safe_get_vertex_ai_project(vertex_params),
+            custom_llm_provider="vertex_ai",
+        )
+        return {
+            **headers,
+            "Authorization": f"Bearer {access_token}",
+            "x-goog-user-project": project_id,
+            "Content-Type": "application/json",
+        }
+
+    def get_complete_url(
+        self,
+        api_base: str | None,
+        api_key: str | None,
+        model: str,
+        optional_params: Mapping[str, object],
+        litellm_params: Mapping[str, object],
+        stream: bool | None = None,
+    ) -> str:
+        vertex_params: Final = dict(litellm_params)
+        location: Final = validate_vertex_transcription_location(
+            self.safe_get_vertex_ai_location(vertex_params), default_location=DEFAULT_GEMINI_TRANSCRIBE_LOCATION
+        )
+        project_id: Final = validate_vertex_transcription_project_id(
+            self.safe_get_vertex_ai_project(vertex_params) or self._resolve_project_id_from_credentials(vertex_params)
+        )
+        base_url: Final = (api_base or get_vertex_base_url(location)).rstrip("/")
+        bare_model: Final = model.removeprefix("vertex_ai/")
+        model_path: Final = f"projects/{project_id}/locations/{location}/publishers/google/models/{bare_model}"
+        return f"{base_url}/v1/{model_path}:generateContent"
+
+    def _resolve_project_id_from_credentials(self, litellm_params: Mapping[str, object]) -> str:
+        vertex_params: Final = dict(litellm_params)
+        _, project_id = self._ensure_access_token(
+            credentials=self.safe_get_vertex_ai_credentials(vertex_params),
+            project_id=None,
+            custom_llm_provider="vertex_ai",
+        )
+        return project_id
+
+    def transform_audio_transcription_request(
+        self,
+        model: str,
+        audio_file: FileTypes,
+        optional_params: Mapping[str, object],
+        litellm_params: Mapping[str, object],
+    ) -> AudioTranscriptionRequestData:
+        processed_audio: Final = process_audio_file(audio_file)
+        request_body: Final = VertexGeminiTranscriptionRequest(
+            contents=(
+                VertexGeminiTranscriptionContent(
+                    role="user",
+                    parts=(
+                        VertexGeminiTranscriptionPart(
+                            inlineData=VertexGeminiTranscriptionInlineData(
+                                mimeType=processed_audio.content_type,
+                                data=base64.b64encode(processed_audio.file_content).decode("utf-8"),
+                            )
+                        ),
+                    ),
+                ),
+            ),
+            generationConfig=VertexGeminiTranscriptionGenerationConfig(
+                audioTranscriptionConfig=_audio_transcription_config(optional_params.get("language"))
+            ),
+        )
+        return AudioTranscriptionRequestData(data=dict(request_body))
+
+    def transform_audio_transcription_response(
+        self,
+        raw_response: Response,
+    ) -> TranscriptionResponse:
+        try:
+            response_json: Final = raw_response.json()
+        except ValueError:
+            raise VertexAIError(
+                status_code=raw_response.status_code,
+                message=f"Received non-JSON response from Vertex AI Gemini transcription: {raw_response.text}",
+            )
+        parsed: Final = VertexGeminiTranscriptionResponse.model_validate(response_json)
+        texts: Final = tuple(
+            part.text
+            for candidate in parsed.candidates
+            if candidate.content is not None
+            for part in candidate.content.parts
+            if part.text
+        )
+        response: Final = TranscriptionResponse(text=" ".join(texts))
+        response["task"] = "transcribe"
+        usage: Final = parsed.usageMetadata
+        if usage is not None:
+            audio_tokens: Final = sum(
+                detail.tokenCount for detail in usage.promptTokensDetails if detail.modality == AUDIO_MODALITY
+            )
+            response.usage = TranscriptionUsageTokensObject(
+                type="tokens",
+                input_tokens=usage.promptTokenCount,
+                output_tokens=usage.candidatesTokenCount,
+                total_tokens=usage.totalTokenCount,
+                input_token_details=TranscriptionUsageInputTokenDetailsObject(
+                    audio_tokens=audio_tokens,
+                    text_tokens=usage.promptTokenCount - audio_tokens,
+                ),
+            )
+        return response
+
+
+def _audio_transcription_config(language: object) -> VertexGeminiTranscriptionAudioConfig:
+    if not isinstance(language, str) or not language:
+        return VertexGeminiTranscriptionAudioConfig()
+    return VertexGeminiTranscriptionAudioConfig(languageCodes=(normalize_transcription_language_to_bcp47(language),))

litellm/llms/vertex_ai/audio_transcription/transformation.py

@@ -35,6 +35,19 @@ SUPPORTED_RESPONSE_FORMATS: Final = ("json", "text")
 _URL_UNSAFE_PROJECT_CHARS: Final = ("/", "?", "#", "\\", ":", " ", "\t", "\n", "\r")
 
 
+def validate_vertex_transcription_location(location: str | None, default_location: str) -> str:
+    try:
+        return validate_vertex_location(location or default_location)
+    except ValueError as e:
+        raise VertexAIError(status_code=400, message=str(e)) from e
+
+
+def validate_vertex_transcription_project_id(project_id: str) -> str:
+    if not project_id or ".." in project_id or any(c in project_id for c in _URL_UNSAFE_PROJECT_CHARS):
+        raise VertexAIError(status_code=400, message=f"Invalid vertex_project format: {project_id!r}")
+    return project_id
+
+
 class VertexAIAudioTranscriptionConfig(BaseAudioTranscriptionConfig, VertexBase):
     def __init__(self) -> None:
         BaseAudioTranscriptionConfig.__init__(self)
@@ -103,27 +116,16 @@ class VertexAIAudioTranscriptionConfig(BaseAudioTranscriptionConfig, VertexBase)
         litellm_params: dict,
         stream: bool | None = None,
     ) -> str:
-        location: Final = self._validate_location(self.safe_get_vertex_ai_location(litellm_params))
-        project_id: Final = self._validate_project_id(
+        location: Final = validate_vertex_transcription_location(
+            self.safe_get_vertex_ai_location(litellm_params), default_location=DEFAULT_SPEECH_TO_TEXT_LOCATION
+        )
+        project_id: Final = validate_vertex_transcription_project_id(
             self.safe_get_vertex_ai_project(litellm_params) or self._resolve_project_id_from_credentials(litellm_params)
         )
         host: Final = "speech.googleapis.com" if location == "global" else f"{location}-speech.googleapis.com"
         base_url: Final = (api_base or f"https://{host}").rstrip("/")
         return f"{base_url}/v2/projects/{project_id}/locations/{location}/recognizers/_:recognize"
 
-    @staticmethod
-    def _validate_location(location: str | None) -> str:
-        try:
-            return validate_vertex_location(location or DEFAULT_SPEECH_TO_TEXT_LOCATION)
-        except ValueError as e:
-            raise VertexAIError(status_code=400, message=str(e)) from e
-
-    @staticmethod
-    def _validate_project_id(project_id: str) -> str:
-        if not project_id or ".." in project_id or any(c in project_id for c in _URL_UNSAFE_PROJECT_CHARS):
-            raise VertexAIError(status_code=400, message=f"Invalid vertex_project format: {project_id!r}")
-        return project_id
-
     def _resolve_project_id_from_credentials(self, litellm_params: dict) -> str:
         _, project_id = self._ensure_access_token(
             credentials=self.safe_get_vertex_ai_credentials(litellm_params),

litellm/llms/vertex_ai/videos/transformation.py

@@ -8,12 +8,14 @@ Based on: https://docs.cloud.google.com/vertex-ai/generative-ai/docs/model-refer
 import base64
 import time
 from collections.abc import Mapping, Sequence
-from typing import TYPE_CHECKING, Any, Final, TypedDict, cast
+from types import MappingProxyType
+from typing import TYPE_CHECKING, Any, ClassVar, Final, TypedDict, cast
 
 import httpx
 from httpx._types import FileContent, RequestFiles
 from typing_extensions import ReadOnly
 
+import litellm
 from litellm.constants import DEFAULT_GOOGLE_VIDEO_DURATION_SECONDS
 from litellm.images.utils import ImageEditRequestUtils
 from litellm.llms.base_llm.videos.transformation import BaseVideoConfig
@@ -119,6 +121,23 @@ class VertexAIVideoConfig(BaseVideoConfig, VertexBase):
     3. Extract video data (base64) from response
     """
 
+    _OPENAI_VIDEO_SIZE_TO_ASPECT_RATIO: ClassVar[Mapping[str, str]] = MappingProxyType(
+        {
+            "1280x720": "16:9",
+            "1920x1080": "16:9",
+            "720x1280": "9:16",
+            "1080x1920": "9:16",
+        }
+    )
+    _OPENAI_VIDEO_SIZE_TO_RESOLUTION: ClassVar[Mapping[str, str]] = MappingProxyType(
+        {
+            "1280x720": "720p",
+            "1920x1080": "1080p",
+            "720x1280": "720p",
+            "1080x1920": "1080p",
+        }
+    )
+
     def __init__(self):
         BaseVideoConfig.__init__(self)
         VertexBase.__init__(self)
@@ -161,6 +180,9 @@ class VertexAIVideoConfig(BaseVideoConfig, VertexBase):
         - prompt → prompt (in instances)
         - input_reference → image (in instances)
         - size → aspectRatio (e.g., "1280x720" → "16:9")
+        - size → resolution for models with resolution-tier pricing when inferable
+          ("1280x720"/"720x1280" → "720p", "1920x1080"/"1080x1920" → "1080p");
+          skipped if ``resolution`` is already set
         - seconds → durationSeconds (defaults to 4 seconds if not provided)
         """
         mapped_params: Final[dict[str, object]] = {}
@@ -175,6 +197,9 @@ class VertexAIVideoConfig(BaseVideoConfig, VertexBase):
         if "parameters" in video_create_optional_params:
             mapped_params["parameters"] = video_create_optional_params["parameters"]
 
+        if "resolution" in video_create_optional_params:
+            mapped_params["resolution"] = video_create_optional_params["resolution"]
+
         # Map size to aspectRatio
         if "size" in video_create_optional_params:
             size: Final = video_create_optional_params["size"]
@@ -182,6 +207,15 @@ class VertexAIVideoConfig(BaseVideoConfig, VertexBase):
                 aspect_ratio: Final = self._convert_size_to_aspect_ratio(size)
                 if aspect_ratio:
                     mapped_params["aspectRatio"] = aspect_ratio
+                nested_params: Final = video_create_optional_params.get("parameters")
+                has_resolution = "resolution" in mapped_params or (
+                    isinstance(nested_params, dict) and nested_params.get("resolution") is not None
+                )
+                supports_resolution = self._supports_resolution_inference(model)
+                if supports_resolution and not has_resolution:
+                    inferred_resolution = self._convert_size_to_resolution(size)
+                    if inferred_resolution is not None:
+                        mapped_params["resolution"] = inferred_resolution
 
         # Map seconds to durationSeconds, default to 4 seconds (matching OpenAI)
         if "seconds" in video_create_optional_params:
@@ -205,14 +239,16 @@ class VertexAIVideoConfig(BaseVideoConfig, VertexBase):
         if not size:
             return None
 
-        aspect_ratio_map: Final = {
-            "1280x720": "16:9",
-            "1920x1080": "16:9",
-            "720x1280": "9:16",
-            "1080x1920": "9:16",
-        }
+        return self._OPENAI_VIDEO_SIZE_TO_ASPECT_RATIO.get(size, "16:9")
 
-        return aspect_ratio_map.get(size, "16:9")
+    def _convert_size_to_resolution(self, size: str) -> str | None:
+        return self._OPENAI_VIDEO_SIZE_TO_RESOLUTION.get(size)
+
+    @staticmethod
+    def _supports_resolution_inference(model: str) -> bool:
+        model_key: Final = model if model.startswith("vertex_ai/") else f"vertex_ai/{model}"
+        model_info: Final = litellm.model_cost.get(model_key)
+        return model_info is not None and model_info.get("output_cost_per_second_1080p") is not None
 
     def validate_environment(
         self,

litellm/main.py

@@ -8612,9 +8612,9 @@ def _joined_streamed_citations(streamed_citations: "tuple[object, ...]") -> "lis
 
 
 def _stream_builder_model_map_cost(response: ModelResponse) -> float | None:
-    model_name: Final = getattr(response, "model", None)
+    model_name: Final = response.model
     usage: Final = getattr(response, "usage", None)
-    if not isinstance(model_name, str) or not model_name or not isinstance(usage, Usage):
+    if not model_name or not isinstance(usage, Usage):
         return None
     try:
         prompt_cost, completion_tokens_cost = litellm.cost_per_token(model=model_name, usage_object=usage)

litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py

@@ -903,8 +903,9 @@ class MCPRequestHandler:
             NotSessionBearer,
             SessionBearerAdmitted,
             SessionBearerInvalid,
+            SessionSigningConfigError,
+            active_session_signing_keys,
             resolve_session_bearer,
-            session_keys_from_master_key,
         )
         from litellm.proxy.proxy_server import master_key
 
@@ -913,7 +914,10 @@ class MCPRequestHandler:
 
         await MCPRequestHandler._run_pre_db_read_auth_checks(request=request, route=route)
 
-        keys: Final = session_keys_from_master_key(master_key)
+        keys: Final = active_session_signing_keys(master_key)
+        if isinstance(keys, SessionSigningConfigError):
+            verbose_logger.error("mcp gateway session admission rejected: %s", keys.detail)
+            raise HTTPException(status_code=500, detail="Server misconfigured: mcp_session_token_signing is invalid")
         result: Final = resolve_session_bearer(authorization_value, keys, datetime.now(timezone.utc))
         match result:
             case SessionBearerAdmitted():

litellm/proxy/_experimental/mcp_server/db.py

@@ -600,23 +600,19 @@ async def get_all_mcp_servers(
     NULL approval_status predates the approval workflow, so those rows are kept explicitly rather
     than dropped by a bare inequality, which SQL evaluates as NULL and would silently hide them.
     """
-    try:
-        where: Final[prisma_db_types.LiteLLM_MCPServerTableWhereInput] = (
-            {"approval_status": approval_status}
-            if approval_status is not None
-            # mutable-ok: prisma where-inputs must be plain dicts, and both `NOT` and `not` drop
-            # NULL rows (measured), so the OR is the only NULL-preserving way to exclude drafts
-            else {"OR": [{"approval_status": None}, {"approval_status": {"not": MCPApprovalStatus.draft}}]}
-        )
-        mcp_servers: Final = await _db_find_mcp_server_rows(prisma_client, where)
+    where: Final[prisma_db_types.LiteLLM_MCPServerTableWhereInput] = (
+        {"approval_status": approval_status}
+        if approval_status is not None
+        # mutable-ok: prisma where-inputs must be plain dicts, and both `NOT` and `not` drop
+        # NULL rows (measured), so the OR is the only NULL-preserving way to exclude drafts
+        else {"OR": [{"approval_status": None}, {"approval_status": {"not": MCPApprovalStatus.draft}}]}
+    )
+    mcp_servers: Final = await _db_find_mcp_server_rows(prisma_client, where)
 
-        tables: Final = [LiteLLM_MCPServerTable.model_validate(mcp_server.model_dump()) for mcp_server in mcp_servers]
-        for table in tables:
-            decrypt_global_env_var_values(table.env_vars)
-        return tables
-    except Exception as e:
-        verbose_proxy_logger.debug("litellm.proxy._experimental.mcp_server.db.py::get_all_mcp_servers - %s", e)
-        return []
+    tables: Final = [LiteLLM_MCPServerTable.model_validate(mcp_server.model_dump()) for mcp_server in mcp_servers]
+    for table in tables:
+        decrypt_global_env_var_values(table.env_vars)
+    return tables
 
 
 async def get_mcp_server(prisma_client: PrismaClient, server_id: str) -> LiteLLM_MCPServerTable | None:

litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py

@@ -9,7 +9,7 @@ from typing import TYPE_CHECKING, Any, Final, Literal, Optional
 from urllib.parse import parse_qsl, urlencode, urlparse, urlunparse
 
 import httpx
-from fastapi import APIRouter, Form, HTTPException, Request
+from fastapi import APIRouter, Depends, Form, HTTPException, Request
 from fastapi.responses import HTMLResponse, JSONResponse, RedirectResponse, Response
 from pydantic import BaseModel, ConfigDict, Field, SecretStr, ValidationError
 
@@ -46,6 +46,7 @@ from litellm.proxy._experimental.mcp_server.gateway_dcr_flow import (
     aggregate_authorize,
     aggregate_token,
     complete_connect_flow,
+    introspect_gateway_token,
     is_gateway_dcr_client_id,
     is_proxy_api_resource,
     native_client_auth_contract,
@@ -67,6 +68,7 @@ from litellm.proxy._experimental.mcp_server.proxy_api_credentials import (
     mint_proxy_credential,
 )
 from litellm.proxy.auth.ip_address_utils import IPAddressUtils
+from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
 from litellm.proxy.common_utils.encrypt_decrypt_utils import (
     decrypt_value_helper,
     encrypt_value_helper,
@@ -1951,6 +1953,26 @@ async def revoke_endpoint(request: Request, token: str = Form(...), client_id: s
     return await revoke_refresh_token(token=token, client_id=client_id, master_key=master_key, cache=user_api_key_cache)
 
 
+@router.post("/introspect", dependencies=[Depends(user_api_key_auth)])
+async def introspect_endpoint(token: str = Form(...)) -> Response:
+    """RFC 7662 introspection for gateway-issued session tokens (``llm_session_`` /
+    ``llm_srefresh_``), so an external gateway can validate them without the signing
+    secret. The caller authenticates with a LiteLLM virtual key (section 2.1, enforced by
+    the route dependency); any token the gateway cannot vouch for answers
+    ``{"active": false}`` with no further detail."""
+    from litellm.proxy.proxy_server import (  # noqa: PLC0415  # circular import at module load
+        master_key,
+        user_api_key_cache,
+    )
+
+    return await introspect_gateway_token(
+        token=token,
+        master_key=master_key,
+        reload_user=_reload_active_user_by_id,
+        cache=user_api_key_cache,
+    )
+
+
 @router.get("/.well-known/litellm-cli-auth")
 async def native_client_auth_discovery(request: Request) -> JSONResponse:
     """The versioned contract a native client (``lite login --pkce``, or a CLI in any other
@@ -2456,6 +2478,7 @@ def _build_aggregate_authorization_server_response(request: Request) -> dict:
         "issuer": f"{request_base_url}/mcp",
         "authorization_endpoint": f"{request_base_url}/authorize",
         "token_endpoint": f"{request_base_url}/token",
+        "introspection_endpoint": f"{request_base_url}/introspect",
         "registration_endpoint": f"{request_base_url}/register",
         "response_types_supported": ["code"],
         "scopes_supported": [],

litellm/proxy/_experimental/mcp_server/gateway_dcr_flow.py

@@ -65,17 +65,24 @@ from litellm.proxy._experimental.mcp_server.oauth_utils import (
 )
 from litellm.proxy._experimental.mcp_server.outbound_credentials.session_credentials import (
     SessionRefreshOpened,
+    SessionSigningConfigError,
+    active_session_signing_keys,
     open_session_refresh_bearer,
-    session_keys_from_master_key,
 )
 from litellm.proxy._experimental.mcp_server.outbound_credentials.session_token import (
+    SESSION_ISSUER,
     SESSION_REFRESH_TTL_SECONDS,
     MintedSessionToken,
+    OpenedSessionToken,
     SessionAudience,
-    SessionKeys,
     SessionPrincipal,
+    SessionSigningKeys,
+    is_session_refresh_token,
+    is_session_token,
     mint_session_refresh_token,
     mint_session_token,
+    open_session_refresh_token,
+    open_session_token,
 )
 from litellm.proxy.common_utils.encrypt_decrypt_utils import (
     decrypt_value_helper,
@@ -884,8 +891,25 @@ class _SingleUseGuard:
         count = await self._cache.async_increment_cache(key, 1, ttl=ttl_seconds, local_only=True)
         return "first" if count == 1 else "replayed"
 
+    async def peek(self, key: str) -> Literal["unclaimed", "claimed", "unavailable"]:
+        """Read-only view of a single-use marker, resolved against the same shared authority as
+        :meth:`claim` so introspection observes exactly the record redemption and revocation wrote.
+        A backend fault is ``"unavailable"`` (fail closed) rather than a guess either way."""
+        from litellm.proxy.proxy_server import redis_usage_cache  # noqa: PLC0415  # circular import at module load
 
-def _session_token_pair(principal: SessionPrincipal, keys: SessionKeys, now: datetime) -> Response:
+        redis_cache: Final = redis_usage_cache or getattr(self._cache, "redis_cache", None)
+        if redis_cache is not None:
+            try:
+                value = await redis_cache.async_get_cache(key)
+            except Exception as e:  # noqa: BLE001  # ANY Redis fault fails the read closed
+                verbose_logger.warning("mcp gateway single-use peek: shared cache backend unavailable: %s", e)
+                return "unavailable"
+            return "unclaimed" if value is None else "claimed"
+        local: Final = await self._cache.async_get_cache(key, local_only=True)
+        return "unclaimed" if local is None else "claimed"
+
+
+def _session_token_pair(principal: SessionPrincipal, keys: SessionSigningKeys, now: datetime) -> Response:
     access: Final = mint_session_token(principal, keys, now)
     refresh: Final = mint_session_refresh_token(principal, keys, now)
     if not isinstance(access, MintedSessionToken) or not isinstance(refresh, MintedSessionToken):
@@ -912,7 +936,7 @@ class _ProxyCredentialTokenResponse(TypedDict):
 
 
 def _proxy_credential_response(
-    minted: MintedProxyCredential, principal: SessionPrincipal, keys: SessionKeys, now: datetime
+    minted: MintedProxyCredential, principal: SessionPrincipal, keys: SessionSigningKeys, now: datetime
 ) -> Response:
     """The proxy-API token response: the access token is the very credential ``lite
     login`` stores (accepted on every proxy route with user and team attribution), and
@@ -998,7 +1022,10 @@ async def aggregate_token(
     if master_key is None:
         verbose_logger.error("mcp_gateway_dcr token grant rejected: no master_key configured")
         return _oauth_error(500, "server_error", "the gateway has no master key configured")
-    keys: Final = session_keys_from_master_key(master_key)
+    keys: Final = active_session_signing_keys(master_key)
+    if isinstance(keys, SessionSigningConfigError):
+        verbose_logger.error("mcp_gateway_dcr token grant rejected: %s", keys.detail)
+        return _oauth_error(500, "server_error", "the gateway session signing configuration is invalid")
     now: Final = datetime.now(timezone.utc)
     issue: Final = _GrantIssuer(
         request=request,
@@ -1043,7 +1070,7 @@ class _GrantIssuer:
         self,
         request: Request,
         resource: str | None,
-        keys: SessionKeys,
+        keys: SessionSigningKeys,
         now: datetime,
         reload_user: ReloadUser,
         mint_proxy_credential: MintProxyCredential,
@@ -1146,7 +1173,7 @@ async def _refresh_token_grant(
     refresh_token: str | None,
     client_id: str,
     resource: str | None,
-    keys: SessionKeys,
+    keys: SessionSigningKeys,
     now: datetime,
     issue: _GrantIssuer,
 ) -> Response:
@@ -1182,7 +1209,10 @@ async def revoke_refresh_token(token: str, client_id: str, master_key: str | Non
     if master_key is None:
         verbose_logger.error("mcp_gateway_dcr revoke rejected: no master_key configured")
         return _oauth_error(500, "server_error", "the gateway has no master key configured")
-    keys: Final = session_keys_from_master_key(master_key)
+    keys: Final = active_session_signing_keys(master_key)
+    if isinstance(keys, SessionSigningConfigError):
+        verbose_logger.error("mcp_gateway_dcr revoke rejected: %s", keys.detail)
+        return _oauth_error(500, "server_error", "the gateway session signing configuration is invalid")
     now: Final = datetime.now(timezone.utc)
     opened: Final = open_session_refresh_bearer(token, keys, now, expected_client_id=client_id)
     if isinstance(opened, SessionRefreshOpened):
@@ -1192,3 +1222,83 @@ async def revoke_refresh_token(token: str, client_id: str, master_key: str | Non
         if burned == "unavailable":
             return _oauth_error(503, "temporarily_unavailable", _CLAIM_UNAVAILABLE_DESCRIPTION)
     return Response(content="{}", media_type="application/json", headers=TOKEN_NO_CACHE_HEADERS)
+
+
+def _inactive_introspection_response() -> Response:
+    """RFC 7662 section 2.2: any token the gateway cannot vouch for, whatever the reason
+    (wrong family, bad signature, expired, revoked, or a deactivated user), answers 200
+    with ``active: false`` and nothing else, so introspection is not a token oracle."""
+    return JSONResponse(status_code=200, content={"active": False}, headers=TOKEN_NO_CACHE_HEADERS)
+
+
+def _active_introspection_response(opened: OpenedSessionToken) -> Response:
+    principal: Final = opened.principal
+    optional_claims: Final = {
+        key: value
+        for key, value in (
+            ("token_type", "Bearer" if opened.kind == "session" else None),
+            ("team_id", principal.team_id),
+            ("resource_server_id", principal.resource_server_id),
+            ("audience", principal.audience),
+        )
+        if value is not None
+    }
+    return JSONResponse(
+        status_code=200,
+        content={
+            "active": True,
+            "iss": SESSION_ISSUER,
+            "sub": principal.user_id,
+            "client_id": principal.client_id,
+            "jti": opened.jti,
+            "iat": opened.iat,
+            "exp": opened.exp,
+            "kind": opened.kind,
+            **optional_claims,
+        },
+        headers=TOKEN_NO_CACHE_HEADERS,
+    )
+
+
+async def introspect_gateway_token(
+    token: str,
+    master_key: str | None,
+    reload_user: ReloadUser,
+    cache: DualCache,
+) -> Response:
+    """RFC 7662 introspection for the gateway's session tokens, so an external gateway
+    (Kong, an API management layer) can validate a LiteLLM-issued MCP session credential
+    without holding the signing secret. The caller is already authenticated by the route
+    (section 2.1). Active means everything admission itself would require: valid signature
+    under the configured session signing keys, unexpired, not a revoked or rotated refresh
+    token, and a litellm user that is still live, so a deactivated user's outstanding
+    tokens introspect as inactive immediately. A shared-backend or DB outage answers 503
+    rather than guessing in either direction."""
+    if master_key is None:
+        verbose_logger.error("mcp_gateway_dcr introspect rejected: no master_key configured")
+        return _oauth_error(500, "server_error", "the gateway has no master key configured")
+    keys: Final = active_session_signing_keys(master_key)
+    if isinstance(keys, SessionSigningConfigError):
+        verbose_logger.error("mcp_gateway_dcr introspect rejected: %s", keys.detail)
+        return _oauth_error(500, "server_error", keys.detail)
+    now: Final = datetime.now(timezone.utc)
+    if is_session_token(token):
+        opened = open_session_token(token, keys, now)
+    elif is_session_refresh_token(token):
+        opened = open_session_refresh_token(token, keys, now)
+    else:
+        return _inactive_introspection_response()
+    if not isinstance(opened, OpenedSessionToken):
+        return _inactive_introspection_response()
+    if opened.kind == "session_refresh":
+        peeked: Final = await _SingleUseGuard(cache).peek(f"{_USED_REFRESH_CACHE_PREFIX}{opened.jti}")
+        if peeked == "unavailable":
+            return _oauth_error(503, "temporarily_unavailable", _CLAIM_UNAVAILABLE_DESCRIPTION)
+        if peeked == "claimed":
+            return _inactive_introspection_response()
+    failure: Final = await reload_user(opened.principal.user_id)
+    if failure == "unavailable":
+        return _oauth_error(503, "temporarily_unavailable", "the gateway database is unavailable; retry")
+    if failure is not None:
+        return _inactive_introspection_response()
+    return _active_introspection_response(opened)

litellm/proxy/_experimental/mcp_server/outbound_credentials/session_credentials.py

@@ -20,13 +20,16 @@ from datetime import datetime
 from functools import lru_cache
 from typing import Final, Literal, TypeAlias
 
-from pydantic import BaseModel, ConfigDict, SecretStr
+from pydantic import BaseModel, ConfigDict, Field, SecretStr, ValidationError
 
 from litellm.proxy._experimental.mcp_server.outbound_credentials.session_token import (
+    AsymmetricSessionKeys,
     OpenedSessionToken,
     SessionExpired,
     SessionKeys,
     SessionPrincipal,
+    SessionRotatedPublicKey,
+    SessionSigningKeys,
     is_session_refresh_token,
     is_session_token,
     open_session_refresh_token,
@@ -68,6 +71,99 @@ def session_keys_from_master_key(master_key: str) -> SessionKeys:
     return SessionKeys(signing_key=SecretStr(signing))
 
 
+class SessionSigningPreviousKey(BaseModel):
+    """One retired key in ``mcp_session_token_signing.previous_public_keys``: its ``kid``
+    and the PEM public half (inline or an ``os.environ/`` reference)."""
+
+    model_config = ConfigDict(frozen=True, extra="forbid")
+    kid: str = Field(min_length=1)
+    public_key: str = Field(min_length=1)
+
+
+class MCPSessionTokenSigningSettings(BaseModel):
+    """The ``general_settings.mcp_session_token_signing`` block: opt-in asymmetric signing
+    for the gateway session tokens. Absent, the gateway keeps the backward-compatible
+    HS256 key derived from ``master_key``. ``private_key`` and each ``public_key`` accept
+    a PEM string inline or an ``os.environ/<NAME>`` (or secret manager) reference."""
+
+    model_config = ConfigDict(frozen=True, extra="forbid")
+    algorithm: Literal["RS256"]
+    kid: str = Field(min_length=1)
+    private_key: str = Field(min_length=1)
+    previous_public_keys: tuple[SessionSigningPreviousKey, ...] = ()
+
+
+class SessionSigningConfigError(BaseModel):
+    """``mcp_session_token_signing`` is present but unusable (bad shape, unresolvable
+    secret reference, or a key that is not a loadable RSA PEM); the caller fails closed
+    with a server error instead of silently falling back to HS256."""
+
+    model_config = ConfigDict(frozen=True)
+    tag: Literal["session_signing_config_error"] = "session_signing_config_error"
+    detail: str
+
+
+def _resolve_key_material(value: str) -> str | None:
+    if not value.startswith("os.environ/"):
+        return value
+    from litellm.secret_managers.main import get_secret_str  # noqa: PLC0415  # heavy import kept off the pure path
+
+    return get_secret_str(value)
+
+
+def resolve_session_signing_keys(
+    master_key: str,
+    raw_settings: object | None,
+) -> SessionSigningKeys | SessionSigningConfigError:
+    """Turn the operator's ``mcp_session_token_signing`` setting into signing key material.
+
+    ``None`` (the setting absent) keeps the backward-compatible HS256 key derived from
+    ``master_key``. A present setting must fully validate into RS256 material; any defect
+    is a ``SessionSigningConfigError`` value so token issuance and admission fail closed
+    rather than minting under a key the operator did not intend.
+    """
+    if raw_settings is None:
+        return session_keys_from_master_key(master_key)
+    try:
+        settings: Final = MCPSessionTokenSigningSettings.model_validate(raw_settings)
+    except ValidationError as exc:
+        return SessionSigningConfigError(detail=f"mcp_session_token_signing is malformed: {exc}")
+    private_pem: Final = _resolve_key_material(settings.private_key)
+    if private_pem is None:
+        return SessionSigningConfigError(detail="mcp_session_token_signing.private_key reference did not resolve")
+    resolved_previous: Final = tuple(
+        (previous.kid, _resolve_key_material(previous.public_key)) for previous in settings.previous_public_keys
+    )
+    unresolved: Final = tuple(kid for kid, pem in resolved_previous if pem is None)
+    if unresolved:
+        return SessionSigningConfigError(
+            detail=f"mcp_session_token_signing.previous_public_keys reference did not resolve for kid(s): {', '.join(unresolved)}"
+        )
+    try:
+        return AsymmetricSessionKeys(
+            private_key_pem=SecretStr(private_pem),
+            kid=settings.kid,
+            previous_public_keys=tuple(
+                SessionRotatedPublicKey(kid=kid, public_key_pem=pem)
+                for kid, pem in resolved_previous
+                if pem is not None
+            ),
+        )
+    except ValidationError as exc:
+        return SessionSigningConfigError(
+            detail=f"mcp_session_token_signing keys are not usable RSA PEM material: {exc}"
+        )
+
+
+def active_session_signing_keys(master_key: str) -> SessionSigningKeys | SessionSigningConfigError:
+    """Wiring helper for the token endpoint and the admission edge: resolve the signing
+    keys from the live ``general_settings.mcp_session_token_signing`` block, or derive the
+    default HS256 key from ``master_key`` when the block is absent."""
+    from litellm.proxy.proxy_server import general_settings  # noqa: PLC0415  # circular import at module load
+
+    return resolve_session_signing_keys(master_key, general_settings.get("mcp_session_token_signing"))
+
+
 class NotSessionBearer(BaseModel):
     """The bearer is not session-shaped; admission continues on its normal path."""
 
@@ -116,7 +212,7 @@ def is_session_bearer_shaped(authorization_value: str) -> bool:
 
 def resolve_session_bearer(
     authorization_value: str,
-    keys: SessionKeys,
+    keys: SessionSigningKeys,
     now: datetime,
 ) -> SessionBearerResult:
     """Classify an ``Authorization`` value presented at the aggregate MCP edge.
@@ -166,7 +262,7 @@ SessionRefreshResult: TypeAlias = SessionRefreshOpened | SessionRefreshInvalid
 
 def open_session_refresh_bearer(
     refresh_value: str,
-    keys: SessionKeys,
+    keys: SessionSigningKeys,
     now: datetime,
     expected_client_id: str,
 ) -> SessionRefreshResult:

litellm/proxy/_experimental/mcp_server/outbound_credentials/session_token.py

@@ -8,8 +8,11 @@ is therefore a stable REFERENCE, not an authorization: admission reloads the liv
 record and policy on every request, so deactivating the user (or their team) kills
 outstanding sessions immediately without a revocation store.
 
-Wire shape: ``llm_session_`` (access) / ``llm_srefresh_`` (refresh) + an HS256 JWT,
-the same signing approach as :mod:`.envelope`. Claims are ``iss``/``iat``/``exp``
+Wire shape: ``llm_session_`` (access) / ``llm_srefresh_`` (refresh) + a JWT signed with
+the injected key material: HS256 under the default master-key-derived secret (the same
+signing approach as :mod:`.envelope`), or RS256 under an operator-provided RSA private
+key (:class:`AsymmetricSessionKeys`) so downstream validators hold only the public half.
+Claims are ``iss``/``iat``/``exp``
 plus ``jti`` (per-mint uniqueness, so two tokens minted in the same second never
 collide and a future revocation list has a stable handle), ``kind``, ``user_id``, and
 ``client_id``; ``client_id`` binds the refresh token
@@ -31,11 +34,16 @@ injected ``now``); the strict pydantic claims model is the sole, total type gate
 from __future__ import annotations
 
 import secrets
+from collections import Counter
 from datetime import datetime, timedelta
+from functools import lru_cache
 from typing import Final, Literal, TypeAlias
 
 import jwt
-from pydantic import BaseModel, ConfigDict, Field, SecretStr, ValidationError
+from cryptography.exceptions import UnsupportedAlgorithm
+from cryptography.hazmat.primitives import serialization
+from cryptography.hazmat.primitives.asymmetric import rsa
+from pydantic import BaseModel, ConfigDict, Field, SecretStr, ValidationError, field_validator, model_validator
 
 SESSION_TOKEN_PREFIX: Final = "llm_session_"
 """Marker prefix on every serialized session ACCESS token so the admission edge can cheaply
@@ -71,6 +79,11 @@ limits while bounding hostile input before JWT parsing."""
 
 _SESSION_JWT_ALGORITHM: Final = "HS256"
 
+_SESSION_RSA_ALGORITHM: Final = "RS256"
+
+_MIN_RSA_KEY_BITS: Final = 2048
+"""RFC 7518 section 3.3: RS256 requires a key of at least 2048 bits."""
+
 SessionTokenKind = Literal["session", "session_refresh"]
 """Which credential a session token is. Stamped into the signed claims and required to match
 on open, so a signature-valid token of one kind cannot be replayed as the other even if its
@@ -120,6 +133,85 @@ class SessionKeys(BaseModel):
     signing_key: SecretStr = Field(min_length=32)
 
 
+class SessionRotatedPublicKey(BaseModel):
+    """The public half of a retired signing key, kept verifiable under its ``kid`` during a
+    rotation window so tokens minted before the rotation stay valid until they expire."""
+
+    model_config = ConfigDict(frozen=True)
+    kid: str = Field(min_length=1)
+    public_key_pem: str = Field(min_length=1)
+
+    @field_validator("public_key_pem")
+    @classmethod
+    def _pem_is_an_rsa_public_key(cls, value: str) -> str:
+        try:
+            loaded: Final = serialization.load_pem_public_key(value.encode())
+        except (ValueError, TypeError, UnsupportedAlgorithm) as exc:
+            raise ValueError(f"public_key_pem is not a loadable PEM public key: {exc}") from exc
+        if not isinstance(loaded, rsa.RSAPublicKey):
+            raise ValueError("public_key_pem must be an RSA public key in PEM format")  # noqa: TRY004  # pydantic validators must raise ValueError
+        if loaded.key_size < _MIN_RSA_KEY_BITS:
+            raise ValueError(f"public_key_pem must be an RSA key of at least {_MIN_RSA_KEY_BITS} bits")
+        return value
+
+
+class AsymmetricSessionKeys(BaseModel):
+    """Injected RS256 key material: the issuer-held RSA private key and the stable ``kid``
+    stamped into every minted token's JOSE header, plus the public halves of previously
+    rotated keys that verification still accepts while their tokens age out. Downstream
+    validators never need the private key: :func:`session_public_key_pem` yields the
+    public half to distribute."""
+
+    model_config = ConfigDict(frozen=True)
+    private_key_pem: SecretStr
+    kid: str = Field(min_length=1)
+    previous_public_keys: tuple[SessionRotatedPublicKey, ...] = ()
+
+    @field_validator("private_key_pem")
+    @classmethod
+    def _pem_is_a_strong_rsa_private_key(cls, value: SecretStr) -> SecretStr:
+        try:
+            loaded: Final = serialization.load_pem_private_key(value.get_secret_value().encode(), password=None)
+        except (ValueError, TypeError, UnsupportedAlgorithm) as exc:
+            raise ValueError(f"private_key_pem is not a loadable unencrypted PEM private key: {exc}") from exc
+        if not isinstance(loaded, rsa.RSAPrivateKey):
+            raise ValueError("private_key_pem must be an unencrypted RSA private key in PEM format")  # noqa: TRY004  # pydantic validators must raise ValueError
+        if loaded.key_size < _MIN_RSA_KEY_BITS:
+            raise ValueError(f"private_key_pem must be an RSA key of at least {_MIN_RSA_KEY_BITS} bits")
+        return value
+
+    @model_validator(mode="after")
+    def _kids_are_unique(self) -> AsymmetricSessionKeys:
+        kids: Final = (self.kid, *(previous.kid for previous in self.previous_public_keys))
+        duplicates: Final = tuple(kid for kid, count in Counter(kids).items() if count > 1)
+        if duplicates:
+            raise ValueError(
+                f"every kid must be unique across the current and previous keys; duplicated: {', '.join(duplicates)}"
+            )
+        return self
+
+
+SessionSigningKeys: TypeAlias = SessionKeys | AsymmetricSessionKeys
+"""Every key material shape the mints and openers accept: the default master-key-derived
+HS256 secret, or operator-configured RS256 RSA keys."""
+
+
+@lru_cache(maxsize=8)
+def _public_key_pem_from_private(private_key_pem: str) -> str:
+    loaded: Final = serialization.load_pem_private_key(private_key_pem.encode(), password=None)
+    return (
+        loaded.public_key()
+        .public_bytes(serialization.Encoding.PEM, serialization.PublicFormat.SubjectPublicKeyInfo)
+        .decode()
+    )
+
+
+def session_public_key_pem(keys: AsymmetricSessionKeys) -> str:
+    """The PEM public half of the current RS256 signing key: the only material a downstream
+    validator (an external gateway verifying ``kid``-matched tokens) ever needs."""
+    return _public_key_pem_from_private(keys.private_key_pem.get_secret_value())
+
+
 class MintedSessionToken(BaseModel):
     """A minted session token: the client-held bearer value and when it expires."""
 
@@ -129,12 +221,17 @@ class MintedSessionToken(BaseModel):
 
 
 class OpenedSessionToken(BaseModel):
-    """A validated session token of either kind: the principal it was minted for, plus the
-    ``jti`` so the token endpoint can enforce single-use rotation on a refresh token."""
+    """A validated session token of either kind: the principal it was minted for, the
+    ``jti`` so the token endpoint can enforce single-use rotation on a refresh token, and
+    the signed ``kind``/``iat``/``exp`` so an introspection response can report the
+    token's metadata without re-decoding."""
 
     model_config = ConfigDict(frozen=True)
     principal: SessionPrincipal
     jti: str
+    kind: SessionTokenKind
+    iat: int
+    exp: int
 
 
 class SessionTokenTooLarge(BaseModel):
@@ -221,7 +318,7 @@ def is_session_refresh_token(candidate: str) -> bool:
 
 def mint_session_token(
     principal: SessionPrincipal,
-    keys: SessionKeys,
+    keys: SessionSigningKeys,
     now: datetime,
 ) -> MintedSessionToken | SessionTokenMintError:
     """Mint the short-lived session ACCESS token for ``principal``.
@@ -241,7 +338,7 @@ def mint_session_token(
 
 def mint_session_refresh_token(
     principal: SessionPrincipal,
-    keys: SessionKeys,
+    keys: SessionSigningKeys,
     now: datetime,
 ) -> MintedSessionToken | SessionTokenMintError:
     """Mint the long-lived session REFRESH token for ``principal``.
@@ -262,7 +359,7 @@ def mint_session_refresh_token(
 
 def open_session_token(
     candidate: str,
-    keys: SessionKeys,
+    keys: SessionSigningKeys,
     now: datetime,
 ) -> OpenedSessionToken | SessionTokenOpenError:
     """Validate a session ACCESS ``candidate`` and recover the principal.
@@ -275,7 +372,7 @@ def open_session_token(
 
 def open_session_refresh_token(
     candidate: str,
-    keys: SessionKeys,
+    keys: SessionSigningKeys,
     now: datetime,
 ) -> OpenedSessionToken | SessionTokenOpenError:
     """Validate a session REFRESH ``candidate`` and recover the principal.
@@ -292,7 +389,7 @@ def _mint(
     prefix: str,
     principal: SessionPrincipal,
     expires_at: datetime,
-    keys: SessionKeys,
+    keys: SessionSigningKeys,
     now: datetime,
 ) -> MintedSessionToken | SessionTokenTooLarge:
     """Sign the claims for either token kind and enforce the size cap. Shared by both mints
@@ -309,20 +406,33 @@ def _mint(
         audience=principal.audience,
         team_id=principal.team_id,
     )
-    token: Final = prefix + jwt.encode(
-        claims.model_dump(exclude_none=True), keys.signing_key.get_secret_value(), algorithm=_SESSION_JWT_ALGORITHM
-    )
+    token: Final = prefix + _sign_claims(claims, keys)
     size_bytes: Final = len(token.encode("utf-8"))
     if size_bytes > MAX_SESSION_TOKEN_BYTES:
         return SessionTokenTooLarge(size_bytes=size_bytes, max_bytes=MAX_SESSION_TOKEN_BYTES)
     return MintedSessionToken(token=SecretStr(token), expires_at=expires_at)
 
 
+def _sign_claims(claims: _SessionClaims, keys: SessionSigningKeys) -> str:
+    """Sign the claim set under whichever key material was injected: RS256 with the ``kid``
+    in the JOSE header (so a validator can pick the right public key), or the default
+    HS256 secret with no header extras (byte-compatible with every pre-RS256 token)."""
+    payload: Final = claims.model_dump(exclude_none=True)
+    if isinstance(keys, AsymmetricSessionKeys):
+        return jwt.encode(
+            payload,
+            keys.private_key_pem.get_secret_value(),
+            algorithm=_SESSION_RSA_ALGORITHM,
+            headers={"kid": keys.kid},
+        )
+    return jwt.encode(payload, keys.signing_key.get_secret_value(), algorithm=_SESSION_JWT_ALGORITHM)
+
+
 def _open(
     candidate: str,
     prefix: str,
     expected_kind: SessionTokenKind,
-    keys: SessionKeys,
+    keys: SessionSigningKeys,
     now: datetime,
 ) -> OpenedSessionToken | SessionTokenOpenError:
     """Prefix-route, size-bound, signature-verify, kind-check, and expiry-check an
@@ -337,7 +447,7 @@ def _open(
         return SessionMalformed()
     if len(candidate.encode("utf-8", "surrogatepass")) > MAX_SESSION_TOKEN_BYTES:
         return SessionMalformed()
-    claims: Final = _decode_claims(candidate.removeprefix(prefix), keys.signing_key)
+    claims: Final = _decode_claims(candidate.removeprefix(prefix), keys)
     if not isinstance(claims, _SessionClaims):
         return claims
     if claims.kind != expected_kind:
@@ -353,17 +463,57 @@ def _open(
             team_id=claims.team_id,
         ),
         jti=claims.jti,
+        kind=claims.kind,
+        iat=claims.iat,
+        exp=claims.exp,
     )
 
 
+class _VerificationMaterial(BaseModel):
+    model_config = ConfigDict(frozen=True)
+    key: SecretStr
+    algorithm: Literal["HS256", "RS256"]
+
+
+def _verification_material(
+    compact: str,
+    keys: SessionSigningKeys,
+) -> _VerificationMaterial | SessionBadSignature | SessionMalformed:
+    """Pick the single key and algorithm the candidate is allowed to verify under.
+
+    HS256 mode has exactly one secret. RS256 mode routes by the JOSE header ``kid``: the
+    current key's derived public half, or a retired key's stored public half during a
+    rotation window. An unknown or missing ``kid`` is ``SessionBadSignature`` (a foreign
+    key), and an undecodable header is ``SessionMalformed``. The algorithm is pinned per
+    key shape, never read from the header, so an HS256 token can never be verified
+    against a public key or vice versa.
+    """
+    if isinstance(keys, SessionKeys):
+        return _VerificationMaterial(key=keys.signing_key, algorithm=_SESSION_JWT_ALGORITHM)
+    try:
+        header: Final = jwt.get_unverified_header(compact)
+    except jwt.InvalidTokenError:
+        return SessionMalformed()
+    kid: Final = header.get("kid")
+    if kid == keys.kid:
+        return _VerificationMaterial(key=SecretStr(session_public_key_pem(keys)), algorithm=_SESSION_RSA_ALGORITHM)
+    for previous in keys.previous_public_keys:
+        if previous.kid == kid:
+            return _VerificationMaterial(key=SecretStr(previous.public_key_pem), algorithm=_SESSION_RSA_ALGORITHM)
+    return SessionBadSignature()
+
+
 def _decode_claims(
     compact: str,
-    signing_key: SecretStr,
+    keys: SessionSigningKeys,
 ) -> _SessionClaims | SessionBadSignature | SessionMalformed:
-    """Verify the HS256 signature and shape of an attacker-controlled compact JWT.
+    """Verify the signature and shape of an attacker-controlled compact JWT.
 
     ``compact`` is fully hostile and bounded to ``MAX_SESSION_TOKEN_BYTES`` by the caller.
-    PyJWT's ``iat``/``nbf``/``exp`` validators are disabled: they raise on hostile claim
+    The accepted algorithm is pinned by :func:`_verification_material` from the injected
+    key shape, so ``alg`` confusion (``none``, or HS256 signed with a public key as the
+    secret) fails before or at signature verification. PyJWT's ``iat``/``nbf``/``exp``
+    validators are disabled: they raise on hostile claim
     types and, for ``iat``/``nbf``, compare against the wall clock rather than the injected
     ``now`` (``exp`` is checked by the caller against ``now``). Apart from a signature
     mismatch, every decode failure is ``SessionMalformed``: a non-UTF-8 candidate surfaces
@@ -371,11 +521,14 @@ def _decode_claims(
     ``TypeError`` from PyJWT's claim validators, and a wrong issuer or structurally invalid
     token as an ``InvalidTokenError``. ``_SessionClaims`` is the total type gate.
     """
+    material: Final = _verification_material(compact, keys)
+    if not isinstance(material, _VerificationMaterial):
+        return material
     try:
         payload: Final = jwt.decode(
             compact,
-            signing_key.get_secret_value(),
-            algorithms=[_SESSION_JWT_ALGORITHM],
+            material.key.get_secret_value(),
+            algorithms=[material.algorithm],
             issuer=SESSION_ISSUER,
             options={
                 "verify_exp": False,

litellm/proxy/_experimental/out/__next.!KGRhc2hib2FyZCk.__PAGE__.txt

@@ -1,9 +1,9 @@
 1:"$Sreact.fragment"
-2:I[347257,["/litellm-asset-prefix/_next/static/chunks/3_06chgeyldml.js","/litellm-asset-prefix/_next/static/chunks/26h-ny89yaww0.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js"],"ClientPageRoot"]
-3:I[871135,["/litellm-asset-prefix/_next/static/chunks/3_06chgeyldml.js","/litellm-asset-prefix/_next/static/chunks/26h-ny89yaww0.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js","/litellm-asset-prefix/_next/static/chunks/37t2cfzl_b58p.js","/litellm-asset-prefix/_next/static/chunks/1zf358k334atp.js","/litellm-asset-prefix/_next/static/chunks/3kil-7y33kpm9.js","/litellm-asset-prefix/_next/static/chunks/2i0218zvrsasm.js","/litellm-asset-prefix/_next/static/chunks/3ytz29phknzsy.js","/litellm-asset-prefix/_next/static/chunks/0aoel7yrv88fp.js","/litellm-asset-prefix/_next/static/chunks/1natmx9lu3mus.js","/litellm-asset-prefix/_next/static/chunks/1dmg55q8kht9j.js","/litellm-asset-prefix/_next/static/chunks/1lrl_8p0h2sbm.js","/litellm-asset-prefix/_next/static/chunks/1e-4-g6x6zyse.js","/litellm-asset-prefix/_next/static/chunks/0gygfcpmiijl8.js","/litellm-asset-prefix/_next/static/chunks/2cu4j3g1tldv4.js","/litellm-asset-prefix/_next/static/chunks/2kmqjpt047tjo.js","/litellm-asset-prefix/_next/static/chunks/1phty1k2nx8fx.js","/litellm-asset-prefix/_next/static/chunks/0u3cfuz-tf0wj.js","/litellm-asset-prefix/_next/static/chunks/39s4-rh6l9sa1.js","/litellm-asset-prefix/_next/static/chunks/0dylouuq8ak8p.js","/litellm-asset-prefix/_next/static/chunks/3a3jpg95umjho.js","/litellm-asset-prefix/_next/static/chunks/28hnu_qv5e_c_.js","/litellm-asset-prefix/_next/static/chunks/0qf1_0kt4uuxa.js","/litellm-asset-prefix/_next/static/chunks/0ab_ntohf1wik.js","/litellm-asset-prefix/_next/static/chunks/3kpec-qy1uzod.js"],"default"]
-6:I[897367,["/litellm-asset-prefix/_next/static/chunks/3_06chgeyldml.js","/litellm-asset-prefix/_next/static/chunks/26h-ny89yaww0.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js"],"OutletBoundary"]
+2:I[347257,["/litellm-asset-prefix/_next/static/chunks/3s39b43k2vde7.js","/litellm-asset-prefix/_next/static/chunks/3kest3gurc9op.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js"],"ClientPageRoot"]
+3:I[871135,["/litellm-asset-prefix/_next/static/chunks/3s39b43k2vde7.js","/litellm-asset-prefix/_next/static/chunks/3kest3gurc9op.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js","/litellm-asset-prefix/_next/static/chunks/1a3mamulxkyhw.js","/litellm-asset-prefix/_next/static/chunks/2yqxc2yxa1-go.js","/litellm-asset-prefix/_next/static/chunks/27gjrlkmq245y.js","/litellm-asset-prefix/_next/static/chunks/0aoel7yrv88fp.js","/litellm-asset-prefix/_next/static/chunks/1nfnjvxf_0-3n.js","/litellm-asset-prefix/_next/static/chunks/2i0218zvrsasm.js","/litellm-asset-prefix/_next/static/chunks/34_wtpkkvqa3n.js","/litellm-asset-prefix/_next/static/chunks/3ytz29phknzsy.js","/litellm-asset-prefix/_next/static/chunks/1l61r88q65pjd.js","/litellm-asset-prefix/_next/static/chunks/0di-9qm-8ex8r.js","/litellm-asset-prefix/_next/static/chunks/3dy-3uqjux30s.js","/litellm-asset-prefix/_next/static/chunks/3wdy9040h4b13.js","/litellm-asset-prefix/_next/static/chunks/13rzpi4q1z_e8.js","/litellm-asset-prefix/_next/static/chunks/0wh5uu7sl34-i.js","/litellm-asset-prefix/_next/static/chunks/0cotqb-2hzyvs.js","/litellm-asset-prefix/_next/static/chunks/16xdxq7qvv37h.js","/litellm-asset-prefix/_next/static/chunks/0dylouuq8ak8p.js","/litellm-asset-prefix/_next/static/chunks/2tj1x2xl0npv1.js","/litellm-asset-prefix/_next/static/chunks/2eonl4rcemkdj.js","/litellm-asset-prefix/_next/static/chunks/0z7zg9587od6_.js","/litellm-asset-prefix/_next/static/chunks/0ab_ntohf1wik.js","/litellm-asset-prefix/_next/static/chunks/1v3m908ycsmt4.js"],"default"]
+6:I[897367,["/litellm-asset-prefix/_next/static/chunks/3s39b43k2vde7.js","/litellm-asset-prefix/_next/static/chunks/3kest3gurc9op.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js"],"OutletBoundary"]
 7:"$Sreact.suspense"
-0:{"rsc":["$","$1","c",{"children":[["$","$L2",null,{"Component":"$3","serverProvidedParams":{"searchParams":{},"params":{},"promises":["$@4","$@5"]}}],[["$","script","script-0",{"src":"/litellm-asset-prefix/_next/static/chunks/1e-4-g6x6zyse.js","async":true}],["$","script","script-1",{"src":"/litellm-asset-prefix/_next/static/chunks/0gygfcpmiijl8.js","async":true}],["$","script","script-2",{"src":"/litellm-asset-prefix/_next/static/chunks/2cu4j3g1tldv4.js","async":true}],["$","script","script-3",{"src":"/litellm-asset-prefix/_next/static/chunks/2kmqjpt047tjo.js","async":true}],["$","script","script-4",{"src":"/litellm-asset-prefix/_next/static/chunks/1phty1k2nx8fx.js","async":true}],["$","script","script-5",{"src":"/litellm-asset-prefix/_next/static/chunks/0u3cfuz-tf0wj.js","async":true}],["$","script","script-6",{"src":"/litellm-asset-prefix/_next/static/chunks/39s4-rh6l9sa1.js","async":true}],["$","script","script-7",{"src":"/litellm-asset-prefix/_next/static/chunks/0dylouuq8ak8p.js","async":true}],["$","script","script-8",{"src":"/litellm-asset-prefix/_next/static/chunks/3a3jpg95umjho.js","async":true}],["$","script","script-9",{"src":"/litellm-asset-prefix/_next/static/chunks/28hnu_qv5e_c_.js","async":true}],["$","script","script-10",{"src":"/litellm-asset-prefix/_next/static/chunks/0qf1_0kt4uuxa.js","async":true}],["$","script","script-11",{"src":"/litellm-asset-prefix/_next/static/chunks/0ab_ntohf1wik.js","async":true}],["$","script","script-12",{"src":"/litellm-asset-prefix/_next/static/chunks/3kpec-qy1uzod.js","async":true}]],["$","$L6",null,{"children":["$","$7",null,{"name":"Next.MetadataOutlet","children":"$@8"}]}]]}],"isPartial":false,"staleTime":300,"varyParams":null,"buildId":"0cE25rDXvGu3tj4HWOGKy"}
+0:{"rsc":["$","$1","c",{"children":[["$","$L2",null,{"Component":"$3","serverProvidedParams":{"searchParams":{},"params":{},"promises":["$@4","$@5"]}}],[["$","script","script-0",{"src":"/litellm-asset-prefix/_next/static/chunks/0di-9qm-8ex8r.js","async":true}],["$","script","script-1",{"src":"/litellm-asset-prefix/_next/static/chunks/3dy-3uqjux30s.js","async":true}],["$","script","script-2",{"src":"/litellm-asset-prefix/_next/static/chunks/3wdy9040h4b13.js","async":true}],["$","script","script-3",{"src":"/litellm-asset-prefix/_next/static/chunks/13rzpi4q1z_e8.js","async":true}],["$","script","script-4",{"src":"/litellm-asset-prefix/_next/static/chunks/0wh5uu7sl34-i.js","async":true}],["$","script","script-5",{"src":"/litellm-asset-prefix/_next/static/chunks/0cotqb-2hzyvs.js","async":true}],["$","script","script-6",{"src":"/litellm-asset-prefix/_next/static/chunks/16xdxq7qvv37h.js","async":true}],["$","script","script-7",{"src":"/litellm-asset-prefix/_next/static/chunks/0dylouuq8ak8p.js","async":true}],["$","script","script-8",{"src":"/litellm-asset-prefix/_next/static/chunks/2tj1x2xl0npv1.js","async":true}],["$","script","script-9",{"src":"/litellm-asset-prefix/_next/static/chunks/2eonl4rcemkdj.js","async":true}],["$","script","script-10",{"src":"/litellm-asset-prefix/_next/static/chunks/0z7zg9587od6_.js","async":true}],["$","script","script-11",{"src":"/litellm-asset-prefix/_next/static/chunks/0ab_ntohf1wik.js","async":true}],["$","script","script-12",{"src":"/litellm-asset-prefix/_next/static/chunks/1v3m908ycsmt4.js","async":true}]],["$","$L6",null,{"children":["$","$7",null,{"name":"Next.MetadataOutlet","children":"$@8"}]}]]}],"isPartial":false,"staleTime":300,"varyParams":null,"buildId":"YAsRgSxdV-OcBfib_67Dt"}
 4:{}
 5:"$0:rsc:props:children:0:props:serverProvidedParams:params"
 8:null

litellm/proxy/_experimental/out/__next.!KGRhc2hib2FyZCk.txt

@@ -1,7 +1,7 @@
 1:"$Sreact.fragment"
-2:I[92825,["/litellm-asset-prefix/_next/static/chunks/3_06chgeyldml.js","/litellm-asset-prefix/_next/static/chunks/26h-ny89yaww0.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js"],"ClientSegmentRoot"]
-3:I[216370,["/litellm-asset-prefix/_next/static/chunks/3_06chgeyldml.js","/litellm-asset-prefix/_next/static/chunks/26h-ny89yaww0.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js","/litellm-asset-prefix/_next/static/chunks/37t2cfzl_b58p.js","/litellm-asset-prefix/_next/static/chunks/1zf358k334atp.js","/litellm-asset-prefix/_next/static/chunks/3kil-7y33kpm9.js","/litellm-asset-prefix/_next/static/chunks/2i0218zvrsasm.js","/litellm-asset-prefix/_next/static/chunks/3ytz29phknzsy.js","/litellm-asset-prefix/_next/static/chunks/0aoel7yrv88fp.js","/litellm-asset-prefix/_next/static/chunks/1natmx9lu3mus.js","/litellm-asset-prefix/_next/static/chunks/1dmg55q8kht9j.js","/litellm-asset-prefix/_next/static/chunks/1lrl_8p0h2sbm.js"],"default"]
-4:I[339756,["/litellm-asset-prefix/_next/static/chunks/3_06chgeyldml.js","/litellm-asset-prefix/_next/static/chunks/26h-ny89yaww0.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js"],"default"]
-5:I[837457,["/litellm-asset-prefix/_next/static/chunks/3_06chgeyldml.js","/litellm-asset-prefix/_next/static/chunks/26h-ny89yaww0.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js"],"default"]
-0:{"rsc":["$","$1","c",{"children":[[["$","script","script-0",{"src":"/litellm-asset-prefix/_next/static/chunks/37t2cfzl_b58p.js","async":true}],["$","script","script-1",{"src":"/litellm-asset-prefix/_next/static/chunks/1zf358k334atp.js","async":true}],["$","script","script-2",{"src":"/litellm-asset-prefix/_next/static/chunks/3kil-7y33kpm9.js","async":true}],["$","script","script-3",{"src":"/litellm-asset-prefix/_next/static/chunks/2i0218zvrsasm.js","async":true}],["$","script","script-4",{"src":"/litellm-asset-prefix/_next/static/chunks/3ytz29phknzsy.js","async":true}],["$","script","script-5",{"src":"/litellm-asset-prefix/_next/static/chunks/0aoel7yrv88fp.js","async":true}],["$","script","script-6",{"src":"/litellm-asset-prefix/_next/static/chunks/1natmx9lu3mus.js","async":true}],["$","script","script-7",{"src":"/litellm-asset-prefix/_next/static/chunks/1dmg55q8kht9j.js","async":true}],["$","script","script-8",{"src":"/litellm-asset-prefix/_next/static/chunks/1lrl_8p0h2sbm.js","async":true}]],["$","$L2",null,{"Component":"$3","slots":{"children":["$","$L4",null,{"parallelRouterKey":"children","template":["$","$L5",null,{}],"notFound":[[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":404}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],[]]}]},"serverProvidedParams":{"params":{},"promises":["$@6"]}}]]}],"isPartial":false,"staleTime":300,"varyParams":null,"buildId":"0cE25rDXvGu3tj4HWOGKy"}
+2:I[92825,["/litellm-asset-prefix/_next/static/chunks/3s39b43k2vde7.js","/litellm-asset-prefix/_next/static/chunks/3kest3gurc9op.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js"],"ClientSegmentRoot"]
+3:I[216370,["/litellm-asset-prefix/_next/static/chunks/3s39b43k2vde7.js","/litellm-asset-prefix/_next/static/chunks/3kest3gurc9op.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js","/litellm-asset-prefix/_next/static/chunks/1a3mamulxkyhw.js","/litellm-asset-prefix/_next/static/chunks/2yqxc2yxa1-go.js","/litellm-asset-prefix/_next/static/chunks/27gjrlkmq245y.js","/litellm-asset-prefix/_next/static/chunks/0aoel7yrv88fp.js","/litellm-asset-prefix/_next/static/chunks/1nfnjvxf_0-3n.js","/litellm-asset-prefix/_next/static/chunks/2i0218zvrsasm.js","/litellm-asset-prefix/_next/static/chunks/34_wtpkkvqa3n.js","/litellm-asset-prefix/_next/static/chunks/3ytz29phknzsy.js","/litellm-asset-prefix/_next/static/chunks/1l61r88q65pjd.js"],"default"]
+4:I[339756,["/litellm-asset-prefix/_next/static/chunks/3s39b43k2vde7.js","/litellm-asset-prefix/_next/static/chunks/3kest3gurc9op.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js"],"default"]
+5:I[837457,["/litellm-asset-prefix/_next/static/chunks/3s39b43k2vde7.js","/litellm-asset-prefix/_next/static/chunks/3kest3gurc9op.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js"],"default"]
+0:{"rsc":["$","$1","c",{"children":[[["$","script","script-0",{"src":"/litellm-asset-prefix/_next/static/chunks/1a3mamulxkyhw.js","async":true}],["$","script","script-1",{"src":"/litellm-asset-prefix/_next/static/chunks/2yqxc2yxa1-go.js","async":true}],["$","script","script-2",{"src":"/litellm-asset-prefix/_next/static/chunks/27gjrlkmq245y.js","async":true}],["$","script","script-3",{"src":"/litellm-asset-prefix/_next/static/chunks/0aoel7yrv88fp.js","async":true}],["$","script","script-4",{"src":"/litellm-asset-prefix/_next/static/chunks/1nfnjvxf_0-3n.js","async":true}],["$","script","script-5",{"src":"/litellm-asset-prefix/_next/static/chunks/2i0218zvrsasm.js","async":true}],["$","script","script-6",{"src":"/litellm-asset-prefix/_next/static/chunks/34_wtpkkvqa3n.js","async":true}],["$","script","script-7",{"src":"/litellm-asset-prefix/_next/static/chunks/3ytz29phknzsy.js","async":true}],["$","script","script-8",{"src":"/litellm-asset-prefix/_next/static/chunks/1l61r88q65pjd.js","async":true}]],["$","$L2",null,{"Component":"$3","slots":{"children":["$","$L4",null,{"parallelRouterKey":"children","template":["$","$L5",null,{}],"notFound":[[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":404}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],[]]}]},"serverProvidedParams":{"params":{},"promises":["$@6"]}}]]}],"isPartial":false,"staleTime":300,"varyParams":null,"buildId":"YAsRgSxdV-OcBfib_67Dt"}
 6:"$0:rsc:props:children:1:props:serverProvidedParams:params"

litellm/proxy/_experimental/out/__next._head.txt

@@ -1,6 +1,6 @@
 1:"$Sreact.fragment"
-2:I[897367,["/litellm-asset-prefix/_next/static/chunks/3_06chgeyldml.js","/litellm-asset-prefix/_next/static/chunks/26h-ny89yaww0.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js"],"ViewportBoundary"]
-3:I[897367,["/litellm-asset-prefix/_next/static/chunks/3_06chgeyldml.js","/litellm-asset-prefix/_next/static/chunks/26h-ny89yaww0.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js"],"MetadataBoundary"]
+2:I[897367,["/litellm-asset-prefix/_next/static/chunks/3s39b43k2vde7.js","/litellm-asset-prefix/_next/static/chunks/3kest3gurc9op.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js"],"ViewportBoundary"]
+3:I[897367,["/litellm-asset-prefix/_next/static/chunks/3s39b43k2vde7.js","/litellm-asset-prefix/_next/static/chunks/3kest3gurc9op.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js"],"MetadataBoundary"]
 4:"$Sreact.suspense"
-5:I[27201,["/litellm-asset-prefix/_next/static/chunks/3_06chgeyldml.js","/litellm-asset-prefix/_next/static/chunks/26h-ny89yaww0.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js"],"IconMark"]
-0:{"rsc":["$","$1","h",{"children":[null,["$","$L2",null,{"children":[["$","meta","0",{"charSet":"utf-8"}],["$","meta","1",{"name":"viewport","content":"width=device-width, initial-scale=1"}]]}],["$","div",null,{"hidden":true,"children":["$","$L3",null,{"children":["$","$4",null,{"name":"Next.Metadata","children":[["$","title","0",{"children":"LiteLLM Dashboard"}],["$","meta","1",{"name":"description","content":"LiteLLM Proxy Admin UI"}],["$","link","2",{"rel":"icon","href":"/favicon.ico?favicon.3arlap5n8tyzg.ico","sizes":"48x48","type":"image/x-icon"}],["$","link","3",{"rel":"icon","href":"/get_favicon"}],["$","$L5","4",{}]]}]}]}],["$","meta",null,{"name":"next-size-adjust","content":""}]]}],"isPartial":false,"staleTime":300,"varyParams":null,"buildId":"0cE25rDXvGu3tj4HWOGKy"}
+5:I[27201,["/litellm-asset-prefix/_next/static/chunks/3s39b43k2vde7.js","/litellm-asset-prefix/_next/static/chunks/3kest3gurc9op.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js"],"IconMark"]
+0:{"rsc":["$","$1","h",{"children":[null,["$","$L2",null,{"children":[["$","meta","0",{"charSet":"utf-8"}],["$","meta","1",{"name":"viewport","content":"width=device-width, initial-scale=1"}]]}],["$","div",null,{"hidden":true,"children":["$","$L3",null,{"children":["$","$4",null,{"name":"Next.Metadata","children":[["$","title","0",{"children":"LiteLLM Dashboard"}],["$","meta","1",{"name":"description","content":"LiteLLM Proxy Admin UI"}],["$","link","2",{"rel":"icon","href":"/favicon.ico?favicon.3arlap5n8tyzg.ico","sizes":"48x48","type":"image/x-icon"}],["$","link","3",{"rel":"icon","href":"/get_favicon"}],["$","$L5","4",{}]]}]}]}],["$","meta",null,{"name":"next-size-adjust","content":""}]]}],"isPartial":false,"staleTime":300,"varyParams":null,"buildId":"YAsRgSxdV-OcBfib_67Dt"}

litellm/proxy/_experimental/out/__next._index.txt

@@ -1,11 +1,11 @@
 1:"$Sreact.fragment"
-2:I[363178,["/litellm-asset-prefix/_next/static/chunks/3_06chgeyldml.js","/litellm-asset-prefix/_next/static/chunks/26h-ny89yaww0.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js"],"ThemeProvider"]
-3:I[12985,["/litellm-asset-prefix/_next/static/chunks/3_06chgeyldml.js","/litellm-asset-prefix/_next/static/chunks/26h-ny89yaww0.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js"],"NuqsAdapter"]
-4:I[867271,["/litellm-asset-prefix/_next/static/chunks/3_06chgeyldml.js","/litellm-asset-prefix/_next/static/chunks/26h-ny89yaww0.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js"],"default"]
-5:I[557951,["/litellm-asset-prefix/_next/static/chunks/3_06chgeyldml.js","/litellm-asset-prefix/_next/static/chunks/26h-ny89yaww0.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js"],"AuthProvider"]
-6:I[339756,["/litellm-asset-prefix/_next/static/chunks/3_06chgeyldml.js","/litellm-asset-prefix/_next/static/chunks/26h-ny89yaww0.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js"],"default"]
-7:I[837457,["/litellm-asset-prefix/_next/static/chunks/3_06chgeyldml.js","/litellm-asset-prefix/_next/static/chunks/26h-ny89yaww0.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js"],"default"]
-8:I[713354,["/litellm-asset-prefix/_next/static/chunks/3_06chgeyldml.js","/litellm-asset-prefix/_next/static/chunks/26h-ny89yaww0.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js"],"Toaster"]
+2:I[363178,["/litellm-asset-prefix/_next/static/chunks/3s39b43k2vde7.js","/litellm-asset-prefix/_next/static/chunks/3kest3gurc9op.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js"],"ThemeProvider"]
+3:I[12985,["/litellm-asset-prefix/_next/static/chunks/3s39b43k2vde7.js","/litellm-asset-prefix/_next/static/chunks/3kest3gurc9op.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js"],"NuqsAdapter"]
+4:I[867271,["/litellm-asset-prefix/_next/static/chunks/3s39b43k2vde7.js","/litellm-asset-prefix/_next/static/chunks/3kest3gurc9op.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js"],"default"]
+5:I[557951,["/litellm-asset-prefix/_next/static/chunks/3s39b43k2vde7.js","/litellm-asset-prefix/_next/static/chunks/3kest3gurc9op.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js"],"AuthProvider"]
+6:I[339756,["/litellm-asset-prefix/_next/static/chunks/3s39b43k2vde7.js","/litellm-asset-prefix/_next/static/chunks/3kest3gurc9op.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js"],"default"]
+7:I[837457,["/litellm-asset-prefix/_next/static/chunks/3s39b43k2vde7.js","/litellm-asset-prefix/_next/static/chunks/3kest3gurc9op.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js"],"default"]
+8:I[713354,["/litellm-asset-prefix/_next/static/chunks/3s39b43k2vde7.js","/litellm-asset-prefix/_next/static/chunks/3kest3gurc9op.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js"],"Toaster"]
 :HL["/litellm-asset-prefix/_next/static/chunks/1kid9zr1--h6y.css","style"]
-:HL["/litellm-asset-prefix/_next/static/chunks/2hicgq-mjp8vy.css","style"]
-0:{"rsc":["$","$1","c",{"children":[[["$","link","0",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/1kid9zr1--h6y.css","precedence":"next"}],["$","link","1",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/2hicgq-mjp8vy.css","precedence":"next"}],["$","script","script-0",{"src":"/litellm-asset-prefix/_next/static/chunks/3_06chgeyldml.js","async":true}],["$","script","script-1",{"src":"/litellm-asset-prefix/_next/static/chunks/26h-ny89yaww0.js","async":true}],["$","script","script-2",{"src":"/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js","async":true}]],["$","html",null,{"lang":"en","suppressHydrationWarning":true,"children":["$","body",null,{"className":"inter_5972bc34-module__OU16Qa__className","children":["$","$L2",null,{"attribute":"class","defaultTheme":"light","enableSystem":true,"disableTransitionOnChange":true,"children":["$","$L3",null,{"children":["$","$L4",null,{"children":[["$","$L5",null,{"children":["$","$L6",null,{"parallelRouterKey":"children","template":["$","$L7",null,{}],"notFound":[[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":404}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],[]]}]}],["$","$L8",null,{}]]}]}]}]}]}]]}],"isPartial":false,"staleTime":300,"varyParams":null,"buildId":"0cE25rDXvGu3tj4HWOGKy"}
+:HL["/litellm-asset-prefix/_next/static/chunks/3idmblk6vi8i5.css","style"]
+0:{"rsc":["$","$1","c",{"children":[[["$","link","0",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/1kid9zr1--h6y.css","precedence":"next"}],["$","link","1",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/3idmblk6vi8i5.css","precedence":"next"}],["$","script","script-0",{"src":"/litellm-asset-prefix/_next/static/chunks/3s39b43k2vde7.js","async":true}],["$","script","script-1",{"src":"/litellm-asset-prefix/_next/static/chunks/3kest3gurc9op.js","async":true}],["$","script","script-2",{"src":"/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js","async":true}]],["$","html",null,{"lang":"en","suppressHydrationWarning":true,"children":["$","body",null,{"className":"inter_5972bc34-module__OU16Qa__className","children":["$","$L2",null,{"attribute":"class","defaultTheme":"light","enableSystem":true,"disableTransitionOnChange":true,"children":["$","$L3",null,{"children":["$","$L4",null,{"children":[["$","$L5",null,{"children":["$","$L6",null,{"parallelRouterKey":"children","template":["$","$L7",null,{}],"notFound":[[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":404}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],[]]}]}],["$","$L8",null,{}]]}]}]}]}]}]]}],"isPartial":false,"staleTime":300,"varyParams":null,"buildId":"YAsRgSxdV-OcBfib_67Dt"}

litellm/proxy/_experimental/out/__next._tree.txt

@@ -1,4 +1,4 @@
 :HL["/litellm-asset-prefix/_next/static/chunks/1kid9zr1--h6y.css","style"]
-:HL["/litellm-asset-prefix/_next/static/chunks/2hicgq-mjp8vy.css","style"]
+:HL["/litellm-asset-prefix/_next/static/chunks/3idmblk6vi8i5.css","style"]
 :HL["/litellm-asset-prefix/_next/static/media/83afe278b6a6bb3c-s.p.2bn3s6zvc0dyp.woff2","font",{"crossOrigin":"","type":"font/woff2"}]
-0:{"tree":{"name":"","param":null,"prefetchHints":16,"slots":{"children":{"name":"(dashboard)","param":null,"prefetchHints":0,"slots":{"children":{"name":"__PAGE__","param":null,"prefetchHints":0,"slots":null}}}}},"staleTime":300,"buildId":"0cE25rDXvGu3tj4HWOGKy"}
+0:{"tree":{"name":"","param":null,"prefetchHints":16,"slots":{"children":{"name":"(dashboard)","param":null,"prefetchHints":0,"slots":{"children":{"name":"__PAGE__","param":null,"prefetchHints":0,"slots":null}}}}},"staleTime":300,"buildId":"YAsRgSxdV-OcBfib_67Dt"}