Merge branch 'litellm_internal_staging' into litellm_fix_chat_anyof_tool_schema

.github/workflows/codspeed.yml

@@ -12,6 +12,7 @@ on:
       - "uv.lock"
       - ".github/workflows/codspeed.yml"
       - ".github/actions/setup-uv-with-retries/**"
+      - ".github/actions/cache-cargo-build/**"
   pull_request:
     branches:
       - main
@@ -23,6 +24,7 @@ on:
       - "uv.lock"
       - ".github/workflows/codspeed.yml"
       - ".github/actions/setup-uv-with-retries/**"
+      - ".github/actions/cache-cargo-build/**"
   # Allow CodSpeed to trigger backtest performance analysis
   # in order to generate initial data
   workflow_dispatch:
@@ -55,6 +57,26 @@ jobs:
         with:
           version: "0.10.9"
 
+      - name: Cache the Rust build
+        uses: ./.github/actions/cache-cargo-build
+
+      # Build the wheel and resolve every dependency outside the CodSpeed
+      # runner: the same maturin build took 42 minutes inside `codspeed run`
+      # versus under 3 minutes as a plain step (LIT-6183)
+      - name: Build environment
+        run: >
+          env PYTEST_DISABLE_PLUGIN_AUTOLOAD=1
+          uv run --frozen --no-default-groups
+          --with pytest==8.3.5
+          --with pytest-codspeed==4.3.0
+          --with "mcp>=1.26.0,<2.0"
+          --with "a2a-sdk>=1.1.0,<2.0"
+          pytest
+          -p pytest_codspeed.plugin
+          tests/benchmarks/
+          --codspeed
+          --collect-only -q
+
       - name: Run benchmarks
         uses: CodSpeedHQ/action@1c8ae4843586d3ba879736b7f6b7b0c990757fab # v4.12.1
         with:

basedpyright-code-budget.json

@@ -1,9 +1,9 @@
 {
   "reportAny": {
-    "limit": 17271
+    "limit": 17270
   },
   "reportArgumentType": {
-    "limit": 2539
+    "limit": 2538
   },
   "reportAssignmentType": {
     "limit": 319
@@ -24,7 +24,7 @@
     "limit": 19
   },
   "reportExplicitAny": {
-    "limit": 5486
+    "limit": 5485
   },
   "reportFunctionMemberAccess": {
     "limit": 7

codecov.yaml

@@ -25,6 +25,8 @@ flag_management:
       carryforward: false
     - name: proxy-db-schema-migration
       carryforward: false
+    - name: circleci
+      carryforward: false
 
 component_management:
   individual_components:

enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py

@@ -2,9 +2,10 @@
 Polls LiteLLM_ManagedObjectTable to check if the batch job is complete, and if the cost has been tracked.
 """
 
+from dataclasses import replace as dataclasses_replace
 from datetime import datetime, timedelta, timezone
 from types import MappingProxyType
-from typing import TYPE_CHECKING, Any, Dict, Final, List, Optional, Tuple
+from typing import TYPE_CHECKING, Any, Dict, Final, List, Literal, Optional, Tuple, cast
 
 from litellm._logging import verbose_proxy_logger
 from litellm._uuid import uuid
@@ -626,6 +627,7 @@ class CheckBatchCost:
         later poll.
         """
         from litellm.batches.batch_utils import (
+            count_error_file_failed_requests,
             _get_file_content_as_dictionary,
             calculate_batch_cost_and_usage,
         )
@@ -761,16 +763,33 @@ class CheckBatchCost:
             model_id=model_id,
             deployment_model=litellm_model_name,
         )
-        batch_cost, batch_usage, batch_models = (
-            await calculate_batch_cost_and_usage(
-                file_content_dictionary=file_content_as_dict,
-                custom_llm_provider=llm_provider,  # type: ignore
-                model_name=model_name,
-                model_info=deployment_model_info,
+        batch_file_provider: Final = cast(
+            Literal["openai", "azure", "vertex_ai", "hosted_vllm", "anthropic"], llm_provider
+        )
+        output_file_result: Final = await calculate_batch_cost_and_usage(
+            file_content_dictionary=file_content_as_dict,
+            custom_llm_provider=batch_file_provider,
+            model_name=model_name,
+            model_info=deployment_model_info,
+        )
+        error_file_failed_requests: Final = await count_error_file_failed_requests(
+            response,
+            custom_llm_provider=batch_file_provider,
+            litellm_params={
+                **credentials,
+                "_litellm_internal_model_credentials": MappingProxyType(dict(credentials)),
+            },
+        )
+        batch_result: Final = (
+            output_file_result
+            if not error_file_failed_requests
+            else dataclasses_replace(
+                output_file_result,
+                failed_requests=output_file_result.failed_requests + error_file_failed_requests,
             )
         )
         logging_obj = LiteLLMLogging(
-            model=batch_models[0],
+            model=batch_result.models[0],
             messages=[{"role": "user", "content": "<retrieve_batch>"}],
             stream=False,
             call_type="aretrieve_batch",
@@ -802,9 +821,11 @@ class CheckBatchCost:
         try:
             await logging_obj.async_success_handler(
                 result=response,
-                batch_cost=batch_cost,
-                batch_usage=batch_usage,
-                batch_models=batch_models,
+                batch_cost=batch_result.cost,
+                batch_usage=batch_result.usage,
+                batch_models=batch_result.models,
+                batch_successful_requests=batch_result.successful_requests,
+                batch_failed_requests=batch_result.failed_requests,
             )
         except Exception:
             await self._release_job_claim(job)

enterprise/litellm_enterprise/proxy/hooks/managed_files.py

@@ -36,6 +36,7 @@ from litellm.llms.base_llm.managed_resources.isolation import (
     build_list_page,
     build_owner_filter,
     can_access_resource,
+    resolve_resource_owner_id,
 )
 from litellm.proxy._types import (
     CallTypes,
@@ -222,7 +223,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
                 file_object=file_object,
                 model_mappings=model_mappings,
                 flat_model_file_ids=list(model_mappings.values()),
-                created_by=user_api_key_dict.user_id,
+                created_by=resolve_resource_owner_id(user_api_key_dict),
                 team_id=user_api_key_dict.team_id,
                 updated_by=user_api_key_dict.user_id,
             )
@@ -238,7 +239,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
             "unified_file_id": file_id,
             "model_mappings": json.dumps(model_mappings),
             "flat_model_file_ids": list(model_mappings.values()),
-            "created_by": user_api_key_dict.user_id,
+            "created_by": resolve_resource_owner_id(user_api_key_dict),
             "team_id": user_api_key_dict.team_id,
             "updated_by": user_api_key_dict.user_id,
         }
@@ -342,7 +343,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
                     "file_object": file_object.model_dump_json(),
                     "model_object_id": model_object_id,
                     "file_purpose": file_purpose,
-                    "created_by": user_api_key_dict.user_id,
+                    "created_by": resolve_resource_owner_id(user_api_key_dict),
                     "team_id": user_api_key_dict.team_id,
                     "updated_by": user_api_key_dict.user_id,
                     "status": file_object.status,

enterprise/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "litellm-enterprise"
-version = "0.1.61"
+version = "0.1.62"
 description = "Package for LiteLLM Enterprise features"
 readme = "README.md"
 requires-python = ">=3.9"
@@ -26,7 +26,7 @@ required-version = ">=0.10.9"
 module-root = ""
 
 [tool.commitizen]
-version = "0.1.61"
+version = "0.1.62"
 version_files = [
     "pyproject.toml:^version",
     "../pyproject.toml:litellm-enterprise==",

litellm-proxy-extras/litellm_proxy_extras/migrations/20260804162853_add_budget_window_spend_table/migration.sql

@@ -0,0 +1,12 @@
+CREATE TABLE IF NOT EXISTS "LiteLLM_BudgetWindowSpend" (
+    "entity_type" TEXT NOT NULL,
+    "entity_id" TEXT NOT NULL,
+    "window_duration" TEXT NOT NULL,
+    "window_start" TIMESTAMP(3) NOT NULL,
+    "spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0,
+    "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updated_at" TIMESTAMP(3) NOT NULL,
+
+    CONSTRAINT "LiteLLM_BudgetWindowSpend_pkey" PRIMARY KEY ("entity_type","entity_id","window_duration")
+);
+

litellm-proxy-extras/litellm_proxy_extras/migrations/20260829000000_add_model_access_group_budget_table/migration.sql

@@ -0,0 +1,20 @@
+-- CreateTable
+CREATE TABLE IF NOT EXISTS "LiteLLM_ModelAccessGroupBudgetTable" (
+    "access_group_name" TEXT NOT NULL,
+    "spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0,
+    "budget_id" TEXT,
+    "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "created_by" TEXT,
+    "updated_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updated_by" TEXT,
+
+    CONSTRAINT "LiteLLM_ModelAccessGroupBudgetTable_pkey" PRIMARY KEY ("access_group_name")
+);
+
+-- AddForeignKey
+DO $$
+BEGIN
+    IF NOT EXISTS (SELECT 1 FROM pg_constraint WHERE conname = 'LiteLLM_ModelAccessGroupBudgetTable_budget_id_fkey') THEN
+        ALTER TABLE "LiteLLM_ModelAccessGroupBudgetTable" ADD CONSTRAINT "LiteLLM_ModelAccessGroupBudgetTable_budget_id_fkey" FOREIGN KEY ("budget_id") REFERENCES "LiteLLM_BudgetTable"("budget_id") ON DELETE SET NULL ON UPDATE CASCADE;
+    END IF;
+END $$;

litellm-proxy-extras/litellm_proxy_extras/schema.prisma

@@ -29,6 +29,7 @@ model LiteLLM_BudgetTable {
   keys LiteLLM_VerificationToken[] // multiple keys can have the same budget
   end_users LiteLLM_EndUserTable[] // multiple end-users can have the same budget
   tags LiteLLM_TagTable[] // multiple tags can have the same budget
+  model_access_groups LiteLLM_ModelAccessGroupBudgetTable[] // multiple model access groups can have the same budget
   team_membership LiteLLM_TeamMembership[] // budgets of Users within a Team 
   organization_membership LiteLLM_OrganizationMembership[] // budgets of Users within a Organization
 }
@@ -585,6 +586,20 @@ model LiteLLM_EndUserTable {
   blocked Boolean @default(false)
 }
 
+// Budget and shared spend for a model access group. The groups themselves are not rows anywhere:
+// they are free-text strings in LiteLLM_ProxyModelTable.model_info.access_groups, so a row here
+// exists only once someone gives that group a budget.
+model LiteLLM_ModelAccessGroupBudgetTable {
+  access_group_name String @id
+  spend Float @default(0.0)
+  budget_id String?
+  litellm_budget_table LiteLLM_BudgetTable? @relation(fields: [budget_id], references: [budget_id])
+  created_at DateTime @default(now()) @map("created_at")
+  created_by String?
+  updated_at DateTime @default(now()) @updatedAt
+  updated_by String?
+}
+
 // Track tags with budgets and spend
 model LiteLLM_TagTable {
   tag_name String @id
@@ -649,6 +664,18 @@ model LiteLLM_SpendLogs {
   @@index([session_id])
 }
 
+model LiteLLM_BudgetWindowSpend {
+  entity_type     String
+  entity_id       String
+  window_duration String
+  window_start    DateTime
+  spend           Float    @default(0.0)
+  created_at      DateTime @default(now())
+  updated_at      DateTime @updatedAt
+
+  @@id([entity_type, entity_id, window_duration])
+}
+
 // View spend, model, api_key per request
 model LiteLLM_ErrorLogs {
   request_id          String   @id @default(uuid())

litellm-proxy-extras/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "litellm-proxy-extras"
-version = "0.4.90"
+version = "0.4.91"
 description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package."
 readme = "README.md"
 requires-python = ">=3.9"
@@ -26,7 +26,7 @@ required-version = ">=0.10.9"
 module-root = ""
 
 [tool.commitizen]
-version = "0.4.90"
+version = "0.4.91"
 version_files = [
     "pyproject.toml:^version",
     "../pyproject.toml:litellm-proxy-extras==",

litellm/batches/batch_utils.py

@@ -1,6 +1,8 @@
 import json
 from collections.abc import Iterable, Iterator, Mapping
 from dataclasses import dataclass
+from dataclasses import replace as dataclasses_replace
+from enum import Enum
 from typing import Any, Final, Literal
 
 import litellm
@@ -12,12 +14,23 @@ from litellm.types.utils import CallTypes, ModelInfo, Usage
 from litellm.utils import token_counter
 
 
+@dataclass(frozen=True, slots=True)
+class BatchCostUsageResult:
+    """Aggregate cost, usage, and per-line pass/fail counts for a completed batch."""
+
+    cost: float
+    usage: Usage
+    models: list[str]
+    successful_requests: int
+    failed_requests: int
+
+
 async def calculate_batch_cost_and_usage(
     file_content_dictionary: list[dict],
     custom_llm_provider: Literal["openai", "azure", "vertex_ai", "hosted_vllm", "anthropic"],
     model_name: str | None = None,
     model_info: ModelInfo | None = None,
-) -> tuple[float, Usage, list[str]]:
+) -> BatchCostUsageResult:
     """
     Calculate the cost and usage of a batch.
 
@@ -32,8 +45,7 @@ async def calculate_batch_cost_and_usage(
         and model_name
         and getattr(litellm, "disable_vertex_batch_output_transformation", False)
     ):
-        batch_cost, batch_usage = calculate_vertex_ai_batch_cost_and_usage(file_content_dictionary, model_name)
-        return batch_cost, batch_usage, [model_name]
+        return calculate_vertex_ai_batch_cost_and_usage(file_content_dictionary, model_name)
 
     return _aggregate_batch_cost_usage_models(
         entries=file_content_dictionary,
@@ -49,7 +61,7 @@ async def _handle_completed_batch(
     model_name: str | None = None,
     litellm_params: dict | None = None,
     model_info: ModelInfo | None = None,
-) -> tuple[float, Usage, list[str]]:
+) -> BatchCostUsageResult:
     """Fetch a completed batch's output file and aggregate its cost, usage, and
     models in a single pass over the JSONL lines, so the parsed file content is
     never materialized in memory.
@@ -72,27 +84,49 @@ async def _handle_completed_batch(
     # The generic retrieval helper keeps raising for callers that explicitly ask
     # for a missing output file.
     if batch.output_file_id is None:
-        return 0.0, Usage(prompt_tokens=0, completion_tokens=0, total_tokens=0), []
+        return BatchCostUsageResult(
+            cost=0.0,
+            usage=Usage(prompt_tokens=0, completion_tokens=0, total_tokens=0),
+            models=[],  # mutable-ok: no output file means no model was ever priced; BatchCostUsageResult.models requires list[str]
+            successful_requests=0,
+            failed_requests=await count_error_file_failed_requests(
+                batch, custom_llm_provider=custom_llm_provider, litellm_params=litellm_params
+            ),
+        )
 
     file_content = await _fetch_batch_output_file_content(batch, custom_llm_provider, litellm_params=litellm_params)
-
-    if (
-        custom_llm_provider == "vertex_ai"
-        and model_name
-        and getattr(litellm, "disable_vertex_batch_output_transformation", False)
-    ):
-        batch_cost, batch_usage = calculate_vertex_ai_batch_cost_and_usage(
-            _get_file_content_as_dictionary(file_content), model_name
-        )
-        return batch_cost, batch_usage, [model_name]
-
-    return _aggregate_batch_cost_usage_models(
-        entries=_iter_batch_output_entries(file_content),
-        custom_llm_provider=custom_llm_provider,
-        model_name=model_name,
-        model_info=model_info,
+    error_file_failed_requests: Final = await count_error_file_failed_requests(
+        batch, custom_llm_provider=custom_llm_provider, litellm_params=litellm_params
     )
 
+    output_file_result: Final = (
+        calculate_vertex_ai_batch_cost_and_usage(_get_file_content_as_dictionary(file_content), model_name)
+        if (
+            custom_llm_provider == "vertex_ai"
+            and model_name
+            and getattr(litellm, "disable_vertex_batch_output_transformation", False)
+        )
+        else _aggregate_batch_cost_usage_models(
+            entries=_iter_batch_output_entries(file_content),
+            custom_llm_provider=custom_llm_provider,
+            model_name=model_name,
+            model_info=model_info,
+        )
+    )
+
+    if not error_file_failed_requests:
+        return output_file_result
+    return dataclasses_replace(
+        output_file_result, failed_requests=output_file_result.failed_requests + error_file_failed_requests
+    )
+
+
+class _LineOutcome(Enum):
+    """A batch output line that yielded no billable stats."""
+
+    PROVIDER_FAILED = "provider_failed"
+    UNCOSTABLE = "uncostable"
+
 
 @dataclass(frozen=True, slots=True)
 class _BatchOutputLineStats:
@@ -102,19 +136,27 @@ class _BatchOutputLineStats:
     total_tokens: int
     cache_read_tokens: int
     cache_creation_tokens: int
+    reasoning_tokens: int
     model: str | None
 
 
-def _iter_successful_output_line_stats(
+def _classify_output_line_stats(
     entries: Iterable[dict],
     custom_llm_provider: Literal["openai", "azure", "vertex_ai", "hosted_vllm", "anthropic", "bedrock"],
     model_name: str | None,
     model_info: ModelInfo | None,
-) -> Iterator[_BatchOutputLineStats]:
+) -> Iterator[_BatchOutputLineStats | _LineOutcome]:
+    """Classify every output line in a single pass, so counting failures never needs
+    a second read of a potentially huge output file. A line the provider reported as
+    failed yields ``PROVIDER_FAILED``; a successful line litellm could not price
+    yields ``UNCOSTABLE`` and still counts as a successful request billed at $0, so
+    the counts stay reconcilable with the provider's own ``request_counts``."""
     for entry in entries:
+        if not _batch_response_was_successful(entry, custom_llm_provider):
+            yield _LineOutcome.PROVIDER_FAILED
+            continue
         stats = _safe_output_line_stats(entry, custom_llm_provider, model_name, model_info)
-        if stats is not None:
-            yield stats
+        yield stats if stats is not None else _LineOutcome.UNCOSTABLE
 
 
 def _safe_output_line_stats(
@@ -123,13 +165,11 @@ def _safe_output_line_stats(
     model_name: str | None,
     model_info: ModelInfo | None,
 ) -> _BatchOutputLineStats | None:
-    """Return the stats for one batch output line, or None for a line that is
-    unsuccessful or cannot be costed, so a single bad line never aborts the
-    whole batch's cost accounting."""
+    """Return the stats for one provider-successful batch output line, or None when
+    it cannot be costed, so a single bad line never aborts the whole batch's cost
+    accounting."""
     custom_id: Final = entry.get("custom_id") if isinstance(entry, dict) else None
     try:
-        if not _batch_response_was_successful(entry, custom_llm_provider):
-            return None
         return _compute_output_line_stats(entry, custom_llm_provider, model_name, model_info)
     except Exception as e:  # noqa: BLE001  # any single line's costing failure must not abort the whole batch
         verbose_logger.warning(
@@ -152,6 +192,7 @@ def _compute_output_line_stats(
     prompt_details: Final = parse_prompt_tokens_details(usage)
     raw_model: Final = response_body.get("model")
     response_model: Final = raw_model if isinstance(raw_model, str) and raw_model else None
+    completion_details: Final = usage.completion_tokens_details
     return _BatchOutputLineStats(
         cost=_output_line_cost(
             response_body=response_body,
@@ -166,6 +207,7 @@ def _compute_output_line_stats(
         total_tokens=usage.total_tokens,
         cache_read_tokens=prompt_details["cache_hit_tokens"],
         cache_creation_tokens=prompt_details["cache_creation_tokens"],
+        reasoning_tokens=(completion_details.reasoning_tokens if completion_details else None) or 0,
         model=response_model,
     )
 
@@ -203,10 +245,14 @@ def _aggregate_batch_cost_usage_models(
     custom_llm_provider: Literal["openai", "azure", "vertex_ai", "hosted_vllm", "anthropic", "bedrock"],
     model_name: str | None = None,
     model_info: ModelInfo | None = None,
-) -> tuple[float, Usage, list[str]]:
-    """Aggregate cost, usage, and models from batch output entries in a single
-    pass, holding one small stats record per line instead of the parsed file."""
-    line_stats: Final = tuple(_iter_successful_output_line_stats(entries, custom_llm_provider, model_name, model_info))
+) -> BatchCostUsageResult:
+    """Aggregate cost, usage, models, and pass/fail counts from batch output
+    entries in a single pass, holding one small stats record per line instead
+    of the parsed file."""
+    all_results: Final = tuple(_classify_output_line_stats(entries, custom_llm_provider, model_name, model_info))
+    line_stats: Final = tuple(result for result in all_results if isinstance(result, _BatchOutputLineStats))
+    failed_requests: Final = sum(1 for result in all_results if result is _LineOutcome.PROVIDER_FAILED)
+    successful_requests: Final = len(all_results) - failed_requests
 
     cache_token_params: Final = {
         key: tokens
@@ -220,18 +266,32 @@ def _aggregate_batch_cost_usage_models(
         total_tokens=sum(stats.total_tokens for stats in line_stats),
         prompt_tokens=sum(stats.prompt_tokens for stats in line_stats),
         completion_tokens=sum(stats.completion_tokens for stats in line_stats),
+        reasoning_tokens=sum(stats.reasoning_tokens for stats in line_stats),
         **cache_token_params,
     )
     batch_models: Final = [model_name] if model_name else [stats.model for stats in line_stats if stats.model]
     total_cost: Final = sum((stats.cost for stats in line_stats), 0.0)
-    verbose_logger.debug("batch output aggregate: cost=%s usage=%s models=%s", total_cost, batch_usage, batch_models)
-    return total_cost, batch_usage, batch_models
+    verbose_logger.debug(
+        "batch output aggregate: cost=%s usage=%s models=%s successful=%d failed=%d",
+        total_cost,
+        batch_usage,
+        batch_models,
+        successful_requests,
+        failed_requests,
+    )
+    return BatchCostUsageResult(
+        cost=total_cost,
+        usage=batch_usage,
+        models=batch_models,
+        successful_requests=successful_requests,
+        failed_requests=failed_requests,
+    )
 
 
 def calculate_vertex_ai_batch_cost_and_usage(
     vertex_ai_batch_responses: list[dict],
     model_name: str | None = None,
-) -> tuple[float, Usage]:
+) -> BatchCostUsageResult:
     """
     Calculate both cost and usage from raw Vertex AI batch responses.
 
@@ -242,6 +302,10 @@ def calculate_vertex_ai_batch_cost_and_usage(
       {"request": ..., "response": {"candidates": [...], "usageMetadata": {...}}}
 
     usageMetadata contains promptTokenCount, candidatesTokenCount, totalTokenCount.
+
+    A row with no ``response`` is counted as failed - the same signal already
+    used to skip it from cost/usage aggregation, since Vertex batch prediction
+    output doesn't establish a distinct error shape in this (non-default) path.
     """
     from litellm.cost_calculator import batch_cost_calculator
 
@@ -249,12 +313,16 @@ def calculate_vertex_ai_batch_cost_and_usage(
     total_tokens = 0
     prompt_tokens = 0
     completion_tokens = 0
+    successful_requests = 0  # rebind-ok: loop accumulator, matches total_cost/total_tokens above
+    failed_requests = 0  # rebind-ok: loop accumulator, matches total_cost/total_tokens above
     actual_model_name: Final = model_name or "gemini-2.0-flash-001"
 
     for response in vertex_ai_batch_responses:
         response_body = response.get("response")
         if response_body is None:
+            failed_requests += 1
             continue
+        successful_requests += 1
 
         usage_metadata = response_body.get("usageMetadata", {})
         _prompt = usage_metadata.get("promptTokenCount", 0) or 0
@@ -282,17 +350,25 @@ def calculate_vertex_ai_batch_cost_and_usage(
         total_tokens += _total
 
     verbose_logger.info(
-        "vertex_ai batch cost: cost=%s, prompt=%d, completion=%d, total=%d",
+        "vertex_ai batch cost: cost=%s, prompt=%d, completion=%d, total=%d, successful=%d, failed=%d",
         total_cost,
         prompt_tokens,
         completion_tokens,
         total_tokens,
+        successful_requests,
+        failed_requests,
     )
 
-    return total_cost, Usage(
-        total_tokens=total_tokens,
-        prompt_tokens=prompt_tokens,
-        completion_tokens=completion_tokens,
+    return BatchCostUsageResult(
+        cost=total_cost,
+        usage=Usage(
+            total_tokens=total_tokens,
+            prompt_tokens=prompt_tokens,
+            completion_tokens=completion_tokens,
+        ),
+        models=[actual_model_name],
+        successful_requests=successful_requests,
+        failed_requests=failed_requests,
     )
 
 
@@ -322,6 +398,36 @@ def _provider_output_file_id(output_file_id: str) -> str:
     return extracted
 
 
+async def _fetch_batch_managed_file_content(
+    file_id: str,
+    custom_llm_provider: Literal["openai", "azure", "vertex_ai", "hosted_vllm", "anthropic"] = "openai",
+    litellm_params: dict | None = None,
+) -> bytes:
+    """
+    Fetch a batch's output or error file and return its raw JSONL bytes.
+
+    Args:
+        file_id: The provider or unified (litellm-managed) file id to fetch
+        custom_llm_provider: The LLM provider
+        litellm_params: Optional litellm parameters containing credentials (api_key, api_base, etc.)
+                       Required for Azure and other providers that need authentication
+    """
+    from litellm.files.main import afile_content
+
+    # Build kwargs for afile_content with credentials from litellm_params
+    file_content_kwargs: Final = {
+        "file_id": _provider_output_file_id(file_id),
+        "custom_llm_provider": custom_llm_provider,
+    }
+
+    # Extract and add credentials for file access
+    credentials: Final = _extract_file_access_credentials(litellm_params)
+    file_content_kwargs.update(credentials)
+
+    _file_content: Final = await afile_content(**file_content_kwargs)
+    return _file_content.content
+
+
 async def _fetch_batch_output_file_content(
     batch: Batch,
     custom_llm_provider: Literal["openai", "azure", "vertex_ai", "hosted_vllm", "anthropic"] = "openai",
@@ -336,25 +442,36 @@ async def _fetch_batch_output_file_content(
         litellm_params: Optional litellm parameters containing credentials (api_key, api_base, etc.)
                        Required for Azure and other providers that need authentication
     """
-    from litellm.files.main import afile_content
-
     if batch.output_file_id is None:
         raise ValueError("Output file id is None cannot retrieve file content")
 
-    file_id: Final = _provider_output_file_id(batch.output_file_id)
+    return await _fetch_batch_managed_file_content(
+        batch.output_file_id, custom_llm_provider=custom_llm_provider, litellm_params=litellm_params
+    )
 
-    # Build kwargs for afile_content with credentials from litellm_params
-    file_content_kwargs: Final = {
-        "file_id": file_id,
-        "custom_llm_provider": custom_llm_provider,
-    }
 
-    # Extract and add credentials for file access
-    credentials: Final = _extract_file_access_credentials(litellm_params)
-    file_content_kwargs.update(credentials)
+async def count_error_file_failed_requests(
+    batch: Batch,
+    custom_llm_provider: Literal["openai", "azure", "vertex_ai", "hosted_vllm", "anthropic"],
+    litellm_params: dict | None,
+) -> int:
+    """Count failed requests reported only in the batch's separate error file.
 
-    _file_content: Final = await afile_content(**file_content_kwargs)
-    return _file_content.content
+    OpenAI-shaped batch providers write successful lines to ``output_file_id``
+    and per-request failures (e.g. a rejected param) to a distinct
+    ``error_file_id`` - they never appear in the output file at all, so
+    counting failures from the output file alone silently undercounts them.
+    """
+    if batch.error_file_id is None:
+        return 0
+    try:
+        error_file_content = await _fetch_batch_managed_file_content(
+            batch.error_file_id, custom_llm_provider=custom_llm_provider, litellm_params=litellm_params
+        )
+    except Exception as e:  # noqa: BLE001  # a failed/missing error file must not abort cost tracking for the batch
+        verbose_logger.debug("Failed to fetch batch error file %s: %s", batch.error_file_id, e)
+        return 0
+    return sum(1 for _ in _iter_batch_input_lines(error_file_content))
 
 
 def _extract_file_access_credentials(litellm_params: dict | None) -> dict:

litellm/constants.py

@@ -35,6 +35,7 @@ DEFAULT_COOLDOWN_TIME_SECONDS: Final = int(os.getenv("DEFAULT_COOLDOWN_TIME_SECO
 DEFAULT_REPLICATE_POLLING_RETRIES: Final = int(os.getenv("DEFAULT_REPLICATE_POLLING_RETRIES", 5))
 DEFAULT_REPLICATE_POLLING_DELAY_SECONDS: Final = int(os.getenv("DEFAULT_REPLICATE_POLLING_DELAY_SECONDS", 1))
 DEFAULT_IMAGE_TOKEN_COUNT: Final = int(os.getenv("DEFAULT_IMAGE_TOKEN_COUNT", 250))
+HF_CONFIG_FETCH_TIMEOUT_SECONDS: Final = 10.0
 
 # Maximum wall-clock seconds a streaming response is allowed to run.
 # Streams exceeding this duration are terminated with a Timeout error.
@@ -288,6 +289,7 @@ REDIS_DAILY_ORG_SPEND_UPDATE_BUFFER_KEY: Final = "litellm_daily_org_spend_update
 REDIS_DAILY_END_USER_SPEND_UPDATE_BUFFER_KEY: Final = "litellm_daily_end_user_spend_update_buffer"
 REDIS_DAILY_AGENT_SPEND_UPDATE_BUFFER_KEY: Final = "litellm_daily_agent_spend_update_buffer"
 REDIS_DAILY_TAG_SPEND_UPDATE_BUFFER_KEY: Final = "litellm_daily_tag_spend_update_buffer"
+REDIS_WINDOW_SPEND_UPDATE_BUFFER_KEY: Final = "litellm_window_spend_update_buffer"
 MAX_REDIS_BUFFER_DEQUEUE_COUNT: Final = int(os.getenv("MAX_REDIS_BUFFER_DEQUEUE_COUNT", 100))
 # Bounds asyncio.Queue() instances (log queues, spend update queues, etc.) to prevent unbounded memory growth
 LITELLM_ASYNCIO_QUEUE_MAXSIZE: Final = int(os.getenv("LITELLM_ASYNCIO_QUEUE_MAXSIZE", 1000))
@@ -1681,6 +1683,7 @@ DEFAULT_MCP_NAMESPACE_CSV_MAX_TOKENS: Final = 16
 # Ceilings on the cached auth registries; larger tables fall back to per-row lookups
 # instead of holding an unbounded id set in every worker.
 TAG_REGISTRY_MAX_SIZE: Final = 5000
+MODEL_ACCESS_GROUP_REGISTRY_MAX_SIZE: Final = 5000
 END_USER_RESTRICTED_REGISTRY_MAX_SIZE: Final = 5000
 # How long a failed registry load is remembered as "unusable", so a degraded Postgres
 # is not re-scanned on every request on top of the per-id lookups it falls back to.

litellm/litellm_core_utils/audio_utils/subtitle_utils.py

@@ -1,19 +1,36 @@
 """Provider-agnostic SRT/WebVTT subtitle synthesis from timestamped transcription tokens."""
 
+import unicodedata
 from collections.abc import Sequence
 from dataclasses import dataclass
-from itertools import accumulate, chain
+from itertools import accumulate, groupby
 from typing import Final
 
 from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError
 
-CUE_MAX_TOKENS: Final = 15
-CUE_MAX_DURATION_MS: Final = 5000
+CUE_MAX_CHARS: Final = 84
+CUE_MAX_DURATION_MS: Final = 7000
+CUE_GAP_MS: Final = 700
 
 SRT_RESPONSE_FORMAT: Final = "srt"
 VTT_RESPONSE_FORMAT: Final = "vtt"
 SUBTITLE_RESPONSE_FORMATS: Final = frozenset((SRT_RESPONSE_FORMAT, VTT_RESPONSE_FORMAT))
 
+_SENTENCE_END_CHARS: Final = (".", "!", "?", "。", "！", "？", "؟", "۔", "।", "॥", "։", "።")
+
+_CJK_RANGES: Final = (
+    (0x3400, 0x4DBF),
+    (0x4E00, 0x9FFF),
+    (0xF900, 0xFAFF),
+    (0x3040, 0x309F),
+    (0x30A0, 0x30FF),
+    (0x31F0, 0x31FF),
+)
+
+_CJK_NO_BREAK_BEFORE: Final = "、。，．！？：；・ー…」』）〉》】〕"
+
+_CJK_NO_BREAK_AFTER: Final = "「『（〈《【〔"
+
 
 @dataclass(frozen=True, slots=True)
 class SubtitleToken:
@@ -31,69 +48,138 @@ class SubtitleCue:
 
 
 @dataclass(frozen=True, slots=True)
-class _CueAccumulator:
-    texts: tuple[str, ...] = ()
-    start_ms: int | None = None
-    end_ms: int | None = None
-    speaker: str | int | None = None
+class _Word:
+    text: str
+    start_ms: int | None
+    end_ms: int | None
+    speaker: str | int | None
 
 
-def _completed_cue(accumulator: _CueAccumulator) -> tuple[SubtitleCue, ...]:
-    if not accumulator.texts or accumulator.start_ms is None:
-        return ()
-    text: Final = "".join(accumulator.texts).strip()
-    if not text:
-        return ()
-    end_ms: Final = accumulator.end_ms if accumulator.end_ms is not None else accumulator.start_ms
-    return (SubtitleCue(start_ms=accumulator.start_ms, end_ms=end_ms, text=text),)
+def _is_cjk(ch: str) -> bool:
+    cp: Final = ord(ch)
+    return any(lo <= cp <= hi for lo, hi in _CJK_RANGES)
 
 
-def _cue_break_reached(accumulator: _CueAccumulator, token: SubtitleToken) -> bool:
-    if len(accumulator.texts) >= CUE_MAX_TOKENS:
-        return True
+def _is_cjk_word_boundary(prev_ch: str, next_ch: str) -> bool:
+    if not (_is_cjk(prev_ch) or _is_cjk(next_ch)):
+        return False
+    return next_ch not in _CJK_NO_BREAK_BEFORE and prev_ch not in _CJK_NO_BREAK_AFTER
+
+
+def _text_width(text: str) -> int:
+    return sum(2 if unicodedata.east_asian_width(ch) in ("W", "F") else 1 for ch in text)
+
+
+def _starts_new_word(prev: SubtitleToken, token: SubtitleToken) -> bool:
+    prev_last: Final = prev.text[-1:]
+    first: Final = token.text[0]
     return (
-        accumulator.start_ms is not None
-        and token.start_ms is not None
-        and token.start_ms - accumulator.start_ms >= CUE_MAX_DURATION_MS
+        first.isspace()
+        or prev_last.isspace()
+        or token.speaker != prev.speaker
+        or _is_cjk_word_boundary(prev_last, first)
     )
 
 
-_AbsorbStep = tuple[tuple[SubtitleCue, ...], _CueAccumulator]
-
-
-def _absorb_token(accumulator: _CueAccumulator, token: SubtitleToken) -> _AbsorbStep:
-    if token.start_ms is None and accumulator.start_ms is None:
-        return (), accumulator
-    if token.speaker is not None and token.speaker != accumulator.speaker:
-        return _completed_cue(accumulator), _CueAccumulator(
-            texts=(token.text,),
-            start_ms=token.start_ms,
-            end_ms=token.end_ms,
-            speaker=token.speaker,
-        )
-    if _cue_break_reached(accumulator, token):
-        return _completed_cue(accumulator), _CueAccumulator(
-            texts=(token.text,),
-            start_ms=token.start_ms,
-            end_ms=token.end_ms,
-            speaker=accumulator.speaker,
-        )
-    return (), _CueAccumulator(
-        texts=(*accumulator.texts, token.text),
-        start_ms=accumulator.start_ms if accumulator.start_ms is not None else token.start_ms,
-        end_ms=token.end_ms if token.end_ms is not None else accumulator.end_ms,
-        speaker=accumulator.speaker,
+def _build_word(group: Sequence[SubtitleToken]) -> _Word:
+    return _Word(
+        text="".join(t.text for t in group),
+        start_ms=next((t.start_ms for t in group if t.start_ms is not None), None),
+        end_ms=next((t.end_ms for t in reversed(group) if t.end_ms is not None), None),
+        speaker=group[0].speaker,
     )
 
 
-def _absorb_step(carry: _AbsorbStep, token: SubtitleToken) -> _AbsorbStep:
-    return _absorb_token(carry[1], token)
+def _merge_tokens_into_words(tokens: Sequence[SubtitleToken]) -> tuple[_Word, ...]:
+    """
+    Merge subword tokens (e.g. ``"Hel"``, ``"lo"``) into whole words.
+
+    A token starts a new word when its text begins with whitespace, when the
+    previous token's text ends with whitespace, when the speaker changes, or
+    at a CJK character boundary (CJK scripts carry no spaces, so without this
+    an entire utterance would fuse into a single unbreakable "word"; CJK
+    punctuation stays attached to the preceding character per kinsoku rules).
+    Each word carries the first/last available timestamps of its tokens.
+    """
+    kept: Final = tuple(t for t in tokens if t.text != "")
+    starts: Final = tuple(i for i, t in enumerate(kept) if i == 0 or _starts_new_word(kept[i - 1], t))
+    return tuple(_build_word(kept[begin:end]) for begin, end in zip(starts, (*starts[1:], len(kept))))
+
+
+def _cue_start(ws: Sequence[_Word]) -> int | None:
+    return next((w.start_ms for w in ws if w.start_ms is not None), None)
+
+
+def _cue_end(ws: Sequence[_Word]) -> int | None:
+    return next((w.end_ms for w in reversed(ws) if w.end_ms is not None), _cue_start(ws))
+
+
+def _cue_text(ws: Sequence[_Word]) -> str:
+    return "".join(w.text for w in ws).strip()
+
+
+def _should_break(cue: Sequence[_Word], word: _Word) -> bool:
+    speaker_changed: Final = word.speaker is not None and any(
+        w.speaker is not None and w.speaker != word.speaker for w in cue
+    )
+    cue_start: Final = _cue_start(cue)
+    cue_end: Final = _cue_end(cue)
+    gap_exceeded: Final = word.start_ms is not None and cue_end is not None and (word.start_ms - cue_end) >= CUE_GAP_MS
+    chars_exceeded: Final = _text_width(_cue_text(cue)) + _text_width(word.text) > CUE_MAX_CHARS
+    word_end: Final = word.end_ms if word.end_ms is not None else word.start_ms
+    duration_exceeded: Final = (
+        word_end is not None and cue_start is not None and (word_end - cue_start) > CUE_MAX_DURATION_MS
+    )
+    return speaker_changed or gap_exceeded or chars_exceeded or duration_exceeded
+
+
+def _cue_start_indices(words: Sequence[_Word]) -> tuple[int, ...]:
+    def next_start(start: int, index: int) -> int:
+        if words[index - 1].text.rstrip().endswith(_SENTENCE_END_CHARS):
+            return index
+        if _should_break(words[start:index], words[index]):
+            return index
+        return start
+
+    if not words:
+        return ()
+    return tuple(start for start, _ in groupby(accumulate(range(1, len(words)), next_start, initial=0)))
+
+
+def _build_cue(ws: Sequence[_Word]) -> SubtitleCue | None:
+    text: Final = _cue_text(ws)
+    start: Final = _cue_start(ws)
+    if not text or start is None:
+        return None
+    end: Final = _cue_end(ws)
+    return SubtitleCue(start_ms=start, end_ms=end if end is not None else start, text=text)
 
 
 def group_subtitle_tokens_into_cues(tokens: Sequence[SubtitleToken]) -> tuple[SubtitleCue, ...]:
-    steps: Final = tuple(accumulate(tokens, _absorb_step, initial=((), _CueAccumulator())))
-    completed: Final = chain.from_iterable(emitted for emitted, _ in steps)
-    return (*completed, *_completed_cue(steps[-1][1]))
+    """
+    Group transcription tokens into subtitle cues aligned to the actual speech.
+
+    Cues only ever break at word boundaries (tokens may be subwords, so they
+    are first merged into words). A new cue starts when:
+      - the speaker changes (if diarization is on),
+      - a silence gap of at least CUE_GAP_MS separates two words, so
+        subtitles never bridge pauses in speech,
+      - adding the next word would exceed CUE_MAX_CHARS of display width
+        (~two subtitle lines; East-Asian wide characters count double), or
+      - adding the next word would make the cue span more than
+        CUE_MAX_DURATION_MS.
+    A cue also ends after sentence-final punctuation, which keeps cue breaks
+    at natural seams. Cue timestamps come straight from token timestamps;
+    words without timestamps stay attached to the surrounding cue, and a cue
+    whose words carry no timestamps at all is dropped.
+    """
+    words: Final = _merge_tokens_into_words(tokens)
+    starts: Final = _cue_start_indices(words)
+    return tuple(
+        cue
+        for begin, end in zip(starts, (*starts[1:], len(words)))
+        if (cue := _build_cue(words[begin:end])) is not None
+    )
 
 
 def _format_timestamp(total_ms: int, millis_separator: str) -> str:

litellm/litellm_core_utils/chat_completion_agentic_loop.py

@@ -1,6 +1,7 @@
 # this is a patch to allow for agentic loops covering llm_http_handler.py and openai sdk based calling flows for the .completion() api
 
 import json
+from collections.abc import Mapping
 from typing import Final, cast
 
 from litellm._logging import verbose_logger
@@ -9,8 +10,11 @@ from litellm.litellm_core_utils.agentic_loop_settings import (
     DEFAULT_MAX_AGENTIC_LOOPS,
     validated_max_agentic_loops,
 )
+from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObject
+from litellm.llms.base_llm.base_model_iterator import MockResponseIterator
 from litellm.types.integrations.custom_logger import (
     CHAT_COMPLETION_AGENTIC_SURFACE,
+    HEADROOM_CONVERTED_STREAM_KEY,
     NON_CODE_INTERPRETER_INTERCEPTION_INTERNAL_PREFIXES,
     AgenticLoopPlan,
     AgenticLoopRequestPatch,
@@ -50,6 +54,12 @@ def _post_hook_overridden(callback: CustomLogger) -> bool:
     return getattr(func, "__func__", func) is not getattr(base, "__func__", base)
 
 
+def _converted_stream_requested(kwargs: Mapping[str, object]) -> bool:
+    return bool(
+        kwargs.get("_code_interpreter_interception_converted_stream") or kwargs.get(HEADROOM_CONVERTED_STREAM_KEY)
+    )
+
+
 def _coerce_int(value: object, default: int) -> int:
     return int(value) if isinstance(value, (int, str)) else default
 
@@ -87,16 +97,24 @@ def _check_agentic_loop_safety(
     return fingerprint
 
 
-def _wrap_response_as_fake_stream(response: object) -> object:
-    if getattr(response, "object", None) == "chat.completion.chunk":
+def _wrap_response_as_fake_stream(
+    response: object,
+    *,
+    model: str,
+    custom_llm_provider: str,
+    logging_obj: object,
+) -> object:
+    if isinstance(response, CustomStreamWrapper):
         return response
-    if not hasattr(response, "choices"):
+    if not isinstance(response, ModelResponse) or not isinstance(logging_obj, LiteLLMLoggingObject):
         return response
-    from litellm.llms.base_llm.base_model_iterator import (
-        convert_model_response_to_streaming,
-    )
 
-    return convert_model_response_to_streaming(cast(ModelResponse, response))
+    return CustomStreamWrapper(
+        completion_stream=MockResponseIterator(model_response=response),
+        model=model,
+        custom_llm_provider=custom_llm_provider,
+        logging_obj=logging_obj,
+    )
 
 
 def _add_agentic_loop_metadata(kwargs_for_followup: dict[str, object]) -> None:
@@ -177,8 +195,13 @@ async def _execute_chat_completion_agentic_plan(
                     model,
                     str(e),
                 )
-        if kwargs.get("_code_interpreter_interception_converted_stream") and not depth:
-            return _wrap_response_as_fake_stream(response_followup)
+        if _converted_stream_requested(kwargs) and not depth:
+            return _wrap_response_as_fake_stream(
+                response_followup,
+                model=model,
+                custom_llm_provider=custom_llm_provider,
+                logging_obj=logging_obj,
+            )
         return response_followup
     finally:
         try:
@@ -302,9 +325,14 @@ async def maybe_run_chat_completion_agentic_loop(
                 str(e),
             )
 
-    if kwargs.get("_code_interpreter_interception_converted_stream") and not depth and hasattr(response, "choices"):
+    if _converted_stream_requested(kwargs) and not depth:
         return cast(
             "ModelResponse | CustomStreamWrapper",
-            _wrap_response_as_fake_stream(response),
+            _wrap_response_as_fake_stream(
+                response,
+                model=model,
+                custom_llm_provider=custom_llm_provider,
+                logging_obj=logging_obj,
+            ),
         )
     return None

litellm/litellm_core_utils/internal_call_metadata.py

@@ -25,6 +25,13 @@ from litellm.types.utils import BACKGROUND_RESPONSE_COST_POLL_CALL_ORIGIN, Inter
 
 BUDGET_RESERVATION_METADATA_KEYS: Final = frozenset({"user_api_key_budget_reservation"})
 
+MODEL_ACCESS_GROUP_METADATA_KEY: Final = "user_api_key_matched_model_access_groups"
+"""Where auth records the model access groups that authorized the request, for the spend writer.
+
+The ``user_api_key`` prefix is load-bearing, not cosmetic: when a request carries both
+``metadata`` and ``litellm_metadata``, ``get_litellm_metadata_from_kwargs`` returns the latter and
+copies a key across only when ``user_api_key`` appears in its name."""
+
 _USER_API_KEY_AUTH_KEY: Final = "user_api_key_auth"
 
 FORWARDABLE_IDENTITY_METADATA_KEYS: Final = frozenset(

litellm/litellm_core_utils/litellm_logging.py

@@ -64,7 +64,10 @@ from litellm.integrations.mlflow import MlflowLogger
 from litellm.integrations.sqs import SQSLogger
 from litellm.litellm_core_utils.core_helpers import is_expected_client_error, reconstruct_model_name
 from litellm.litellm_core_utils.get_litellm_params import get_litellm_params
-from litellm.litellm_core_utils.internal_call_metadata import is_unbilled_non_inference_call
+from litellm.litellm_core_utils.internal_call_metadata import (
+    MODEL_ACCESS_GROUP_METADATA_KEY,
+    is_unbilled_non_inference_call,
+)
 from litellm.litellm_core_utils.llm_cost_calc.guardrail_cost import (
     cost_breakdown_with_guardrail,
     guardrail_information_cost,
@@ -2872,6 +2875,8 @@ class Logging(LiteLLMLoggingBaseClass):
             batch_cost: Final = kwargs.get("batch_cost", None)
             batch_usage = kwargs.get("batch_usage", None)
             batch_models = kwargs.get("batch_models", None)
+            batch_successful_requests: Final = kwargs.get("batch_successful_requests", None)
+            batch_failed_requests: Final = kwargs.get("batch_failed_requests", None)
             has_explicit_batch_data: Final = all(x is not None for x in (batch_cost, batch_usage, batch_models))
 
             should_compute_batch_data: Final = (
@@ -2880,14 +2885,12 @@ class Logging(LiteLLMLoggingBaseClass):
             if has_explicit_batch_data:
                 result._hidden_params["response_cost"] = batch_cost
                 result._hidden_params["batch_models"] = batch_models
+                result._hidden_params["batch_successful_requests"] = batch_successful_requests  # pyright: ignore[reportPrivateUsage]  # rebind-ok: same result._hidden_params pattern as response_cost/batch_models above
+                result._hidden_params["batch_failed_requests"] = batch_failed_requests  # pyright: ignore[reportPrivateUsage]  # rebind-ok: same pattern as above
                 result.usage = batch_usage
 
             elif should_compute_batch_data:
-                (
-                    response_cost,
-                    batch_usage,
-                    batch_models,
-                ) = await _handle_completed_batch(
+                batch_result: Final = await _handle_completed_batch(
                     batch=result,
                     custom_llm_provider=self.custom_llm_provider,
                     model_name=self.get_deployment_model_for_cost(),
@@ -2895,9 +2898,11 @@ class Logging(LiteLLMLoggingBaseClass):
                     model_info=self.get_router_deployment_model_info(),
                 )
 
-                result._hidden_params["response_cost"] = response_cost
-                result._hidden_params["batch_models"] = batch_models
-                result.usage = batch_usage
+                result._hidden_params["response_cost"] = batch_result.cost
+                result._hidden_params["batch_models"] = batch_result.models
+                result._hidden_params["batch_successful_requests"] = batch_result.successful_requests  # pyright: ignore[reportPrivateUsage]  # rebind-ok: same pattern as above
+                result._hidden_params["batch_failed_requests"] = batch_result.failed_requests  # pyright: ignore[reportPrivateUsage]  # rebind-ok: same pattern as above
+                result.usage = batch_result.usage
 
         start_time, end_time, result = self._success_handler_helper_fn(
             start_time=start_time,
@@ -5049,6 +5054,42 @@ def is_valid_sha256_hash(value: str) -> bool:
     return bool(re.fullmatch(r"[a-fA-F0-9]{64}", value))
 
 
+def coerce_model_access_groups(value: object) -> tuple[str, ...]:
+    """Model access group names out of untrusted request metadata, deduped and order preserving."""
+    if not isinstance(value, (list, tuple)):
+        return ()
+    return tuple(dict.fromkeys(group for group in value if isinstance(group, str) and group))
+
+
+def _model_access_groups_on_auth_object(user_api_key_auth: object) -> object:
+    if isinstance(user_api_key_auth, Mapping):
+        return user_api_key_auth.get("matched_model_access_groups")
+    return getattr(user_api_key_auth, "matched_model_access_groups", None)
+
+
+def _model_access_groups_from_metadata(metadata: Mapping[str, object]) -> tuple[str, ...]:
+    stamped: Final = coerce_model_access_groups(metadata.get(MODEL_ACCESS_GROUP_METADATA_KEY))
+    if stamped:
+        return stamped
+    return coerce_model_access_groups(_model_access_groups_on_auth_object(metadata.get("user_api_key_auth")))
+
+
+def request_model_access_groups_from_litellm_params(litellm_params: Mapping[str, object]) -> tuple[str, ...]:
+    """Access groups the auth layer stamped onto this request, from whichever metadata field carries them.
+
+    Detached internal sub-calls only inherit the identity keys, so the auth object is the
+    fallback there, exactly as _get_budget_reservation_from_metadata does for reservations.
+    """
+    for metadata_variable_name in ("metadata", "litellm_metadata"):
+        metadata = litellm_params.get(metadata_variable_name)
+        if not isinstance(metadata, Mapping):
+            continue
+        model_access_groups = _model_access_groups_from_metadata(metadata)
+        if model_access_groups:
+            return model_access_groups
+    return ()
+
+
 class StandardLoggingPayloadSetup:
     @staticmethod
     def cleanup_timestamps(
@@ -5422,6 +5463,8 @@ class StandardLoggingPayloadSetup:
             additional_headers=None,
             litellm_overhead_time_ms=None,
             batch_models=None,
+            batch_successful_requests=None,
+            batch_failed_requests=None,
             litellm_model_name=None,
             usage_object=None,
         )
@@ -5812,6 +5855,8 @@ def _extract_response_obj_and_hidden_params(
                     response_cost=None,
                     litellm_overhead_time_ms=None,
                     batch_models=None,
+                    batch_successful_requests=None,
+                    batch_failed_requests=None,
                     litellm_model_name=None,
                     usage_object=None,
                 )
@@ -5896,6 +5941,7 @@ def get_standard_logging_object_payload(
         request_tags: Final = StandardLoggingPayloadSetup._get_request_tags(
             litellm_params=litellm_params, proxy_server_request=proxy_server_request
         )
+        request_model_access_groups: Final = request_model_access_groups_from_litellm_params(litellm_params)
 
         # cleanup timestamps
         (
@@ -6058,6 +6104,7 @@ def get_standard_logging_object_payload(
             prompt_tokens=usage_dict.get("prompt_tokens", 0),
             completion_tokens=usage_dict.get("completion_tokens", 0),
             request_tags=request_tags,
+            request_model_access_groups=request_model_access_groups,
             end_user=end_user_id,
             api_base=StandardLoggingPayloadSetup.strip_trailing_slash(litellm_params.get("api_base", "")) or "",
             model_group=_model_group,
@@ -6228,6 +6275,8 @@ def create_dummy_standard_logging_payload() -> StandardLoggingPayload:
         additional_headers=None,
         litellm_overhead_time_ms=None,
         batch_models=None,
+        batch_successful_requests=None,
+        batch_failed_requests=None,
         litellm_model_name=None,
         usage_object=None,
     )
@@ -6269,6 +6318,7 @@ def create_dummy_standard_logging_payload() -> StandardLoggingPayload:
         cache_key=None,
         saved_cache_cost=saved_cache_cost,
         request_tags=[],
+        request_model_access_groups=(),
         end_user=None,
         requester_ip_address="127.0.0.1",
         messages=messages,

litellm/llms/anthropic/chat/transformation.py

@@ -1268,7 +1268,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
             )
 
     @staticmethod
-    def _cap_thinking_budget_to_max_tokens(
+    def cap_thinking_budget_to_max_tokens(
         thinking: AnthropicThinkingParam, max_tokens: int | None
     ) -> AnthropicThinkingParam | None:
         """Cap a legacy ``thinking.budget_tokens`` below ``max_tokens`` (Anthropic
@@ -1530,7 +1530,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
                         llm_provider=self._resolved_provider,
                     )
                     capped_thinking = (
-                        AnthropicConfig._cap_thinking_budget_to_max_tokens(legacy_thinking, max_tokens)
+                        AnthropicConfig.cap_thinking_budget_to_max_tokens(legacy_thinking, max_tokens)
                         if legacy_thinking is not None
                         else None
                     )

litellm/llms/anthropic/common_utils.py

@@ -1099,6 +1099,25 @@ def is_empty_thinking_block(block: object) -> bool:
     return not isinstance(thinking, str) or not thinking.strip()
 
 
+def is_empty_unsigned_thinking_block(block: object) -> bool:
+    """
+    True for an empty ``{"type": "thinking"}`` block carrying no signature.
+
+    The emit-side predicate: response paths drop a thinking block only when it
+    holds nothing the client could need.  A signature-only block is a real
+    provider response (Bedrock Converse under adaptive thinking emits a
+    reasoning block with empty text and only a signature) and the client needs
+    the signature to replay reasoning across tool-use turns, so it must be
+    emitted.  Request paths keep using :func:`is_empty_thinking_block`:
+    Anthropic rejects empty thinking blocks in request history regardless of
+    signature, and the inbound strip self-heals a replayed signature-only
+    block.
+    """
+    if not isinstance(block, dict) or not is_empty_thinking_block(block):
+        return False
+    return not block.get("signature")
+
+
 def normalize_anthropic_tool_use_id(raw_id: str) -> str:
     """
     Normalize a tool_use / tool_result id for Anthropic's ``^[a-zA-Z0-9_-]+$``

litellm/llms/anthropic/experimental_pass_through/adapters/streaming_iterator.py

@@ -1029,7 +1029,7 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper):
 
     @staticmethod
     def _is_blank_delta(chunk: "ModelResponseStream") -> bool:
-        from litellm.llms.anthropic.common_utils import is_empty_thinking_block
+        from litellm.llms.anthropic.common_utils import is_empty_unsigned_thinking_block
 
         choice: Final = chunk.choices[0]
         if choice.finish_reason is not None:
@@ -1041,11 +1041,14 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper):
             return False
         if getattr(delta, "reasoning_content", None):
             return False
-        # thinking_blocks whose entries are all empty (even if signed) must not
+        # thinking_blocks whose entries are all empty AND unsigned must not
         # open a block: the emitted {"type": "thinking", "thinking": ""} gets
-        # replayed as history and Anthropic rejects it (LIT-6357).
+        # replayed as history and Anthropic rejects it (LIT-6357). A signed
+        # entry opens the block so the client receives the replay signature.
         thinking_blocks: Final = getattr(delta, "thinking_blocks", None)
-        if thinking_blocks and any(isinstance(b, dict) and not is_empty_thinking_block(b) for b in thinking_blocks):
+        if thinking_blocks and any(
+            isinstance(b, dict) and not is_empty_unsigned_thinking_block(b) for b in thinking_blocks
+        ):
             return False
         return True
 

litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py

@@ -90,7 +90,7 @@ from litellm.litellm_core_utils.reasoning_effort_utils import (
     reasoning_effort_from_thinking_budget,
 )
 from litellm.llms.anthropic.common_utils import (
-    is_empty_thinking_block,
+    is_empty_unsigned_thinking_block,
     normalize_anthropic_tool_use_id,
 )
 from litellm.llms.anthropic.experimental_pass_through.context_management import (
@@ -1267,7 +1267,7 @@ class LiteLLMAnthropicMessagesAdapter:
             if hasattr(choice.message, "thinking_blocks") and choice.message.thinking_blocks:
                 for thinking_block in choice.message.thinking_blocks:
                     if thinking_block.get("type") == "thinking":
-                        if is_empty_thinking_block(thinking_block):
+                        if is_empty_unsigned_thinking_block(thinking_block):
                             continue
                         thinking_value = thinking_block.get("thinking", "")
                         signature_value = thinking_block.get("signature", "")

litellm/llms/anthropic/experimental_pass_through/messages/transformation.py

@@ -40,6 +40,11 @@ DROP_UNSUPPORTED_ADAPTIVE_EFFORT_WARNING: Final = (
     "minimum thinking budget."
 )
 
+DROP_UNFITTING_REASONING_EFFORT_WARNING: Final = (
+    "Dropping `thinking` mapped from reasoning_effort=%s for model=%s: max_tokens=%s "
+    "is too small to fit the minimum thinking budget."
+)
+
 
 class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
     @property
@@ -335,11 +340,15 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
         return headers, api_base
 
     @staticmethod
-    def _translate_reasoning_effort_to_anthropic(model: str, optional_params: dict, custom_llm_provider: str) -> None:
+    def _translate_reasoning_effort_to_anthropic(
+        model: str, optional_params: dict, max_tokens: int | None, custom_llm_provider: str
+    ) -> None:
         """Map OpenAI-style ``reasoning_effort`` to native Anthropic params.
 
         Caller-supplied ``thinking`` / ``output_config`` win over the alias.
-        ``effort='none'`` clears both. Invalid efforts raise a 400.
+        ``effort='none'`` clears both. Invalid efforts raise a 400. A mapped
+        thinking budget is capped below ``max_tokens`` and dropped when even
+        the minimum budget cannot fit.
         """
         from litellm.exceptions import BadRequestError as _BadRequestError
         from litellm.llms.anthropic.chat.transformation import (
@@ -365,7 +374,12 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
             optional_params.pop("output_config", None)
             return
 
-        optional_params.setdefault("thinking", mapped_thinking)
+        fitted_thinking: Final = AnthropicConfig.cap_thinking_budget_to_max_tokens(mapped_thinking, max_tokens)
+        if fitted_thinking is None:
+            verbose_logger.warning(DROP_UNFITTING_REASONING_EFFORT_WARNING, reasoning_effort, model, max_tokens)
+            return
+
+        optional_params.setdefault("thinking", fitted_thinking)
         if AnthropicModelInfo._is_adaptive_thinking_model(model, custom_llm_provider):
             mapped_effort: Final = REASONING_EFFORT_TO_OUTPUT_CONFIG_EFFORT.get(reasoning_effort)
             if mapped_effort is None:
@@ -510,7 +524,7 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
         except _BadRequestError as e:
             raise AnthropicError(message=str(e.message), status_code=400)
         capped_thinking: Final = (
-            AnthropicConfig._cap_thinking_budget_to_max_tokens(legacy_thinking, max_tokens)
+            AnthropicConfig.cap_thinking_budget_to_max_tokens(legacy_thinking, max_tokens)
             if legacy_thinking is not None
             else None
         )
@@ -582,6 +596,7 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
         self._translate_reasoning_effort_to_anthropic(
             model=model,
             optional_params=anthropic_messages_optional_request_params,
+            max_tokens=max_tokens,
             custom_llm_provider=self._resolved_provider,
         )
 

litellm/llms/azure/azure.py

@@ -846,6 +846,7 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM):
         api_key: str,
         data: dict,
         headers: dict,
+        deployment_name: str | None = None,
     ) -> httpx.Response:
         """
         Implemented for azure dall-e-2 image gen calls
@@ -957,7 +958,7 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM):
                 content=json.dumps(result).encode("utf-8"),
                 request=httpx.Request(method="POST", url="https://api.openai.com/v1"),
             )
-        request_json: Final = azure_deployment_image_generation_json_body(api_base, data)
+        request_json: Final = azure_deployment_image_generation_json_body(api_base, data, deployment_name)
         return await async_handler.post(
             url=api_base,
             json=request_json,
@@ -973,6 +974,7 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM):
         api_key: str,
         data: dict,
         headers: dict,
+        deployment_name: str | None = None,
     ) -> httpx.Response:
         """
         Implemented for azure dall-e-2 image gen calls
@@ -1073,7 +1075,7 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM):
                 content=json.dumps(result).encode("utf-8"),
                 request=httpx.Request(method="POST", url="https://api.openai.com/v1"),
             )
-        request_json: Final = azure_deployment_image_generation_json_body(api_base, data)
+        request_json: Final = azure_deployment_image_generation_json_body(api_base, data, deployment_name)
         return sync_handler.post(
             url=api_base,
             json=request_json,
@@ -1091,9 +1093,10 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM):
             AzureFoundryMAIImageGenerationConfig,
         )
 
-        api_base: str = azure_client_params.get("azure_endpoint", "")  # "https://example-endpoint.openai.azure.com"
-        if api_base.endswith("/"):
-            api_base = api_base.rstrip("/")
+        # deployment-scoped endpoints are moved to "base_url" by select_azure_base_url_or_endpoint
+        api_base: str = (azure_client_params.get("azure_endpoint") or azure_client_params.get("base_url") or "").rstrip(
+            "/"
+        )
         api_version: Final[str] = azure_client_params.get("api_version", "")
         if model is None:
             model = ""
@@ -1113,6 +1116,14 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM):
                 api_version=api_version,
             )
 
+        v1_url: Final = BaseAzureLLM.get_azure_v1_image_url(
+            api_base=api_base,
+            api_version=api_version,
+            route="/openai/images/generations",
+        )
+        if v1_url is not None:
+            return v1_url
+
         if "/openai/deployments/" in api_base:
             base_url_with_deployment = api_base
         else:
@@ -1167,6 +1178,7 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM):
                 api_key=api_key,
                 data=data,
                 headers=headers,
+                deployment_name=model,
             )
 
             provider_config: Final = get_azure_image_generation_config(data.get("model", "dall-e-2"))
@@ -1302,6 +1314,7 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM):
                 api_key=api_key or "",
                 data=data,
                 headers=headers,
+                deployment_name=model,
             )
             provider_config: Final = get_azure_image_generation_config(data.get("model", "dall-e-2"))
             if isinstance(provider_config, AzureFoundryMAIImageGenerationConfig):

litellm/llms/azure/common_utils.py

@@ -4,6 +4,7 @@ import json
 import os
 from collections.abc import Callable, Mapping
 from functools import lru_cache
+from types import MappingProxyType
 from typing import Any, Final, Literal, NamedTuple, cast
 
 import httpx
@@ -789,6 +790,32 @@ class BaseAzureLLM(BaseOpenAILLM):
 
         return str(final_url)
 
+    @staticmethod
+    def get_azure_v1_image_url(api_base: str, api_version: str | None, route: str) -> str | None:
+        """
+        Azure's v1 surface serves images at ``/openai/v1/images/{generations,edits}`` and routes by
+        ``model`` in the request body, so any deployment path and stale ``api-version`` in
+        ``api_base`` have to be dropped.
+
+        Returns None when ``api_version`` is a dated one, which still uses the deployment route.
+        """
+        if not BaseAzureLLM._is_azure_v1_api_version(api_version):
+            return None
+
+        base_url: Final = httpx.URL(api_base)
+        openai_path_start: Final = base_url.path.find("/openai")
+        resource_base: Final = str(
+            base_url.copy_with(
+                path=base_url.path if openai_path_start == -1 else base_url.path[:openai_path_start],
+                params=httpx.QueryParams(tuple((k, v) for k, v in base_url.params.multi_items() if k != "api-version")),
+            )
+        )
+        return BaseAzureLLM._get_base_azure_url(
+            api_base=resource_base,
+            litellm_params=MappingProxyType({"api_version": api_version}),
+            route=route,
+        )
+
     @staticmethod
     def _is_azure_v1_api_version(api_version: str | None) -> bool:
         if api_version is None:

litellm/llms/azure/image_edit/transformation.py

@@ -93,8 +93,6 @@ class AzureImageEditConfig(OpenAIImageEditConfig):
             raise ValueError(
                 f"api_base is required for Azure AI Studio. Please set the api_base parameter. Passed `api_base={api_base}`"
             )
-        original_url: Final = httpx.URL(api_base)
-
         # Resolve api_version: litellm_params > litellm.api_version > AZURE_API_VERSION env > default.
         # Mirrors the fallback chain used by the Azure chat path in common_utils.py,
         # so callers that set a global / env api_version don't get an unversioned URL.
@@ -105,6 +103,16 @@ class AzureImageEditConfig(OpenAIImageEditConfig):
             or litellm.AZURE_DEFAULT_API_VERSION
         )
 
+        v1_url: Final = BaseAzureLLM.get_azure_v1_image_url(
+            api_base=api_base,
+            api_version=api_version,
+            route="/openai/images/edits",
+        )
+        if v1_url is not None:
+            return v1_url
+
+        original_url: Final = httpx.URL(api_base)
+
         # Create a new dictionary with existing params
         query_params: Final = dict(original_url.params)
 

litellm/llms/azure/image_generation/http_utils.py

@@ -1,7 +1,9 @@
 """HTTP helpers for Azure OpenAI image generation (REST, not SDK)."""
 
+from typing import Final
 
-def azure_deployment_image_generation_json_body(api_base: str, data: dict) -> dict:
+
+def azure_deployment_image_generation_json_body(api_base: str, data: dict, deployment_name: str | None = None) -> dict:
     """
     Build the JSON body for Azure OpenAI image generation POSTs.
 
@@ -9,9 +11,20 @@ def azure_deployment_image_generation_json_body(api_base: str, data: dict) -> di
     deployment in the URL only; sending ``model`` in the body (especially the deployment
     name) breaks some models (e.g. gpt-image-2). See LiteLLM #26316.
 
+    For the v1 surface (``.../openai/v1/images/...``), Azure routes by the deployment
+    name in the body ``model`` field, so the deployment name must replace any base
+    model name there or Azure answers 404 DeploymentNotFound.
+
     Provider-style URLs (e.g. ``/providers/...`` for FLUX on Azure AI) keep all keys
     so non–OpenAI-deployment payloads still work.
     """
-    if "images/generations" in api_base and "/openai/deployments/" in api_base:
-        return {k: v for k, v in data.items() if k != "model"}
-    return data
+    drop_model: Final = "images/generations" in api_base and "/openai/deployments/" in api_base
+    v1_route: Final = "/openai/v1/images/" in api_base and bool(deployment_name)
+    if not drop_model and not v1_route:
+        return data
+    entries: Final = (
+        tuple((k, v) for k, v in data.items() if k != "model")
+        if drop_model
+        else (*data.items(), ("model", deployment_name))
+    )
+    return {k: v for k, v in entries}

litellm/llms/base_llm/managed_resources/base_managed_resource.py

@@ -12,6 +12,7 @@ from litellm.llms.base_llm.managed_resources.isolation import (
     build_list_page,
     build_owner_filter,
     can_access_resource,
+    resolve_resource_owner_id,
 )
 from litellm.proxy._types import UserAPIKeyAuth
 from litellm.types.utils import SpecialEnums
@@ -157,7 +158,7 @@ class BaseManagedResource(ABC, Generic[ResourceObjectType]):
             "resource_object": resource_object,
             "model_mappings": model_mappings,
             "flat_model_resource_ids": list(model_mappings.values()),
-            "created_by": user_api_key_dict.user_id,
+            "created_by": resolve_resource_owner_id(user_api_key_dict),
             "team_id": user_api_key_dict.team_id,
             "updated_by": user_api_key_dict.user_id,
         }
@@ -179,7 +180,7 @@ class BaseManagedResource(ABC, Generic[ResourceObjectType]):
             "unified_resource_id": unified_resource_id,
             "model_mappings": json.dumps(model_mappings),
             "flat_model_resource_ids": list(model_mappings.values()),
-            "created_by": user_api_key_dict.user_id,
+            "created_by": resolve_resource_owner_id(user_api_key_dict),
             "team_id": user_api_key_dict.team_id,
             "updated_by": user_api_key_dict.user_id,
         }

litellm/llms/base_llm/managed_resources/isolation.py

@@ -3,10 +3,11 @@ Tenant-isolation helpers for managed file/batch/vector-store resources.
 
 Returns a Prisma filter and an ownership check that scope managed resources
 to the caller's identity: proxy admins see everything, user-keyed callers
-see records they created, and service-account keys (no user_id) fall back
-to the resource's owning team. Callers with no admin role and no
-identifying ids are denied so an empty user_id can never select an
-unscoped query.
+see records they created, service-account keys (no user_id) fall back to
+the resource's owning team, and keys with neither a user_id nor a team_id
+fall back to their own hashed token so they can still reach the resources
+they created. Callers with no admin role and no identifying ids at all
+are denied so an empty user_id can never select an unscoped query.
 """
 
 from typing import Any, Final
@@ -19,6 +20,32 @@ from litellm.proxy._types import (
 )
 
 
+def resolve_resource_owner_id(
+    user_api_key_dict: UserAPIKeyAuth,
+) -> str | None:
+    """Return the identity to stamp on (and match against) a managed
+    resource's ``created_by``.
+
+    A key with neither a user_id nor a team_id would otherwise stamp
+    ``created_by=None`` and be locked out of its own resources, so it owns
+    them under its hashed token instead, using the ``key:`` scope prefix
+    already used by ``proxy/common_utils/resource_ownership.py``. ``None``
+    means the caller has no usable identity of its own and must fall back
+    to team scoping, or be denied.
+    """
+    if user_api_key_dict.user_id is not None:
+        return user_api_key_dict.user_id
+
+    if user_api_key_dict.team_id is not None:
+        return None
+
+    token: Final = user_api_key_dict.token or user_api_key_dict.api_key
+    if token:
+        return f"key:{token}"
+
+    return None
+
+
 def build_list_page(items: list[Any], has_more: bool = False) -> dict[str, Any]:
     """Build the OpenAI-style paginated list response shape used by managed
     file/batch/vector-store listings. ``first_id`` and ``last_id`` are
@@ -39,7 +66,8 @@ def build_owner_filter(
     to records the caller is allowed to see.
 
     - ``{}`` means no scoping (proxy admins).
-    - ``{"created_by": <user_id>}`` for user-keyed callers.
+    - ``{"created_by": <owner_id>}`` for user-keyed callers, and for keys
+      with no user_id and no team_id (owner id is their hashed token).
     - ``{"team_id": <team_id>}`` for service-account callers
       that have a team but no user_id.
     - ``{"OR": [...]}`` when the caller has both — listing must include
@@ -62,12 +90,13 @@ def build_owner_filter(
             ]
         }
 
-    if user_id is not None:
-        return {"created_by": user_id}
-
     if team_id is not None:
         return {"team_id": team_id}
 
+    owner_id: Final = resolve_resource_owner_id(user_api_key_dict)
+    if owner_id is not None:
+        return {"created_by": owner_id}
+
     return None
 
 
@@ -86,8 +115,8 @@ def can_access_resource(
     if _user_has_admin_view(user_api_key_dict):
         return True
 
-    user_id: Final = user_api_key_dict.user_id
-    if user_id is not None and created_by is not None and created_by == user_id:
+    owner_id: Final = resolve_resource_owner_id(user_api_key_dict)
+    if owner_id is not None and created_by is not None and created_by == owner_id:
         return True
 
     team_id: Final = user_api_key_dict.team_id

litellm/llms/bedrock/chat/converse_transformation.py

@@ -924,7 +924,7 @@ class AmazonConverseConfig(BaseConfig):
                         custom_llm_provider="bedrock",
                     )
                     capped = (
-                        AnthropicConfig._cap_thinking_budget_to_max_tokens(legacy_thinking, max_tokens)
+                        AnthropicConfig.cap_thinking_budget_to_max_tokens(legacy_thinking, max_tokens)
                         if legacy_thinking is not None
                         else None
                     )

litellm/llms/bedrock/embed/cohere_transformation.py

@@ -20,7 +20,9 @@ class BedrockCohereEmbeddingConfig:
     def map_openai_params(self, non_default_params: dict, optional_params: dict) -> dict:
         for k, v in non_default_params.items():
             if k == "encoding_format":
-                optional_params["embedding_types"] = v if isinstance(v, list) else [v]
+                optional_params["embedding_types"] = [
+                    "float" if fmt == "base64" else fmt for fmt in (tuple(v) if isinstance(v, list) else (v,))
+                ]
             elif k == "dimensions":
                 optional_params["output_dimension"] = v
         return optional_params

litellm/llms/openai/responses/transformation.py

@@ -33,6 +33,7 @@ else:
 
 _NO_TOOL_UPDATE: Final[Mapping[str, object]] = MappingProxyType({})
 _MODEL_FAMILIES_REJECTING_TOP_LEVEL_SCHEMA_COMBINATORS: Final = ("gpt-4", "gpt-3.5", "chatgpt-4o", "o1", "o3", "o4")
+_PROVIDERS_WITH_COMBINATOR_REJECTING_VALIDATOR: Final = frozenset({LlmProviders.AZURE, LlmProviders.OPENAI})
 
 
 class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig):
@@ -172,7 +173,9 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig):
         input = self._validate_input_param(input)
         tools = response_api_optional_request_params.get("tools")
         input, tools = self.remove_cache_control_flag_from_input_and_tools(model=model, input=input, tools=tools)
-        sanitized_tools: Final = self._flatten_tool_schema_combinators_for_openai(model=model, tools=tools)
+        sanitized_tools: Final = self._flatten_tool_schema_combinators_for_openai(
+            model=model, tools=tools, litellm_params=litellm_params
+        )
         if sanitized_tools is not None:
             response_api_optional_request_params["tools"] = sanitized_tools
         final_request_params: Final = dict(
@@ -217,6 +220,7 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig):
         self,
         model: str,
         tools: list[ALL_RESPONSES_API_TOOL_PARAMS] | None,  # mutable-ok: request tools are a JSON list
+        litellm_params: GenericLiteLLMParams,
     ) -> list[ALL_RESPONSES_API_TOOL_PARAMS] | None:  # mutable-ok: request tools are a JSON list
         """Flatten top-level schema combinators only where OpenAI's validator rejects them.
 
@@ -224,10 +228,14 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig):
         Codex talks to natively) accept them, and so do GPT-5 and later models,
         which also call tools better with the union intact. Codex wraps MCP tools
         inside namespace entries, so nested ``tools`` arrays are walked too.
+        Azure OpenAI shares the validator but names deployments arbitrarily, so
+        the router's declared ``model_info.base_model`` wins over the deployment
+        name and an unrecognized name without one is left untouched.
         """
-        if tools is None or self.custom_llm_provider != LlmProviders.OPENAI:
+        if tools is None or self.custom_llm_provider not in _PROVIDERS_WITH_COMBINATOR_REJECTING_VALIDATOR:
             return tools
-        if not self._rejects_top_level_schema_combinators(model):
+        gate_model: Final = self._combinator_gate_model(model=model, litellm_params=litellm_params)
+        if not self._rejects_top_level_schema_combinators(gate_model):
             return tools
         flattened: Final = [  # mutable-ok: request tools are a JSON list
             self._flattened_tool_or_passthrough(tool) for tool in tools
@@ -244,6 +252,12 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig):
         base_model: Final = bare_model.split(":")[1] if bare_model.startswith("ft:") else bare_model
         return base_model.startswith(_MODEL_FAMILIES_REJECTING_TOP_LEVEL_SCHEMA_COMBINATORS)
 
+    @staticmethod
+    def _combinator_gate_model(model: str, litellm_params: GenericLiteLLMParams) -> str:
+        model_info: Final[object] = getattr(litellm_params, "model_info", None)
+        base_model: Final[object] = model_info.get("base_model") if isinstance(model_info, dict) else None
+        return base_model if isinstance(base_model, str) and base_model else model
+
     @staticmethod
     def _flattened_tool_entry(
         entry: Mapping[str, object],
@@ -714,7 +728,9 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig):
         input = self._validate_input_param(input)
         tools = response_api_optional_request_params.get("tools")
         input, tools = self.remove_cache_control_flag_from_input_and_tools(model=model, input=input, tools=tools)
-        sanitized_tools: Final = self._flatten_tool_schema_combinators_for_openai(model=model, tools=tools)
+        sanitized_tools: Final = self._flatten_tool_schema_combinators_for_openai(
+            model=model, tools=tools, litellm_params=litellm_params
+        )
         if sanitized_tools is not None:
             response_api_optional_request_params["tools"] = sanitized_tools
         data: Final = dict(ResponsesAPIRequestParams(model=model, input=input, **response_api_optional_request_params))

litellm/llms/soniox/common_utils.py

@@ -138,13 +138,25 @@ def _soniox_token_to_subtitle_token(token: SonioxToken) -> SubtitleToken:
     )
 
 
+def _subtitle_tokens(tokens: Sequence[SonioxToken]) -> tuple[SubtitleToken, ...]:
+    """
+    Convert Soniox tokens for subtitle rendering, excluding translation tokens
+    (``translation_status == "translation"``): Soniox does not timestamp them,
+    so they cannot be aligned to the audio and would otherwise mix translated
+    text into original-language cues.
+    """
+    return tuple(
+        _soniox_token_to_subtitle_token(token) for token in tokens if token.get("translation_status") != "translation"
+    )
+
+
 def render_soniox_tokens_as_srt(tokens: Sequence[SonioxToken]) -> str:
     """
     Render Soniox tokens as SRT (SubRip) subtitle format.
 
     Returns an empty string if no tokens have timestamp data.
     """
-    return render_subtitle_tokens_as_srt(tuple(_soniox_token_to_subtitle_token(token) for token in tokens))
+    return render_subtitle_tokens_as_srt(_subtitle_tokens(tokens))
 
 
 def render_soniox_tokens_as_vtt(tokens: Sequence[SonioxToken]) -> str:
@@ -153,4 +165,4 @@ def render_soniox_tokens_as_vtt(tokens: Sequence[SonioxToken]) -> str:
 
     Returns the VTT header even if no cues are present.
     """
-    return render_subtitle_tokens_as_vtt(tuple(_soniox_token_to_subtitle_token(token) for token in tokens))
+    return render_subtitle_tokens_as_vtt(_subtitle_tokens(tokens))

litellm/llms/vertex_ai/audio_transcription/gemini_transcribe_transformation.py

@@ -0,0 +1,216 @@
+import base64
+from collections.abc import Mapping, Sequence
+from typing import Final
+
+from httpx import Headers, Response
+
+import litellm
+from litellm.exceptions import UnsupportedParamsError
+from litellm.litellm_core_utils.audio_utils.utils import (
+    normalize_transcription_language_to_bcp47,
+    process_audio_file,
+)
+from litellm.llms.base_llm.audio_transcription.transformation import (
+    AudioTranscriptionRequestData,
+    BaseAudioTranscriptionConfig,
+)
+from litellm.llms.base_llm.chat.transformation import BaseLLMException
+from litellm.llms.vertex_ai.audio_transcription.transformation import (
+    SUPPORTED_RESPONSE_FORMATS,
+    validate_vertex_transcription_location,
+    validate_vertex_transcription_project_id,
+)
+from litellm.llms.vertex_ai.common_utils import VertexAIError, get_vertex_base_url
+from litellm.llms.vertex_ai.vertex_llm_base import VertexBase
+from litellm.types.llms.openai import (
+    AllMessageValues,
+    OpenAIAudioTranscriptionOptionalParams,
+)
+from litellm.types.llms.vertex_ai_gemini_transcription import (
+    VertexGeminiTranscriptionAudioConfig,
+    VertexGeminiTranscriptionContent,
+    VertexGeminiTranscriptionGenerationConfig,
+    VertexGeminiTranscriptionInlineData,
+    VertexGeminiTranscriptionPart,
+    VertexGeminiTranscriptionRequest,
+    VertexGeminiTranscriptionResponse,
+)
+from litellm.types.utils import (
+    FileTypes,
+    TranscriptionResponse,
+    TranscriptionUsageInputTokenDetailsObject,
+    TranscriptionUsageTokensObject,
+)
+
+DEFAULT_GEMINI_TRANSCRIBE_LOCATION: Final = "global"
+AUDIO_MODALITY: Final = "AUDIO"
+
+
+class VertexGeminiAudioTranscriptionConfig(BaseAudioTranscriptionConfig, VertexBase):
+    def __init__(self) -> None:
+        BaseAudioTranscriptionConfig.__init__(self)
+        VertexBase.__init__(self)
+
+    def get_supported_openai_params(
+        self, model: str
+    ) -> list[OpenAIAudioTranscriptionOptionalParams]:  # mutable-ok: BaseAudioTranscriptionConfig signature
+        return ["language", "response_format"]
+
+    def map_openai_params(
+        self,
+        non_default_params: Mapping[str, object],
+        optional_params: Mapping[str, object],
+        model: str,
+        drop_params: bool,
+    ) -> dict[str, object]:  # mutable-ok: BaseAudioTranscriptionConfig signature
+        supported_params: Final = frozenset(self.get_supported_openai_params(model))
+        mapped: Final = {
+            **optional_params,
+            **{k: v for k, v in non_default_params.items() if k in supported_params},
+        }
+        response_format: Final = mapped.get("response_format")
+        if response_format is None or response_format in SUPPORTED_RESPONSE_FORMATS:
+            return mapped
+        if drop_params or litellm.drop_params:
+            return {k: v for k, v in mapped.items() if k != "response_format"}
+        raise UnsupportedParamsError(
+            status_code=400,
+            message=(
+                f"Vertex AI Gemini transcription does not support response_format={response_format!r}. "
+                f"Supported values: {', '.join(SUPPORTED_RESPONSE_FORMATS)}. "
+                "To drop unsupported openai params from the call, set `litellm.drop_params = True`"
+            ),
+        )
+
+    def get_error_class(
+        self,
+        error_message: str,
+        status_code: int,
+        headers: dict | Headers,  # mutable-ok: base signature and VertexAIError take dict | Headers
+    ) -> BaseLLMException:
+        return VertexAIError(status_code=status_code, message=error_message, headers=headers)
+
+    def validate_environment(
+        self,
+        headers: Mapping[str, str],
+        model: str,
+        messages: Sequence[AllMessageValues],
+        optional_params: Mapping[str, object],
+        litellm_params: Mapping[str, object],
+        api_key: str | None = None,
+        api_base: str | None = None,
+    ) -> dict[str, str]:  # mutable-ok: BaseAudioTranscriptionConfig signature
+        vertex_params: Final = dict(litellm_params)
+        access_token, project_id = self._ensure_access_token(
+            credentials=self.safe_get_vertex_ai_credentials(vertex_params),
+            project_id=self.safe_get_vertex_ai_project(vertex_params),
+            custom_llm_provider="vertex_ai",
+        )
+        return {
+            **headers,
+            "Authorization": f"Bearer {access_token}",
+            "x-goog-user-project": project_id,
+            "Content-Type": "application/json",
+        }
+
+    def get_complete_url(
+        self,
+        api_base: str | None,
+        api_key: str | None,
+        model: str,
+        optional_params: Mapping[str, object],
+        litellm_params: Mapping[str, object],
+        stream: bool | None = None,
+    ) -> str:
+        vertex_params: Final = dict(litellm_params)
+        location: Final = validate_vertex_transcription_location(
+            self.safe_get_vertex_ai_location(vertex_params), default_location=DEFAULT_GEMINI_TRANSCRIBE_LOCATION
+        )
+        project_id: Final = validate_vertex_transcription_project_id(
+            self.safe_get_vertex_ai_project(vertex_params) or self._resolve_project_id_from_credentials(vertex_params)
+        )
+        base_url: Final = (api_base or get_vertex_base_url(location)).rstrip("/")
+        bare_model: Final = model.removeprefix("vertex_ai/")
+        model_path: Final = f"projects/{project_id}/locations/{location}/publishers/google/models/{bare_model}"
+        return f"{base_url}/v1/{model_path}:generateContent"
+
+    def _resolve_project_id_from_credentials(self, litellm_params: Mapping[str, object]) -> str:
+        vertex_params: Final = dict(litellm_params)
+        _, project_id = self._ensure_access_token(
+            credentials=self.safe_get_vertex_ai_credentials(vertex_params),
+            project_id=None,
+            custom_llm_provider="vertex_ai",
+        )
+        return project_id
+
+    def transform_audio_transcription_request(
+        self,
+        model: str,
+        audio_file: FileTypes,
+        optional_params: Mapping[str, object],
+        litellm_params: Mapping[str, object],
+    ) -> AudioTranscriptionRequestData:
+        processed_audio: Final = process_audio_file(audio_file)
+        request_body: Final = VertexGeminiTranscriptionRequest(
+            contents=(
+                VertexGeminiTranscriptionContent(
+                    role="user",
+                    parts=(
+                        VertexGeminiTranscriptionPart(
+                            inlineData=VertexGeminiTranscriptionInlineData(
+                                mimeType=processed_audio.content_type,
+                                data=base64.b64encode(processed_audio.file_content).decode("utf-8"),
+                            )
+                        ),
+                    ),
+                ),
+            ),
+            generationConfig=VertexGeminiTranscriptionGenerationConfig(
+                audioTranscriptionConfig=_audio_transcription_config(optional_params.get("language"))
+            ),
+        )
+        return AudioTranscriptionRequestData(data=dict(request_body))
+
+    def transform_audio_transcription_response(
+        self,
+        raw_response: Response,
+    ) -> TranscriptionResponse:
+        try:
+            response_json: Final = raw_response.json()
+        except ValueError:
+            raise VertexAIError(
+                status_code=raw_response.status_code,
+                message=f"Received non-JSON response from Vertex AI Gemini transcription: {raw_response.text}",
+            )
+        parsed: Final = VertexGeminiTranscriptionResponse.model_validate(response_json)
+        texts: Final = tuple(
+            part.text
+            for candidate in parsed.candidates
+            if candidate.content is not None
+            for part in candidate.content.parts
+            if part.text
+        )
+        response: Final = TranscriptionResponse(text=" ".join(texts))
+        response["task"] = "transcribe"
+        usage: Final = parsed.usageMetadata
+        if usage is not None:
+            audio_tokens: Final = sum(
+                detail.tokenCount for detail in usage.promptTokensDetails if detail.modality == AUDIO_MODALITY
+            )
+            response.usage = TranscriptionUsageTokensObject(
+                type="tokens",
+                input_tokens=usage.promptTokenCount,
+                output_tokens=usage.candidatesTokenCount,
+                total_tokens=usage.totalTokenCount,
+                input_token_details=TranscriptionUsageInputTokenDetailsObject(
+                    audio_tokens=audio_tokens,
+                    text_tokens=usage.promptTokenCount - audio_tokens,
+                ),
+            )
+        return response
+
+
+def _audio_transcription_config(language: object) -> VertexGeminiTranscriptionAudioConfig:
+    if not isinstance(language, str) or not language:
+        return VertexGeminiTranscriptionAudioConfig()
+    return VertexGeminiTranscriptionAudioConfig(languageCodes=(normalize_transcription_language_to_bcp47(language),))

litellm/llms/vertex_ai/audio_transcription/transformation.py

@@ -35,6 +35,19 @@ SUPPORTED_RESPONSE_FORMATS: Final = ("json", "text")
 _URL_UNSAFE_PROJECT_CHARS: Final = ("/", "?", "#", "\\", ":", " ", "\t", "\n", "\r")
 
 
+def validate_vertex_transcription_location(location: str | None, default_location: str) -> str:
+    try:
+        return validate_vertex_location(location or default_location)
+    except ValueError as e:
+        raise VertexAIError(status_code=400, message=str(e)) from e
+
+
+def validate_vertex_transcription_project_id(project_id: str) -> str:
+    if not project_id or ".." in project_id or any(c in project_id for c in _URL_UNSAFE_PROJECT_CHARS):
+        raise VertexAIError(status_code=400, message=f"Invalid vertex_project format: {project_id!r}")
+    return project_id
+
+
 class VertexAIAudioTranscriptionConfig(BaseAudioTranscriptionConfig, VertexBase):
     def __init__(self) -> None:
         BaseAudioTranscriptionConfig.__init__(self)
@@ -103,27 +116,16 @@ class VertexAIAudioTranscriptionConfig(BaseAudioTranscriptionConfig, VertexBase)
         litellm_params: dict,
         stream: bool | None = None,
     ) -> str:
-        location: Final = self._validate_location(self.safe_get_vertex_ai_location(litellm_params))
-        project_id: Final = self._validate_project_id(
+        location: Final = validate_vertex_transcription_location(
+            self.safe_get_vertex_ai_location(litellm_params), default_location=DEFAULT_SPEECH_TO_TEXT_LOCATION
+        )
+        project_id: Final = validate_vertex_transcription_project_id(
             self.safe_get_vertex_ai_project(litellm_params) or self._resolve_project_id_from_credentials(litellm_params)
         )
         host: Final = "speech.googleapis.com" if location == "global" else f"{location}-speech.googleapis.com"
         base_url: Final = (api_base or f"https://{host}").rstrip("/")
         return f"{base_url}/v2/projects/{project_id}/locations/{location}/recognizers/_:recognize"
 
-    @staticmethod
-    def _validate_location(location: str | None) -> str:
-        try:
-            return validate_vertex_location(location or DEFAULT_SPEECH_TO_TEXT_LOCATION)
-        except ValueError as e:
-            raise VertexAIError(status_code=400, message=str(e)) from e
-
-    @staticmethod
-    def _validate_project_id(project_id: str) -> str:
-        if not project_id or ".." in project_id or any(c in project_id for c in _URL_UNSAFE_PROJECT_CHARS):
-            raise VertexAIError(status_code=400, message=f"Invalid vertex_project format: {project_id!r}")
-        return project_id
-
     def _resolve_project_id_from_credentials(self, litellm_params: dict) -> str:
         _, project_id = self._ensure_access_token(
             credentials=self.safe_get_vertex_ai_credentials(litellm_params),

litellm/llms/vertex_ai/videos/transformation.py

@@ -8,12 +8,14 @@ Based on: https://docs.cloud.google.com/vertex-ai/generative-ai/docs/model-refer
 import base64
 import time
 from collections.abc import Mapping, Sequence
-from typing import TYPE_CHECKING, Any, Final, TypedDict, cast
+from types import MappingProxyType
+from typing import TYPE_CHECKING, Any, ClassVar, Final, TypedDict, cast
 
 import httpx
 from httpx._types import FileContent, RequestFiles
 from typing_extensions import ReadOnly
 
+import litellm
 from litellm.constants import DEFAULT_GOOGLE_VIDEO_DURATION_SECONDS
 from litellm.images.utils import ImageEditRequestUtils
 from litellm.llms.base_llm.videos.transformation import BaseVideoConfig
@@ -119,6 +121,23 @@ class VertexAIVideoConfig(BaseVideoConfig, VertexBase):
     3. Extract video data (base64) from response
     """
 
+    _OPENAI_VIDEO_SIZE_TO_ASPECT_RATIO: ClassVar[Mapping[str, str]] = MappingProxyType(
+        {
+            "1280x720": "16:9",
+            "1920x1080": "16:9",
+            "720x1280": "9:16",
+            "1080x1920": "9:16",
+        }
+    )
+    _OPENAI_VIDEO_SIZE_TO_RESOLUTION: ClassVar[Mapping[str, str]] = MappingProxyType(
+        {
+            "1280x720": "720p",
+            "1920x1080": "1080p",
+            "720x1280": "720p",
+            "1080x1920": "1080p",
+        }
+    )
+
     def __init__(self):
         BaseVideoConfig.__init__(self)
         VertexBase.__init__(self)
@@ -161,6 +180,9 @@ class VertexAIVideoConfig(BaseVideoConfig, VertexBase):
         - prompt → prompt (in instances)
         - input_reference → image (in instances)
         - size → aspectRatio (e.g., "1280x720" → "16:9")
+        - size → resolution for models with resolution-tier pricing when inferable
+          ("1280x720"/"720x1280" → "720p", "1920x1080"/"1080x1920" → "1080p");
+          skipped if ``resolution`` is already set
         - seconds → durationSeconds (defaults to 4 seconds if not provided)
         """
         mapped_params: Final[dict[str, object]] = {}
@@ -175,6 +197,9 @@ class VertexAIVideoConfig(BaseVideoConfig, VertexBase):
         if "parameters" in video_create_optional_params:
             mapped_params["parameters"] = video_create_optional_params["parameters"]
 
+        if "resolution" in video_create_optional_params:
+            mapped_params["resolution"] = video_create_optional_params["resolution"]
+
         # Map size to aspectRatio
         if "size" in video_create_optional_params:
             size: Final = video_create_optional_params["size"]
@@ -182,6 +207,15 @@ class VertexAIVideoConfig(BaseVideoConfig, VertexBase):
                 aspect_ratio: Final = self._convert_size_to_aspect_ratio(size)
                 if aspect_ratio:
                     mapped_params["aspectRatio"] = aspect_ratio
+                nested_params: Final = video_create_optional_params.get("parameters")
+                has_resolution = "resolution" in mapped_params or (
+                    isinstance(nested_params, dict) and nested_params.get("resolution") is not None
+                )
+                supports_resolution = self._supports_resolution_inference(model)
+                if supports_resolution and not has_resolution:
+                    inferred_resolution = self._convert_size_to_resolution(size)
+                    if inferred_resolution is not None:
+                        mapped_params["resolution"] = inferred_resolution
 
         # Map seconds to durationSeconds, default to 4 seconds (matching OpenAI)
         if "seconds" in video_create_optional_params:
@@ -205,14 +239,16 @@ class VertexAIVideoConfig(BaseVideoConfig, VertexBase):
         if not size:
             return None
 
-        aspect_ratio_map: Final = {
-            "1280x720": "16:9",
-            "1920x1080": "16:9",
-            "720x1280": "9:16",
-            "1080x1920": "9:16",
-        }
+        return self._OPENAI_VIDEO_SIZE_TO_ASPECT_RATIO.get(size, "16:9")
 
-        return aspect_ratio_map.get(size, "16:9")
+    def _convert_size_to_resolution(self, size: str) -> str | None:
+        return self._OPENAI_VIDEO_SIZE_TO_RESOLUTION.get(size)
+
+    @staticmethod
+    def _supports_resolution_inference(model: str) -> bool:
+        model_key: Final = model if model.startswith("vertex_ai/") else f"vertex_ai/{model}"
+        model_info: Final = litellm.model_cost.get(model_key)
+        return model_info is not None and model_info.get("output_cost_per_second_1080p") is not None
 
     def validate_environment(
         self,

litellm/main.py

@@ -8612,9 +8612,9 @@ def _joined_streamed_citations(streamed_citations: "tuple[object, ...]") -> "lis
 
 
 def _stream_builder_model_map_cost(response: ModelResponse) -> float | None:
-    model_name: Final = getattr(response, "model", None)
+    model_name: Final = response.model
     usage: Final = getattr(response, "usage", None)
-    if not isinstance(model_name, str) or not model_name or not isinstance(usage, Usage):
+    if not model_name or not isinstance(usage, Usage):
         return None
     try:
         prompt_cost, completion_tokens_cost = litellm.cost_per_token(model=model_name, usage_object=usage)

litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py

@@ -9,7 +9,7 @@ from typing import TYPE_CHECKING, Any, Final, Literal, Optional
 from urllib.parse import parse_qsl, urlencode, urlparse, urlunparse
 
 import httpx
-from fastapi import APIRouter, Form, HTTPException, Request
+from fastapi import APIRouter, Depends, Form, HTTPException, Request
 from fastapi.responses import HTMLResponse, JSONResponse, RedirectResponse, Response
 from pydantic import BaseModel, ConfigDict, Field, SecretStr, ValidationError
 
@@ -46,6 +46,7 @@ from litellm.proxy._experimental.mcp_server.gateway_dcr_flow import (
     aggregate_authorize,
     aggregate_token,
     complete_connect_flow,
+    introspect_gateway_token,
     is_gateway_dcr_client_id,
     is_proxy_api_resource,
     native_client_auth_contract,
@@ -67,6 +68,7 @@ from litellm.proxy._experimental.mcp_server.proxy_api_credentials import (
     mint_proxy_credential,
 )
 from litellm.proxy.auth.ip_address_utils import IPAddressUtils
+from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
 from litellm.proxy.common_utils.encrypt_decrypt_utils import (
     decrypt_value_helper,
     encrypt_value_helper,
@@ -1951,6 +1953,26 @@ async def revoke_endpoint(request: Request, token: str = Form(...), client_id: s
     return await revoke_refresh_token(token=token, client_id=client_id, master_key=master_key, cache=user_api_key_cache)
 
 
+@router.post("/introspect", dependencies=[Depends(user_api_key_auth)])
+async def introspect_endpoint(token: str = Form(...)) -> Response:
+    """RFC 7662 introspection for gateway-issued session tokens (``llm_session_`` /
+    ``llm_srefresh_``), so an external gateway can validate them without the signing
+    secret. The caller authenticates with a LiteLLM virtual key (section 2.1, enforced by
+    the route dependency); any token the gateway cannot vouch for answers
+    ``{"active": false}`` with no further detail."""
+    from litellm.proxy.proxy_server import (  # noqa: PLC0415  # circular import at module load
+        master_key,
+        user_api_key_cache,
+    )
+
+    return await introspect_gateway_token(
+        token=token,
+        master_key=master_key,
+        reload_user=_reload_active_user_by_id,
+        cache=user_api_key_cache,
+    )
+
+
 @router.get("/.well-known/litellm-cli-auth")
 async def native_client_auth_discovery(request: Request) -> JSONResponse:
     """The versioned contract a native client (``lite login --pkce``, or a CLI in any other
@@ -2456,6 +2478,7 @@ def _build_aggregate_authorization_server_response(request: Request) -> dict:
         "issuer": f"{request_base_url}/mcp",
         "authorization_endpoint": f"{request_base_url}/authorize",
         "token_endpoint": f"{request_base_url}/token",
+        "introspection_endpoint": f"{request_base_url}/introspect",
         "registration_endpoint": f"{request_base_url}/register",
         "response_types_supported": ["code"],
         "scopes_supported": [],

litellm/proxy/_experimental/mcp_server/gateway_dcr_flow.py

@@ -70,13 +70,19 @@ from litellm.proxy._experimental.mcp_server.outbound_credentials.session_credent
     open_session_refresh_bearer,
 )
 from litellm.proxy._experimental.mcp_server.outbound_credentials.session_token import (
+    SESSION_ISSUER,
     SESSION_REFRESH_TTL_SECONDS,
     MintedSessionToken,
+    OpenedSessionToken,
     SessionAudience,
     SessionPrincipal,
     SessionSigningKeys,
+    is_session_refresh_token,
+    is_session_token,
     mint_session_refresh_token,
     mint_session_token,
+    open_session_refresh_token,
+    open_session_token,
 )
 from litellm.proxy.common_utils.encrypt_decrypt_utils import (
     decrypt_value_helper,
@@ -885,6 +891,23 @@ class _SingleUseGuard:
         count = await self._cache.async_increment_cache(key, 1, ttl=ttl_seconds, local_only=True)
         return "first" if count == 1 else "replayed"
 
+    async def peek(self, key: str) -> Literal["unclaimed", "claimed", "unavailable"]:
+        """Read-only view of a single-use marker, resolved against the same shared authority as
+        :meth:`claim` so introspection observes exactly the record redemption and revocation wrote.
+        A backend fault is ``"unavailable"`` (fail closed) rather than a guess either way."""
+        from litellm.proxy.proxy_server import redis_usage_cache  # noqa: PLC0415  # circular import at module load
+
+        redis_cache: Final = redis_usage_cache or getattr(self._cache, "redis_cache", None)
+        if redis_cache is not None:
+            try:
+                value = await redis_cache.async_get_cache(key)
+            except Exception as e:  # noqa: BLE001  # ANY Redis fault fails the read closed
+                verbose_logger.warning("mcp gateway single-use peek: shared cache backend unavailable: %s", e)
+                return "unavailable"
+            return "unclaimed" if value is None else "claimed"
+        local: Final = await self._cache.async_get_cache(key, local_only=True)
+        return "unclaimed" if local is None else "claimed"
+
 
 def _session_token_pair(principal: SessionPrincipal, keys: SessionSigningKeys, now: datetime) -> Response:
     access: Final = mint_session_token(principal, keys, now)
@@ -1199,3 +1222,83 @@ async def revoke_refresh_token(token: str, client_id: str, master_key: str | Non
         if burned == "unavailable":
             return _oauth_error(503, "temporarily_unavailable", _CLAIM_UNAVAILABLE_DESCRIPTION)
     return Response(content="{}", media_type="application/json", headers=TOKEN_NO_CACHE_HEADERS)
+
+
+def _inactive_introspection_response() -> Response:
+    """RFC 7662 section 2.2: any token the gateway cannot vouch for, whatever the reason
+    (wrong family, bad signature, expired, revoked, or a deactivated user), answers 200
+    with ``active: false`` and nothing else, so introspection is not a token oracle."""
+    return JSONResponse(status_code=200, content={"active": False}, headers=TOKEN_NO_CACHE_HEADERS)
+
+
+def _active_introspection_response(opened: OpenedSessionToken) -> Response:
+    principal: Final = opened.principal
+    optional_claims: Final = {
+        key: value
+        for key, value in (
+            ("token_type", "Bearer" if opened.kind == "session" else None),
+            ("team_id", principal.team_id),
+            ("resource_server_id", principal.resource_server_id),
+            ("audience", principal.audience),
+        )
+        if value is not None
+    }
+    return JSONResponse(
+        status_code=200,
+        content={
+            "active": True,
+            "iss": SESSION_ISSUER,
+            "sub": principal.user_id,
+            "client_id": principal.client_id,
+            "jti": opened.jti,
+            "iat": opened.iat,
+            "exp": opened.exp,
+            "kind": opened.kind,
+            **optional_claims,
+        },
+        headers=TOKEN_NO_CACHE_HEADERS,
+    )
+
+
+async def introspect_gateway_token(
+    token: str,
+    master_key: str | None,
+    reload_user: ReloadUser,
+    cache: DualCache,
+) -> Response:
+    """RFC 7662 introspection for the gateway's session tokens, so an external gateway
+    (Kong, an API management layer) can validate a LiteLLM-issued MCP session credential
+    without holding the signing secret. The caller is already authenticated by the route
+    (section 2.1). Active means everything admission itself would require: valid signature
+    under the configured session signing keys, unexpired, not a revoked or rotated refresh
+    token, and a litellm user that is still live, so a deactivated user's outstanding
+    tokens introspect as inactive immediately. A shared-backend or DB outage answers 503
+    rather than guessing in either direction."""
+    if master_key is None:
+        verbose_logger.error("mcp_gateway_dcr introspect rejected: no master_key configured")
+        return _oauth_error(500, "server_error", "the gateway has no master key configured")
+    keys: Final = active_session_signing_keys(master_key)
+    if isinstance(keys, SessionSigningConfigError):
+        verbose_logger.error("mcp_gateway_dcr introspect rejected: %s", keys.detail)
+        return _oauth_error(500, "server_error", keys.detail)
+    now: Final = datetime.now(timezone.utc)
+    if is_session_token(token):
+        opened = open_session_token(token, keys, now)
+    elif is_session_refresh_token(token):
+        opened = open_session_refresh_token(token, keys, now)
+    else:
+        return _inactive_introspection_response()
+    if not isinstance(opened, OpenedSessionToken):
+        return _inactive_introspection_response()
+    if opened.kind == "session_refresh":
+        peeked: Final = await _SingleUseGuard(cache).peek(f"{_USED_REFRESH_CACHE_PREFIX}{opened.jti}")
+        if peeked == "unavailable":
+            return _oauth_error(503, "temporarily_unavailable", _CLAIM_UNAVAILABLE_DESCRIPTION)
+        if peeked == "claimed":
+            return _inactive_introspection_response()
+    failure: Final = await reload_user(opened.principal.user_id)
+    if failure == "unavailable":
+        return _oauth_error(503, "temporarily_unavailable", "the gateway database is unavailable; retry")
+    if failure is not None:
+        return _inactive_introspection_response()
+    return _active_introspection_response(opened)

litellm/proxy/_experimental/mcp_server/outbound_credentials/session_token.py

@@ -221,12 +221,17 @@ class MintedSessionToken(BaseModel):
 
 
 class OpenedSessionToken(BaseModel):
-    """A validated session token of either kind: the principal it was minted for, plus the
-    ``jti`` so the token endpoint can enforce single-use rotation on a refresh token."""
+    """A validated session token of either kind: the principal it was minted for, the
+    ``jti`` so the token endpoint can enforce single-use rotation on a refresh token, and
+    the signed ``kind``/``iat``/``exp`` so an introspection response can report the
+    token's metadata without re-decoding."""
 
     model_config = ConfigDict(frozen=True)
     principal: SessionPrincipal
     jti: str
+    kind: SessionTokenKind
+    iat: int
+    exp: int
 
 
 class SessionTokenTooLarge(BaseModel):
@@ -458,6 +463,9 @@ def _open(
             team_id=claims.team_id,
         ),
         jti=claims.jti,
+        kind=claims.kind,
+        iat=claims.iat,
+        exp=claims.exp,
     )
 
 

litellm/proxy/_lazy_features.py

@@ -161,6 +161,7 @@ LAZY_FEATURES: Final[tuple[LazyFeature, ...]] = (
             "/callback",
             "/register",
             "/revoke",
+            "/introspect",
         ),
         # Catches the /{mcp_server_name}/authorize|token|register variants.
         path_suffixes=("/authorize", "/token", "/register"),

litellm/proxy/_lazy_openapi_snapshot.json

@@ -461,6 +461,145 @@
   "access_groups": {
     "components": {
       "schemas": {
+        "AccessGroupBudget": {
+          "properties": {
+            "budget_duration": {
+              "anyOf": [
+                {
+                  "type": "string"
+                },
+                {
+                  "type": "null"
+                }
+              ],
+              "title": "Budget Duration"
+            },
+            "budget_id": {
+              "title": "Budget Id",
+              "type": "string"
+            },
+            "budget_reset_at": {
+              "anyOf": [
+                {
+                  "format": "date-time",
+                  "type": "string"
+                },
+                {
+                  "type": "null"
+                }
+              ],
+              "title": "Budget Reset At"
+            },
+            "max_budget": {
+              "anyOf": [
+                {
+                  "type": "number"
+                },
+                {
+                  "type": "null"
+                }
+              ],
+              "title": "Max Budget"
+            },
+            "soft_budget": {
+              "anyOf": [
+                {
+                  "type": "number"
+                },
+                {
+                  "type": "null"
+                }
+              ],
+              "title": "Soft Budget"
+            }
+          },
+          "required": [
+            "budget_id"
+          ],
+          "title": "AccessGroupBudget",
+          "type": "object"
+        },
+        "AccessGroupBudgetRequest": {
+          "additionalProperties": false,
+          "properties": {
+            "budget_duration": {
+              "anyOf": [
+                {
+                  "type": "string"
+                },
+                {
+                  "type": "null"
+                }
+              ],
+              "title": "Budget Duration"
+            },
+            "budget_id": {
+              "anyOf": [
+                {
+                  "type": "string"
+                },
+                {
+                  "type": "null"
+                }
+              ],
+              "title": "Budget Id"
+            },
+            "max_budget": {
+              "anyOf": [
+                {
+                  "minimum": 0.0,
+                  "type": "number"
+                },
+                {
+                  "type": "null"
+                }
+              ],
+              "title": "Max Budget"
+            },
+            "soft_budget": {
+              "anyOf": [
+                {
+                  "minimum": 0.0,
+                  "type": "number"
+                },
+                {
+                  "type": "null"
+                }
+              ],
+              "title": "Soft Budget"
+            }
+          },
+          "title": "AccessGroupBudgetRequest",
+          "type": "object"
+        },
+        "AccessGroupBudgetResponse": {
+          "properties": {
+            "access_group": {
+              "title": "Access Group",
+              "type": "string"
+            },
+            "budget": {
+              "anyOf": [
+                {
+                  "$ref": "#/components/schemas/AccessGroupBudget"
+                },
+                {
+                  "type": "null"
+                }
+              ]
+            },
+            "spend": {
+              "title": "Spend",
+              "type": "number"
+            }
+          },
+          "required": [
+            "access_group",
+            "spend"
+          ],
+          "title": "AccessGroupBudgetResponse",
+          "type": "object"
+        },
         "AccessGroupCreateRequest": {
           "properties": {
             "access_agent_ids": {
@@ -561,6 +700,16 @@
               "title": "Access Group",
               "type": "string"
             },
+            "budget": {
+              "anyOf": [
+                {
+                  "$ref": "#/components/schemas/AccessGroupBudget"
+                },
+                {
+                  "type": "null"
+                }
+              ]
+            },
             "deployment_count": {
               "title": "Deployment Count",
               "type": "integer"
@@ -571,6 +720,17 @@
               },
               "title": "Model Names",
               "type": "array"
+            },
+            "spend": {
+              "anyOf": [
+                {
+                  "type": "number"
+                },
+                {
+                  "type": "null"
+                }
+              ],
+              "title": "Spend"
             }
           },
           "required": [
@@ -782,6 +942,29 @@
           "title": "AccessGroupUpdateRequest",
           "type": "object"
         },
+        "DeleteAccessGroupBudgetResponse": {
+          "properties": {
+            "access_group": {
+              "title": "Access Group",
+              "type": "string"
+            },
+            "budget_deleted": {
+              "title": "Budget Deleted",
+              "type": "boolean"
+            },
+            "message": {
+              "title": "Message",
+              "type": "string"
+            }
+          },
+          "required": [
+            "access_group",
+            "budget_deleted",
+            "message"
+          ],
+          "title": "DeleteAccessGroupBudgetResponse",
+          "type": "object"
+        },
         "DeleteModelGroupResponse": {
           "properties": {
             "access_group": {
@@ -1072,6 +1255,156 @@
           ]
         }
       },
+      "/access_group/{access_group}/budget": {
+        "delete": {
+          "description": "Clear the shared budget of an access group, leaving the group itself in place.\n\nExample:\n```bash\ncurl -X DELETE 'http://localhost:4000/access_group/production-models/budget' \\\n  -H 'Authorization: Bearer sk-1234'\n```\n\nParameters:\n- access_group: str - The access group name (URL path parameter)\n\nReturns:\n- DeleteAccessGroupBudgetResponse; budget_deleted is false when there was nothing to clear\n\nRaises:\n- HTTPException 404: If access group not found",
+          "operationId": "delete_access_group_budget_access_group__access_group__budget_delete",
+          "parameters": [
+            {
+              "in": "path",
+              "name": "access_group",
+              "required": true,
+              "schema": {
+                "title": "Access Group",
+                "type": "string"
+              }
+            }
+          ],
+          "responses": {
+            "200": {
+              "content": {
+                "application/json": {
+                  "schema": {
+                    "$ref": "#/components/schemas/DeleteAccessGroupBudgetResponse"
+                  }
+                }
+              },
+              "description": "Successful Response"
+            },
+            "422": {
+              "content": {
+                "application/json": {
+                  "schema": {
+                    "$ref": "#/components/schemas/HTTPValidationError"
+                  }
+                }
+              },
+              "description": "Validation Error"
+            }
+          },
+          "security": [
+            {
+              "APIKeyHeader": []
+            }
+          ],
+          "summary": "Delete Access Group Budget",
+          "tags": [
+            "access_groups"
+          ]
+        },
+        "get": {
+          "description": "Get the shared budget of an access group, and the spend drawn against it.\n\nExample:\n```bash\ncurl -X GET 'http://localhost:4000/access_group/production-models/budget' \\\n  -H 'Authorization: Bearer sk-1234'\n```\n\nParameters:\n- access_group: str - The access group name (URL path parameter)\n\nReturns:\n- AccessGroupBudgetResponse; budget is null when the group has no budget set\n\nRaises:\n- HTTPException 404: If access group not found",
+          "operationId": "get_access_group_budget_access_group__access_group__budget_get",
+          "parameters": [
+            {
+              "in": "path",
+              "name": "access_group",
+              "required": true,
+              "schema": {
+                "title": "Access Group",
+                "type": "string"
+              }
+            }
+          ],
+          "responses": {
+            "200": {
+              "content": {
+                "application/json": {
+                  "schema": {
+                    "$ref": "#/components/schemas/AccessGroupBudgetResponse"
+                  }
+                }
+              },
+              "description": "Successful Response"
+            },
+            "422": {
+              "content": {
+                "application/json": {
+                  "schema": {
+                    "$ref": "#/components/schemas/HTTPValidationError"
+                  }
+                }
+              },
+              "description": "Validation Error"
+            }
+          },
+          "security": [
+            {
+              "APIKeyHeader": []
+            }
+          ],
+          "summary": "Get Access Group Budget",
+          "tags": [
+            "access_groups"
+          ]
+        },
+        "put": {
+          "description": "Set or replace the shared budget of an access group. Idempotent.\n\nEvery key that can reach a model in the group draws from this one budget.\n\nExample:\n```bash\ncurl -X PUT 'http://localhost:4000/access_group/production-models/budget' \\\n  -H 'Authorization: Bearer sk-1234' \\\n  -H 'Content-Type: application/json' \\\n  -d '{\n    \"max_budget\": 100.0,\n    \"budget_duration\": \"30d\"\n  }'\n```\n\nParameters:\n- access_group: str - The access group name (URL path parameter)\n- max_budget: Optional[float] - Requests fail once the group's shared spend exceeds this\n- soft_budget: Optional[float] - Fires an alert when reached; requests still succeed\n- budget_duration: Optional[str] - Frequency of resetting the group's spend (e.g. '30d')\n- budget_id: Optional[str] - Link an existing budget instead of creating one\n\nReturns:\n- AccessGroupBudgetResponse with the stored budget and current spend\n\nRaises:\n- HTTPException 400: If no budget field is given, or budget_duration cannot be parsed\n- HTTPException 404: If access group not found",
+          "operationId": "set_access_group_budget_access_group__access_group__budget_put",
+          "parameters": [
+            {
+              "in": "path",
+              "name": "access_group",
+              "required": true,
+              "schema": {
+                "title": "Access Group",
+                "type": "string"
+              }
+            }
+          ],
+          "requestBody": {
+            "content": {
+              "application/json": {
+                "schema": {
+                  "$ref": "#/components/schemas/AccessGroupBudgetRequest"
+                }
+              }
+            },
+            "required": true
+          },
+          "responses": {
+            "200": {
+              "content": {
+                "application/json": {
+                  "schema": {
+                    "$ref": "#/components/schemas/AccessGroupBudgetResponse"
+                  }
+                }
+              },
+              "description": "Successful Response"
+            },
+            "422": {
+              "content": {
+                "application/json": {
+                  "schema": {
+                    "$ref": "#/components/schemas/HTTPValidationError"
+                  }
+                }
+              },
+              "description": "Validation Error"
+            }
+          },
+          "security": [
+            {
+              "APIKeyHeader": []
+            }
+          ],
+          "summary": "Set Access Group Budget",
+          "tags": [
+            "access_groups"
+          ]
+        }
+      },
       "/access_group/{access_group}/delete": {
         "delete": {
           "description": "Delete an access group.\n\nRemoves the access group from all deployments that have it.\n\nExample:\n```bash\ncurl -X DELETE 'http://localhost:4000/access_group/production-models/delete' \\\n  -H 'Authorization: Bearer sk-1234'\n```\n\nParameters:\n- access_group: str - The access group name (URL path parameter)\n\nReturns:\n- DeleteModelGroupResponse with deletion details\n\nRaises:\n- HTTPException 404: If access group not found",
@@ -1122,7 +1455,7 @@
       },
       "/access_group/{access_group}/info": {
         "get": {
-          "description": "Get information about a specific access group.\n\nExample:\n```bash\ncurl -X GET 'http://localhost:4000/access_group/production-models/info' \\\n  -H 'Authorization: Bearer sk-1234'\n```\n\nParameters:\n- access_group: str - The access group name (URL path parameter)\n\nReturns:\n- AccessGroupInfo with the access group details\n\nRaises:\n- HTTPException 404: If access group not found",
+          "description": "Get information about a specific access group.\n\nExample:\n```bash\ncurl -X GET 'http://localhost:4000/access_group/production-models/info' \\\n  -H 'Authorization: Bearer sk-1234'\n```\n\nParameters:\n- access_group: str - The access group name (URL path parameter)\n\nReturns:\n- AccessGroupInfo with the access group details, its shared budget and its spend\n\nRaises:\n- HTTPException 404: If access group not found",
           "operationId": "get_access_group_info_access_group__access_group__info_get",
           "parameters": [
             {
@@ -6459,6 +6792,109 @@
           "title": "ConfigOverrideSettingsResponse",
           "type": "object"
         },
+        "CyberArkConfig": {
+          "description": "Configuration for CyberArk Conjur secret manager integration.",
+          "properties": {
+            "client_cert": {
+              "anyOf": [
+                {
+                  "type": "string"
+                },
+                {
+                  "type": "null"
+                }
+              ],
+              "description": "Path to the client TLS certificate for certificate-based authentication",
+              "title": "Client Cert"
+            },
+            "client_key": {
+              "anyOf": [
+                {
+                  "type": "string"
+                },
+                {
+                  "type": "null"
+                }
+              ],
+              "description": "Path to the client TLS private key for certificate-based authentication",
+              "title": "Client Key"
+            },
+            "cyberark_account": {
+              "anyOf": [
+                {
+                  "type": "string"
+                },
+                {
+                  "type": "null"
+                }
+              ],
+              "description": "The Conjur organization account name",
+              "title": "Cyberark Account"
+            },
+            "cyberark_api_base": {
+              "anyOf": [
+                {
+                  "type": "string"
+                },
+                {
+                  "type": "null"
+                }
+              ],
+              "description": "The address of the CyberArk Conjur server (e.g., https://conjur.example.com)",
+              "title": "Cyberark Api Base"
+            },
+            "cyberark_api_key": {
+              "anyOf": [
+                {
+                  "type": "string"
+                },
+                {
+                  "type": "null"
+                }
+              ],
+              "description": "API key for Conjur API-key authentication",
+              "title": "Cyberark Api Key"
+            },
+            "cyberark_username": {
+              "anyOf": [
+                {
+                  "type": "string"
+                },
+                {
+                  "type": "null"
+                }
+              ],
+              "description": "The Conjur username (login) to authenticate as",
+              "title": "Cyberark Username"
+            },
+            "refresh_interval": {
+              "anyOf": [
+                {
+                  "type": "string"
+                },
+                {
+                  "type": "null"
+                }
+              ],
+              "description": "Auth token cache TTL in seconds (default: 300)",
+              "title": "Refresh Interval"
+            },
+            "ssl_verify": {
+              "anyOf": [
+                {
+                  "type": "string"
+                },
+                {
+                  "type": "null"
+                }
+              ],
+              "description": "Set to false to disable SSL verification (e.g., for self-signed certificates)",
+              "title": "Ssl Verify"
+            }
+          },
+          "title": "CyberArkConfig",
+          "type": "object"
+        },
         "HTTPValidationError": {
           "properties": {
             "detail": {
@@ -6654,6 +7090,192 @@
       }
     },
     "paths": {
+      "/config_overrides/cyberark": {
+        "delete": {
+          "description": "Delete CyberArk Conjur configuration. Idempotent.",
+          "operationId": "delete_cyberark_config_config_overrides_cyberark_delete",
+          "parameters": [
+            {
+              "description": "The litellm-changed-by header enables tracking of actions performed by authorized users on behalf of other users, providing an audit trail for accountability",
+              "in": "header",
+              "name": "litellm-changed-by",
+              "required": false,
+              "schema": {
+                "anyOf": [
+                  {
+                    "type": "string"
+                  },
+                  {
+                    "type": "null"
+                  }
+                ],
+                "description": "The litellm-changed-by header enables tracking of actions performed by authorized users on behalf of other users, providing an audit trail for accountability",
+                "title": "Litellm-Changed-By"
+              }
+            }
+          ],
+          "responses": {
+            "200": {
+              "content": {
+                "application/json": {
+                  "schema": {
+                    "additionalProperties": {
+                      "type": "string"
+                    },
+                    "title": "Response Delete Cyberark Config Config Overrides Cyberark Delete",
+                    "type": "object"
+                  }
+                }
+              },
+              "description": "Successful Response"
+            },
+            "422": {
+              "content": {
+                "application/json": {
+                  "schema": {
+                    "$ref": "#/components/schemas/HTTPValidationError"
+                  }
+                }
+              },
+              "description": "Validation Error"
+            }
+          },
+          "security": [
+            {
+              "APIKeyHeader": []
+            }
+          ],
+          "summary": "Delete Cyberark Config",
+          "tags": [
+            "config_overrides"
+          ]
+        },
+        "get": {
+          "description": "Get current CyberArk Conjur configuration.\nReturns decrypted values from DB, or falls back to current env vars.\nSensitive fields are masked before leaving the server.",
+          "operationId": "get_cyberark_config_config_overrides_cyberark_get",
+          "responses": {
+            "200": {
+              "content": {
+                "application/json": {
+                  "schema": {
+                    "$ref": "#/components/schemas/ConfigOverrideSettingsResponse"
+                  }
+                }
+              },
+              "description": "Successful Response"
+            }
+          },
+          "security": [
+            {
+              "APIKeyHeader": []
+            }
+          ],
+          "summary": "Get Cyberark Config",
+          "tags": [
+            "config_overrides"
+          ]
+        },
+        "post": {
+          "description": "Update CyberArk Conjur secret manager configuration.\nSets environment variables, encrypts sensitive fields, and stores in DB.\nReinitializes the secret manager on this pod.",
+          "operationId": "update_cyberark_config_config_overrides_cyberark_post",
+          "parameters": [
+            {
+              "description": "The litellm-changed-by header enables tracking of actions performed by authorized users on behalf of other users, providing an audit trail for accountability",
+              "in": "header",
+              "name": "litellm-changed-by",
+              "required": false,
+              "schema": {
+                "anyOf": [
+                  {
+                    "type": "string"
+                  },
+                  {
+                    "type": "null"
+                  }
+                ],
+                "description": "The litellm-changed-by header enables tracking of actions performed by authorized users on behalf of other users, providing an audit trail for accountability",
+                "title": "Litellm-Changed-By"
+              }
+            }
+          ],
+          "requestBody": {
+            "content": {
+              "application/json": {
+                "schema": {
+                  "$ref": "#/components/schemas/CyberArkConfig"
+                }
+              }
+            },
+            "required": true
+          },
+          "responses": {
+            "200": {
+              "content": {
+                "application/json": {
+                  "schema": {
+                    "additionalProperties": {
+                      "type": "string"
+                    },
+                    "title": "Response Update Cyberark Config Config Overrides Cyberark Post",
+                    "type": "object"
+                  }
+                }
+              },
+              "description": "Successful Response"
+            },
+            "422": {
+              "content": {
+                "application/json": {
+                  "schema": {
+                    "$ref": "#/components/schemas/HTTPValidationError"
+                  }
+                }
+              },
+              "description": "Validation Error"
+            }
+          },
+          "security": [
+            {
+              "APIKeyHeader": []
+            }
+          ],
+          "summary": "Update Cyberark Config",
+          "tags": [
+            "config_overrides"
+          ]
+        }
+      },
+      "/config_overrides/cyberark/test_connection": {
+        "post": {
+          "description": "Test the connection to the currently configured CyberArk Conjur server.\nUses the already-initialized secret manager client. Does not modify any state.",
+          "operationId": "test_cyberark_connection_config_overrides_cyberark_test_connection_post",
+          "responses": {
+            "200": {
+              "content": {
+                "application/json": {
+                  "schema": {
+                    "additionalProperties": {
+                      "type": "string"
+                    },
+                    "title": "Response Test Cyberark Connection Config Overrides Cyberark Test Connection Post",
+                    "type": "object"
+                  }
+                }
+              },
+              "description": "Successful Response"
+            }
+          },
+          "security": [
+            {
+              "APIKeyHeader": []
+            }
+          ],
+          "summary": "Test Cyberark Connection",
+          "tags": [
+            "config_overrides"
+          ]
+        }
+      },
       "/config_overrides/hashicorp_vault": {
         "delete": {
           "description": "Delete Hashicorp Vault configuration. Idempotent.",
@@ -16555,6 +17177,19 @@
           "title": "Body_authorize_complete_authorize_complete_post",
           "type": "object"
         },
+        "Body_introspect_endpoint_introspect_post": {
+          "properties": {
+            "token": {
+              "title": "Token",
+              "type": "string"
+            }
+          },
+          "required": [
+            "token"
+          ],
+          "title": "Body_introspect_endpoint_introspect_post",
+          "type": "object"
+        },
         "Body_revoke_endpoint_revoke_post": {
           "properties": {
             "client_id": {
@@ -19134,6 +19769,51 @@
           ]
         }
       },
+      "/introspect": {
+        "post": {
+          "description": "RFC 7662 introspection for gateway-issued session tokens (``llm_session_`` /\n``llm_srefresh_``), so an external gateway can validate them without the signing\nsecret. The caller authenticates with a LiteLLM virtual key (section 2.1, enforced by\nthe route dependency); any token the gateway cannot vouch for answers\n``{\"active\": false}`` with no further detail.",
+          "operationId": "introspect_endpoint_introspect_post",
+          "requestBody": {
+            "content": {
+              "application/x-www-form-urlencoded": {
+                "schema": {
+                  "$ref": "#/components/schemas/Body_introspect_endpoint_introspect_post"
+                }
+              }
+            },
+            "required": true
+          },
+          "responses": {
+            "200": {
+              "content": {
+                "application/json": {
+                  "schema": {}
+                }
+              },
+              "description": "Successful Response"
+            },
+            "422": {
+              "content": {
+                "application/json": {
+                  "schema": {
+                    "$ref": "#/components/schemas/HTTPValidationError"
+                  }
+                }
+              },
+              "description": "Validation Error"
+            }
+          },
+          "security": [
+            {
+              "APIKeyHeader": []
+            }
+          ],
+          "summary": "Introspect Endpoint",
+          "tags": [
+            "mcp_discoverable"
+          ]
+        }
+      },
       "/register": {
         "post": {
           "operationId": "register_client_register_post",

litellm/proxy/_types.py

@@ -241,6 +241,7 @@ class Litellm_EntityType(enum.Enum):
     PROJECT = "project"
     TAG = "tag"
     AGENT = "agent"
+    MODEL_ACCESS_GROUP = "model_access_group"
 
     # global proxy level entity
     PROXY = "proxy"
@@ -504,6 +505,7 @@ class LiteLLMRoutes(enum.Enum):
         "/mcp-rest/tools/list",
         "/mcp-rest/tools/call",
         "/v1/mcp/tools",
+        "/introspect",
     ]
 
     # MCP server CRUD routes — control-plane. Gated by DISABLE_ADMIN_ENDPOINTS.
@@ -2886,6 +2888,7 @@ class UserAPIKeyAuth(LiteLLM_VerificationTokenView):  # the expected response ob
         ),
     )
     budget_reservation: dict[str, Any] | None = Field(default=None, exclude=True)
+    matched_model_access_groups: list[str] | None = Field(default=None, exclude=True)
     budget_throttle_pct: float | None = Field(default=None, exclude=True)
     user: Any | None = None  # Expanded user object when expand=user is used
     created_by_user: Any | None = None  # Expanded created_by user when expand=user is used
@@ -3573,6 +3576,8 @@ class SpendLogsMetadata(TypedDict):
     status: StandardLoggingPayloadStatus
     proxy_server_request: str | None
     batch_models: list[str] | None
+    batch_successful_requests: int | None  # writable-ok: built by assignment like every sibling key in this TypedDict
+    batch_failed_requests: int | None  # writable-ok: built by assignment like every sibling key in this TypedDict
     error_information: StandardLoggingPayloadErrorInformation | None
     usage_object: dict | None
     model_map_information: StandardLoggingModelInformation | None
@@ -4918,6 +4923,7 @@ class DBSpendUpdateTransactions(TypedDict):
     org_list_transactions: dict[str, float] | None
     tag_list_transactions: dict[str, float] | None
     agent_list_transactions: dict[str, float] | None
+    model_access_group_list_transactions: ReadOnly[dict[str, float] | None]
 
 
 class SpendUpdateQueueItem(TypedDict, total=False):

litellm/proxy/anthropic_endpoints/endpoints.py

@@ -22,6 +22,7 @@ from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
 from litellm.proxy.common_request_processing import (
     ProxyBaseLLMRequestProcessing,
     create_response,
+    proxy_exception_from_http_exception,
 )
 from litellm.proxy.common_utils.http_parsing_utils import _read_request_body
 from litellm.types.utils import TokenCountResponse
@@ -214,6 +215,9 @@ async def anthropic_response(
             litellm_logging_obj=None,
         )
 
+        if isinstance(e, HTTPException):
+            raise proxy_exception_from_http_exception(e, headers)
+
         error_msg: Final = f"{e}"
         raise ProxyException(
             message=getattr(e, "message", error_msg),

litellm/proxy/auth/auth_checks.py

@@ -32,6 +32,7 @@ from litellm.constants import (
     DEFAULT_MAX_RECURSE_DEPTH,
     EMAIL_BUDGET_ALERT_MAX_SPEND_ALERT_PERCENTAGE,
     END_USER_RESTRICTED_REGISTRY_MAX_SIZE,
+    MODEL_ACCESS_GROUP_REGISTRY_MAX_SIZE,
     REGISTRY_ERROR_NEGATIVE_CACHE_TTL,
     TAG_REGISTRY_MAX_SIZE,
 )
@@ -78,11 +79,15 @@ from litellm.proxy.common_utils.http_parsing_utils import (
 from litellm.proxy.common_utils.timezone_utils import get_budget_reset_time
 from litellm.proxy.common_utils.user_api_key_cache import (
     END_USER_RESTRICTED_REGISTRY_OVERFLOW_SENTINEL,
+    MODEL_ACCESS_GROUP_REGISTRY_OVERFLOW_SENTINEL,
     TAG_REGISTRY_OVERFLOW_SENTINEL,
     UserApiKeyCache,
     end_user_cache_key,
     end_user_restricted_registry_cache_key,
     get_management_object_ttl,
+    model_access_group_cache_key,
+    model_access_group_registry_cache_key,
+    model_access_group_spend_counter_key,
     object_permission_cache_key,
     tag_cache_key,
     tag_registry_cache_key,
@@ -107,12 +112,14 @@ from litellm.repositories.table_repositories import (
     EndUserRepository,
     JWTKeyMappingRepository,
     ManagedVectorStoresRepository,
+    ModelAccessGroupBudgetRepository,
     TagRepository,
     TeamMembershipRepository,
 )
 from litellm.repositories.team_repository import TeamRepository
 from litellm.repositories.user_repository import UserRepository
 from litellm.router import Router
+from litellm.types.proxy.model_access_group_budget import ModelAccessGroupBudget
 from litellm.utils import get_utc_datetime
 
 from .auth_checks_organization import (
@@ -251,6 +258,43 @@ def _end_user_table(repo: _PrismaTableHolder[_PrismaEndUserRow]) -> _PrismaAuthT
     return repo.table
 
 
+class _PrismaMaxBudgetRow(Protocol):
+    @property
+    def max_budget(self) -> float | None: ...
+
+
+class _PrismaModelAccessGroupBudgetRow(Protocol):
+    access_group_name: str
+
+    @property
+    def spend(self) -> float | None: ...
+
+    @property
+    def litellm_budget_table(self) -> _PrismaMaxBudgetRow | None: ...
+
+
+def _model_access_group_budget_table(
+    repo: _PrismaTableHolder[_PrismaModelAccessGroupBudgetRow],
+) -> _PrismaAuthTable[_PrismaModelAccessGroupBudgetRow]:
+    return repo.table
+
+
+class _MemberModelScope(Protocol):
+    @property
+    def allowed_models(self) -> Sequence[str] | None: ...
+
+
+class _TeamMembershipModelScope(Protocol):
+    @property
+    def litellm_budget_table(self) -> _MemberModelScope | None: ...
+
+
+def _member_allowed_models(membership: _TeamMembershipModelScope) -> Sequence[str]:
+    """The member's own model scope, read through a narrowed view of the membership row."""
+    budget_table: Final = membership.litellm_budget_table
+    return () if budget_table is None else (budget_table.allowed_models or ())
+
+
 class _RawCacheRead(Protocol):
     async def async_get_cache(self, *, key: str) -> object: ...
 
@@ -807,6 +851,7 @@ async def common_checks(
     1.1. If project is blocked
     2. If team can call model
     2.2 If project can call model
+    2.3 Which model access groups authorized this request
     3. If team is in budget
     3.0.2. If project is in budget
     3.0.3. If project is over soft budget (alert only)
@@ -925,6 +970,18 @@ async def common_checks(
             proxy_logging_obj=proxy_logging_obj,
         )
 
+    # 2.3 Which model access groups authorized this request
+    matched_model_access_groups: Final = await stamp_matched_model_access_groups(
+        model=_model,
+        valid_token=valid_token,
+        team_object=team_object,
+        project_object=project_object,
+        llm_router=llm_router,
+        prisma_client=prisma_client,
+        user_api_key_cache=user_api_key_cache,
+        proxy_logging_obj=proxy_logging_obj,
+    )
+
     # Run before apply_key_tags_pre_auth injects key metadata.tags into request_body.
     _reject_clientside_metadata_tags_check(general_settings, request_body, route)
 
@@ -1004,6 +1061,13 @@ async def common_checks(
                     proxy_logging_obj=proxy_logging_obj,
                     valid_token=valid_token,
                 ),
+                _model_access_group_max_budget_check(
+                    matched_model_access_groups=matched_model_access_groups,
+                    prisma_client=prisma_client,
+                    user_api_key_cache=user_api_key_cache,
+                )
+                if matched_model_access_groups
+                else None,
                 _user_max_budget_check(),
                 _check_team_member_budget(
                     team_object=team_object,
@@ -1444,6 +1508,7 @@ _REGISTRY_NOT_CACHED: Final = _RegistryNotCached()
 #: One lock per registry; module-level because the stampede to collapse is worker-wide.
 _TAG_REGISTRY_LOAD_LOCK: Final = asyncio.Lock()
 _END_USER_REGISTRY_LOAD_LOCK: Final = asyncio.Lock()
+_MODEL_ACCESS_GROUP_REGISTRY_LOAD_LOCK: Final = asyncio.Lock()
 
 
 async def _cached_registry(
@@ -1836,6 +1901,105 @@ async def _load_tag_registry(
     )
 
 
+async def _load_model_access_group_registry(
+    prisma_client: PrismaClient,
+    user_api_key_cache: UserApiKeyCache,
+) -> frozenset[str] | None:
+    """The set of model access group names that have a row in ``LiteLLM_ModelAccessGroupBudgetTable``."""
+
+    async def fetch_ids() -> tuple[str, ...]:
+        registry_rows: Final = await _model_access_group_budget_table(
+            ModelAccessGroupBudgetRepository(prisma_client)
+        ).find_many(take=MODEL_ACCESS_GROUP_REGISTRY_MAX_SIZE + 1)
+        return tuple(row.access_group_name for row in registry_rows)
+
+    return await _load_bounded_registry(
+        cache_key=model_access_group_registry_cache_key(),
+        overflow_sentinel=MODEL_ACCESS_GROUP_REGISTRY_OVERFLOW_SENTINEL,
+        max_size=MODEL_ACCESS_GROUP_REGISTRY_MAX_SIZE,
+        load_lock=_MODEL_ACCESS_GROUP_REGISTRY_LOAD_LOCK,
+        fetch_ids=fetch_ids,
+        user_api_key_cache=user_api_key_cache,
+    )
+
+
+async def _fetch_uncached_model_access_group_budgets(
+    uncached_groups: Sequence[str],
+    prisma_client: PrismaClient,
+    user_api_key_cache: UserApiKeyCache,
+) -> tuple[tuple[str, ModelAccessGroupBudget], ...]:
+    """Budget rows for the groups a cache probe missed.
+
+    No registry gate here, unlike the tag path: the names only ever come from
+    ``matched_model_access_groups``, which :func:`collect_matched_model_access_groups` already
+    intersected with the registry, so a name that has no row cannot reach this.
+    """
+    if not uncached_groups:
+        return ()
+
+    try:
+        db_rows: Final = await _model_access_group_budget_table(
+            ModelAccessGroupBudgetRepository(prisma_client)
+        ).find_many(
+            where={"access_group_name": {"in": list(uncached_groups)}},
+            include={"litellm_budget_table": True},
+        )
+        fetched: Final = tuple((row.access_group_name, _model_access_group_budget(row)) for row in db_rows)
+        for fetched_name, fetched_obj in fetched:
+            await user_api_key_cache.async_set_cache(
+                key=model_access_group_cache_key(fetched_name),
+                value=fetched_obj,
+                model_type=ModelAccessGroupBudget,
+                ttl=get_management_object_ttl(user_api_key_cache),
+            )
+    except Exception as e:  # noqa: BLE001  # fail-safe: a budget fetch error must yield "no budget rows", never break auth
+        verbose_proxy_logger.debug("Error batch fetching model access group budgets from database: %s", e)
+        return ()
+    else:
+        return fetched
+
+
+def _model_access_group_budget(row: _PrismaModelAccessGroupBudgetRow) -> ModelAccessGroupBudget:
+    budget_table: Final = row.litellm_budget_table
+    return ModelAccessGroupBudget(
+        access_group_name=row.access_group_name,
+        spend=row.spend or 0.0,
+        max_budget=None if budget_table is None else budget_table.max_budget,
+    )
+
+
+@log_db_metrics
+async def get_model_access_group_budgets_batch(
+    access_group_names: Sequence[str],
+    prisma_client: PrismaClient | None,
+    user_api_key_cache: UserApiKeyCache,
+) -> dict[str, ModelAccessGroupBudget]:
+    """Budget rows for the given model access groups, served from cache where possible.
+
+    Shared by the two enforcement paths so they read one row per group per request: the
+    reservation counters when reservations are on, and :func:`_model_access_group_max_budget_check`
+    when ``disable_budget_reservation`` turns them off.
+    """
+    if prisma_client is None or not access_group_names:
+        return {}
+
+    probed: Final = [
+        (
+            group,
+            await user_api_key_cache.async_get_cache(
+                key=model_access_group_cache_key(group), model_type=ModelAccessGroupBudget
+            ),
+        )
+        for group in access_group_names
+    ]
+    fetched: Final = await _fetch_uncached_model_access_group_budgets(
+        uncached_groups=tuple(group for group, budget in probed if budget is None),
+        prisma_client=prisma_client,
+        user_api_key_cache=user_api_key_cache,
+    )
+    return {group: budget for group, budget in (*probed, *fetched) if budget is not None}
+
+
 async def _fetch_uncached_tags(
     uncached_tags: Sequence[str],
     prisma_client: PrismaClient,
@@ -3882,6 +4046,192 @@ def _resolve_key_models_for_auth_check(valid_token: UserAPIKeyAuth) -> list[str]
     return models
 
 
+def _model_access_groups_serving_model(
+    model: str | Sequence[str],
+    llm_router: Router,
+    team_id: str | None,
+) -> frozenset[str]:
+    """Every model access group whose deployments serve the requested model(s)."""
+    requested: Final = (model,) if isinstance(model, str) else tuple(model)
+    return frozenset(
+        group
+        for requested_model in requested
+        for group in llm_router.get_model_access_groups(model_name=requested_model, team_id=team_id)
+    )
+
+
+async def _team_member_granted_models(
+    valid_token: UserAPIKeyAuth,
+    team_object: LiteLLM_TeamTable | None,
+    prisma_client: PrismaClient,
+    user_api_key_cache: UserApiKeyCache,
+    proxy_logging_obj: ProxyLogging,
+) -> Sequence[str]:
+    """The member's own ``allowed_models`` scope; empty when the member is not narrowed below the team."""
+    if team_object is None or valid_token.user_id is None:
+        return ()
+
+    team_membership: Final = await get_team_membership(
+        user_id=valid_token.user_id,
+        team_id=team_object.team_id,
+        prisma_client=prisma_client,
+        user_api_key_cache=user_api_key_cache,
+        proxy_logging_obj=proxy_logging_obj,
+    )
+    return () if team_membership is None else _member_allowed_models(team_membership)
+
+
+async def _org_granted_models(
+    valid_token: UserAPIKeyAuth,
+    team_object: LiteLLM_TeamTable | None,
+    prisma_client: PrismaClient,
+    user_api_key_cache: UserApiKeyCache,
+    proxy_logging_obj: ProxyLogging,
+) -> Sequence[str]:
+    """The org allowlist reached through the key, or through its team when the key names no org."""
+    org_id: Final = valid_token.org_id or (team_object.organization_id if team_object is not None else None)
+    if org_id is None:
+        return ()
+
+    try:
+        org_object: Final = await get_org_object(
+            org_id=org_id,
+            prisma_client=prisma_client,
+            user_api_key_cache=user_api_key_cache,
+            proxy_logging_obj=proxy_logging_obj,
+        )
+    except Exception as e:  # noqa: BLE001  # fail-safe: attribution degrades to "no org grant", it must never break auth
+        verbose_proxy_logger.debug("access group attribution: org lookup failed: %s", e)
+        return ()
+    return org_object.models if org_object is not None else ()
+
+
+async def _granted_model_lists(
+    valid_token: UserAPIKeyAuth,
+    team_object: LiteLLM_TeamTable | None,
+    project_object: LiteLLM_ProjectTableCachedObj | None,
+    prisma_client: PrismaClient,
+    user_api_key_cache: UserApiKeyCache,
+    proxy_logging_obj: ProxyLogging,
+) -> tuple[Sequence[str], ...]:
+    """One model allowlist per level that participates in authorizing the request."""
+    return (
+        _resolve_key_models_for_auth_check(valid_token=valid_token),
+        team_object.models if team_object is not None else (),
+        await _team_member_granted_models(
+            valid_token=valid_token,
+            team_object=team_object,
+            prisma_client=prisma_client,
+            user_api_key_cache=user_api_key_cache,
+            proxy_logging_obj=proxy_logging_obj,
+        ),
+        project_object.models if project_object is not None else (),
+        await _org_granted_models(
+            valid_token=valid_token,
+            team_object=team_object,
+            prisma_client=prisma_client,
+            user_api_key_cache=user_api_key_cache,
+            proxy_logging_obj=proxy_logging_obj,
+        ),
+    )
+
+
+async def collect_matched_model_access_groups(
+    model: str | Sequence[str] | None,
+    valid_token: UserAPIKeyAuth | None,
+    team_object: LiteLLM_TeamTable | None,
+    project_object: LiteLLM_ProjectTableCachedObj | None,
+    llm_router: Router | None,
+    prisma_client: PrismaClient | None,
+    user_api_key_cache: UserApiKeyCache,
+    proxy_logging_obj: ProxyLogging,
+) -> tuple[str, ...]:
+    """
+    The budgeted model access groups that authorized this request, sorted and deduplicated.
+
+    A group is charged only when its name appears on an allowlist the caller was granted -- key,
+    team, team-member scope, project or org -- *and* that group serves the requested model. Asking
+    for a model that merely belongs to a group attributes nothing, because nothing about the caller
+    named the group.
+
+    Levels are unioned, never ranked: a team granted ``*`` whose member is scoped to one group is
+    still a caller gated by that group. An unrestricted allowlist (empty, ``*``) names no group and
+    so contributes nothing.
+
+    The whole walk is gated on the budget registry, because collecting every match costs a full scan
+    of each allowlist where the plain access check stops at the first hit. An empty registry means no
+    group carries a budget, so there is nothing to attribute and no work worth doing.
+    """
+    if model is None or valid_token is None or llm_router is None or prisma_client is None:
+        return ()
+
+    registry: Final = await _load_model_access_group_registry(
+        prisma_client=prisma_client,
+        user_api_key_cache=user_api_key_cache,
+    )
+    if registry is not None and not registry:
+        return ()
+
+    covering_groups: Final = _model_access_groups_serving_model(
+        model=model,
+        llm_router=llm_router,
+        team_id=valid_token.team_id,
+    )
+    budgeted_groups: Final = covering_groups if registry is None else covering_groups & registry
+    if not budgeted_groups:
+        return ()
+
+    granted: Final = frozenset(
+        granted_model
+        for granted_models in await _granted_model_lists(
+            valid_token=valid_token,
+            team_object=team_object,
+            project_object=project_object,
+            prisma_client=prisma_client,
+            user_api_key_cache=user_api_key_cache,
+            proxy_logging_obj=proxy_logging_obj,
+        )
+        for granted_model in granted_models
+    )
+    return tuple(sorted(budgeted_groups & granted))
+
+
+async def stamp_matched_model_access_groups(
+    model: str | Sequence[str] | None,
+    valid_token: UserAPIKeyAuth | None,
+    team_object: LiteLLM_TeamTable | None,
+    project_object: LiteLLM_ProjectTableCachedObj | None,
+    llm_router: Router | None,
+    prisma_client: PrismaClient | None,
+    user_api_key_cache: UserApiKeyCache,
+    proxy_logging_obj: ProxyLogging,
+) -> tuple[str, ...]:
+    """Record the groups that authorized this request on its auth object, for the post-call spend
+    writer and the reservation counters, and hand them back for the budget check."""
+    if valid_token is None:
+        return ()
+
+    try:
+        matched: Final = await collect_matched_model_access_groups(
+            model=model,
+            valid_token=valid_token,
+            team_object=team_object,
+            project_object=project_object,
+            llm_router=llm_router,
+            prisma_client=prisma_client,
+            user_api_key_cache=user_api_key_cache,
+            proxy_logging_obj=proxy_logging_obj,
+        )
+    except Exception as e:  # noqa: BLE001  # fail-safe: attribution is spend telemetry, it must never break auth
+        verbose_proxy_logger.debug("model access group attribution failed: %s", e)
+        return ()
+    if not matched:
+        return ()
+    matched_groups: Final = list(matched)  # mutable-ok: the auth field is typed list[str] | None
+    valid_token.matched_model_access_groups = matched_groups  # rebind-ok: request-scoped carrier for the writer
+    return matched
+
+
 async def can_key_call_model(
     model: str | list[str],
     llm_model_list: list | None,
@@ -4476,6 +4826,7 @@ async def _virtual_key_multi_budget_check(
             max_budget=w["max_budget"],
             window_entity_type="Key",
             window_entity_id=valid_token.token,
+            window_duration=str(w["budget_duration"]),
             window_start=get_budget_window_start(w),
         )
         if math.isfinite(w["max_budget"]) and window_spend >= w["max_budget"]:
@@ -4849,6 +5200,7 @@ async def _team_multi_budget_check(
             max_budget=w["max_budget"],
             window_entity_type="Team",
             window_entity_id=team_object.team_id,
+            window_duration=str(w["budget_duration"]),
             window_start=get_budget_window_start(w),
         )
         if math.isfinite(w["max_budget"]) and window_spend >= w["max_budget"]:
@@ -5256,6 +5608,61 @@ async def _tag_max_budget_check(
             )
 
 
+async def _model_access_group_max_budget_check(
+    matched_model_access_groups: Sequence[str],
+    prisma_client: PrismaClient | None,
+    user_api_key_cache: UserApiKeyCache,
+) -> None:
+    """Block the request when a model access group that authorized it is over its max budget.
+
+    Only the groups auth already matched are charged and therefore only they are checked, so a
+    request that no budgeted group authorized costs nothing here.
+
+    Like the tag check this is a plain read with no reservation, so concurrent requests can
+    overshoot the ceiling slightly. The reservation counters are the precise path; this one covers
+    the ``disable_budget_reservation`` case.
+
+    The ceiling is exclusive, unlike the tag check it otherwise mirrors: a pool whose recorded
+    spend has reached ``max_budget`` has nothing left to give, so the next request is refused.
+    Keys and organizations already draw the line there. A non-positive budget means no budget,
+    matching what the reservation path treats as unbudgeted.
+
+    Raises:
+        BudgetExceededError if a matched group is over its max budget.
+    """
+    if prisma_client is None or not matched_model_access_groups:
+        return
+
+    budgets: Final = await get_model_access_group_budgets_batch(
+        access_group_names=matched_model_access_groups,
+        prisma_client=prisma_client,
+        user_api_key_cache=user_api_key_cache,
+    )
+
+    from litellm.proxy.proxy_server import get_current_spend
+
+    for group in matched_model_access_groups:
+        budget = budgets.get(group)
+        if budget is None or budget.max_budget is None or budget.max_budget <= 0:
+            continue
+
+        group_spend = await get_current_spend(
+            counter_key=model_access_group_spend_counter_key(group),
+            fallback_spend=budget.spend,
+            max_budget=budget.max_budget,
+            fallback_authoritative=True,
+        )
+        if group_spend < budget.max_budget:
+            continue
+        raise litellm.BudgetExceededError(
+            current_cost=group_spend,
+            max_budget=budget.max_budget,
+            message=f"Budget has been exceeded! Model access group={group} Current cost: {group_spend}, Max budget: {budget.max_budget}",
+            entity_type=Litellm_EntityType.MODEL_ACCESS_GROUP.value,
+            entity_id=group,
+        )
+
+
 def is_model_allowed_by_pattern(model: str, allowed_model_pattern: str) -> bool:
     """
     Check if a model matches an allowed pattern.

litellm/proxy/client/chat.py

@@ -8,16 +8,19 @@ from .exceptions import UnauthorizedError
 
 
 class ChatClient:
-    def __init__(self, base_url: str, api_key: str | None = None):
+    def __init__(self, base_url: str, api_key: str | None = None, timeout: int = 600):
         """
         Initialize the ChatClient.
 
         Args:
             base_url (str): The base URL of the LiteLLM proxy server (e.g., "http://localhost:8000")
             api_key (Optional[str]): API key for authentication. If provided, it will be sent as a Bearer token.
+            timeout (int): Request timeout in seconds (default: 600, the OpenAI SDK default, since a completion
+                can legitimately take minutes)
         """
         self._base_url = base_url.rstrip("/")  # Remove trailing slash if present
         self._api_key = api_key
+        self._timeout = timeout
 
     def _get_headers(self) -> dict[str, str]:
         """
@@ -96,7 +99,7 @@ class ChatClient:
         # Prepare and send the request
         session: Final = requests.Session()
         try:
-            response: Final = session.send(request.prepare())
+            response: Final = session.send(request.prepare(), timeout=self._timeout)
             response.raise_for_status()
             return response.json()
         except requests.exceptions.HTTPError as e:
@@ -161,7 +164,9 @@ class ChatClient:
         # Make streaming request
         session: Final = requests.Session()
         try:
-            response: Final = session.post(url, headers=self._get_headers(), json=data, stream=True)
+            response: Final = session.post(
+                url, headers=self._get_headers(), json=data, stream=True, timeout=self._timeout
+            )
             response.raise_for_status()
 
             # Parse SSE stream

litellm/proxy/client/cli/commands/auth.py

@@ -99,11 +99,6 @@ class CliPollData(TypedDict, total=False):
     team_id: str
 
 
-class CliPollRequestKwargs(TypedDict, total=False):
-    timeout: int
-    headers: dict[str, str]
-
-
 class CliSsoStartData(TypedDict):
     login_id: str
     poll_secret: str
@@ -518,10 +513,7 @@ def _poll_for_ready_data(
 ) -> CliPollData | None:
     for attempt in range(total_timeout // poll_interval):
         try:
-            request_kwargs: CliPollRequestKwargs = {"timeout": request_timeout}
-            if headers is not None:
-                request_kwargs["headers"] = headers
-            response = requests.get(url, **request_kwargs)
+            response = requests.get(url, headers=headers, timeout=request_timeout)
             if response.status_code == 200:
                 data: CliPollData = response.json()
                 status = data.get("status")

litellm/proxy/client/client.py

@@ -24,7 +24,8 @@ class Client:
         Args:
             base_url (str): The base URL of the LiteLLM proxy server (e.g., "http://localhost:4000")
             api_key (Optional[str]): API key for authentication. If provided, it will be sent as a Bearer token.
-            timeout: Request timeout in seconds (default: 30)
+            timeout: Request timeout in seconds for management calls (default: 30). Chat completions keep
+                ChatClient's own 600 second default, since a completion can legitimately take minutes
         """
         self._base_url = base_url.rstrip("/")
         # Only use the stored CLI key when it was issued for this server.
@@ -33,9 +34,9 @@ class Client:
         # Initialize resource clients
 
         self.http = HTTPClient(base_url=base_url, api_key=self._api_key, timeout=timeout)
-        self.models = ModelsManagementClient(base_url=self._base_url, api_key=self._api_key)
-        self.model_groups = ModelGroupsManagementClient(base_url=self._base_url, api_key=self._api_key)
+        self.models = ModelsManagementClient(base_url=self._base_url, api_key=self._api_key, timeout=timeout)
+        self.model_groups = ModelGroupsManagementClient(base_url=self._base_url, api_key=self._api_key, timeout=timeout)
         self.chat = ChatClient(base_url=self._base_url, api_key=self._api_key)
-        self.keys = KeysManagementClient(base_url=self._base_url, api_key=self._api_key)
-        self.credentials = CredentialsManagementClient(base_url=self._base_url, api_key=self._api_key)
-        self.teams = TeamsManagementClient(base_url=self._base_url, api_key=self._api_key)
+        self.keys = KeysManagementClient(base_url=self._base_url, api_key=self._api_key, timeout=timeout)
+        self.credentials = CredentialsManagementClient(base_url=self._base_url, api_key=self._api_key, timeout=timeout)
+        self.teams = TeamsManagementClient(base_url=self._base_url, api_key=self._api_key, timeout=timeout)

litellm/proxy/client/credentials.py

@@ -6,16 +6,18 @@ from .exceptions import UnauthorizedError
 
 
 class CredentialsManagementClient:
-    def __init__(self, base_url: str, api_key: str | None = None):
+    def __init__(self, base_url: str, api_key: str | None = None, timeout: int = 30):
         """
         Initialize the CredentialsManagementClient.
 
         Args:
             base_url (str): The base URL of the LiteLLM proxy server (e.g., "http://localhost:8000")
             api_key (Optional[str]): API key for authentication. If provided, it will be sent as a Bearer token.
+            timeout (int): Request timeout in seconds (default: 30)
         """
         self._base_url = base_url.rstrip("/")  # Remove trailing slash if present
         self._api_key = api_key
+        self._timeout = timeout
 
     def _get_headers(self) -> dict[str, str]:
         """
@@ -56,7 +58,7 @@ class CredentialsManagementClient:
 
         session: Final = requests.Session()
         try:
-            response: Final = session.send(request.prepare())
+            response: Final = session.send(request.prepare(), timeout=self._timeout)
             response.raise_for_status()
             return response.json()
         except requests.exceptions.HTTPError as e:
@@ -103,7 +105,7 @@ class CredentialsManagementClient:
 
         session: Final = requests.Session()
         try:
-            response: Final = session.send(request.prepare())
+            response: Final = session.send(request.prepare(), timeout=self._timeout)
             response.raise_for_status()
             return response.json()
         except requests.exceptions.HTTPError as e:
@@ -140,7 +142,7 @@ class CredentialsManagementClient:
 
         session: Final = requests.Session()
         try:
-            response: Final = session.send(request.prepare())
+            response: Final = session.send(request.prepare(), timeout=self._timeout)
             response.raise_for_status()
             return response.json()
         except requests.exceptions.HTTPError as e:
@@ -177,7 +179,7 @@ class CredentialsManagementClient:
 
         session: Final = requests.Session()
         try:
-            response: Final = session.send(request.prepare())
+            response: Final = session.send(request.prepare(), timeout=self._timeout)
             response.raise_for_status()
             return response.json()
         except requests.exceptions.HTTPError as e:

litellm/proxy/client/keys.py

@@ -9,16 +9,18 @@ from .exceptions import UnauthorizedError
 
 
 class KeysManagementClient:
-    def __init__(self, base_url: str, api_key: str | None = None):
+    def __init__(self, base_url: str, api_key: str | None = None, timeout: int = 30):
         """
         Initialize the KeysManagementClient.
 
         Args:
             base_url (str): The base URL of the LiteLLM proxy server (e.g., "http://localhost:8000")
             api_key (Optional[str]): API key for authentication. If provided, it will be sent as a Bearer token.
+            timeout (int): Request timeout in seconds (default: 30)
         """
         self._base_url = base_url.rstrip("/")  # Remove trailing slash if present
         self._api_key = api_key
+        self._timeout = timeout
 
     def _get_headers(self) -> dict[str, str]:
         """
@@ -99,7 +101,7 @@ class KeysManagementClient:
 
         session: Final = requests.Session()
         try:
-            response: Final = session.send(request.prepare())
+            response: Final = session.send(request.prepare(), timeout=self._timeout)
             response.raise_for_status()
             return response.json()
         except requests.exceptions.HTTPError as e:
@@ -174,7 +176,7 @@ class KeysManagementClient:
 
         session: Final = requests.Session()
         try:
-            response: Final = session.send(request.prepare())
+            response: Final = session.send(request.prepare(), timeout=self._timeout)
             response.raise_for_status()
             return response.json()
         except requests.exceptions.HTTPError as e:
@@ -218,7 +220,7 @@ class KeysManagementClient:
 
         session: Final = requests.Session()
         try:
-            response: Final = session.send(request.prepare())
+            response: Final = session.send(request.prepare(), timeout=self._timeout)
             response.raise_for_status()
             return response.json()
         except requests.exceptions.HTTPError as e:
@@ -279,7 +281,7 @@ class KeysManagementClient:
         session: Final = requests.Session()
         response_text: str | None = None
         try:
-            response: Final = session.send(request.prepare())
+            response: Final = session.send(request.prepare(), timeout=self._timeout)
             response_text = response.text
             response.raise_for_status()
             return response.json()
@@ -309,7 +311,7 @@ class KeysManagementClient:
 
         session: Final = requests.Session()
         try:
-            response: Final = session.send(request.prepare())
+            response: Final = session.send(request.prepare(), timeout=self._timeout)
             response.raise_for_status()
             return response.json()
         except requests.exceptions.HTTPError as e:

litellm/proxy/client/model_groups.py

@@ -6,16 +6,18 @@ from .exceptions import UnauthorizedError
 
 
 class ModelGroupsManagementClient:
-    def __init__(self, base_url: str, api_key: str | None = None):
+    def __init__(self, base_url: str, api_key: str | None = None, timeout: int = 30):
         """
         Initialize the ModelGroupsManagementClient.
 
         Args:
             base_url (str): The base URL of the LiteLLM proxy server (e.g., "http://localhost:8000")
             api_key (Optional[str]): API key for authentication. If provided, it will be sent as a Bearer token.
+            timeout (int): Request timeout in seconds (default: 30)
         """
         self._base_url = base_url.rstrip("/")  # Remove trailing slash if present
         self._api_key = api_key
+        self._timeout = timeout
 
     def _get_headers(self) -> dict[str, str]:
         """
@@ -53,7 +55,7 @@ class ModelGroupsManagementClient:
         # Prepare and send the request
         session: Final = requests.Session()
         try:
-            response: Final = session.send(request.prepare())
+            response: Final = session.send(request.prepare(), timeout=self._timeout)
             response.raise_for_status()
             return response.json()["data"]
         except requests.exceptions.HTTPError as e:

litellm/proxy/client/models.py

@@ -7,16 +7,18 @@ from .exceptions import NotFoundError, UnauthorizedError
 
 
 class ModelsManagementClient:
-    def __init__(self, base_url: str, api_key: str | None = None):
+    def __init__(self, base_url: str, api_key: str | None = None, timeout: int = 30):
         """
         Initialize the ModelsManagementClient.
 
         Args:
             base_url (str): The base URL of the LiteLLM proxy server (e.g., "http://localhost:8000")
             api_key (Optional[str]): API key for authentication. If provided, it will be sent as a Bearer token.
+            timeout (int): Request timeout in seconds (default: 30)
         """
         self._base_url = base_url.rstrip("/")  # Remove trailing slash if present
         self._api_key = api_key
+        self._timeout = timeout
 
     def _get_headers(self) -> dict[str, str]:
         """
@@ -55,7 +57,7 @@ class ModelsManagementClient:
         # Prepare and send the request
         session: Final = requests.Session()
         try:
-            response: Final = session.send(request.prepare())
+            response: Final = session.send(request.prepare(), timeout=self._timeout)
             response.raise_for_status()
             return response.json()["data"]
         except requests.exceptions.HTTPError as e:
@@ -104,7 +106,7 @@ class ModelsManagementClient:
         # Prepare and send the request
         session: Final = requests.Session()
         try:
-            response: Final = session.send(request.prepare())
+            response: Final = session.send(request.prepare(), timeout=self._timeout)
             response.raise_for_status()
             return response.json()
         except requests.exceptions.HTTPError as e:
@@ -140,7 +142,7 @@ class ModelsManagementClient:
         # Prepare and send the request
         session: Final = requests.Session()
         try:
-            response: Final = session.send(request.prepare())
+            response: Final = session.send(request.prepare(), timeout=self._timeout)
             response.raise_for_status()
             return response.json()
         except requests.exceptions.HTTPError as e:
@@ -232,7 +234,7 @@ class ModelsManagementClient:
         # Prepare and send the request
         session: Final = requests.Session()
         try:
-            response: Final = session.send(request.prepare())
+            response: Final = session.send(request.prepare(), timeout=self._timeout)
             response.raise_for_status()
             return response.json()["data"]
         except requests.exceptions.HTTPError as e:
@@ -282,7 +284,7 @@ class ModelsManagementClient:
         # Prepare and send the request
         session: Final = requests.Session()
         try:
-            response: Final = session.send(request.prepare())
+            response: Final = session.send(request.prepare(), timeout=self._timeout)
             response.raise_for_status()
             return response.json()
         except requests.exceptions.HTTPError as e:

litellm/proxy/client/teams.py

@@ -11,16 +11,18 @@ from .exceptions import UnauthorizedError
 class TeamsManagementClient:
     """Client for managing teams in LiteLLM proxy."""
 
-    def __init__(self, base_url: str, api_key: str | None = None):
+    def __init__(self, base_url: str, api_key: str | None = None, timeout: int = 30):
         """
         Initialize the TeamsManagementClient.
 
         Args:
             base_url (str): The base URL of the LiteLLM proxy server (e.g., "http://localhost:4000")
             api_key (Optional[str]): API key for authentication. If provided, it will be sent as a Bearer token.
+            timeout (int): Request timeout in seconds (default: 30)
         """
         self._base_url = base_url.rstrip("/")  # Remove trailing slash if present
         self._api_key = api_key
+        self._timeout = timeout
 
     def _get_headers(self) -> dict[str, str]:
         """
@@ -60,7 +62,7 @@ class TeamsManagementClient:
         if organization_id:
             params["organization_id"] = organization_id
 
-        response: Final = requests.get(url, headers=self._get_headers(), params=params)
+        response: Final = requests.get(url, headers=self._get_headers(), params=params, timeout=self._timeout)
 
         if response.status_code == 401:
             raise UnauthorizedError("Authentication failed. Check your API key.")
@@ -117,7 +119,7 @@ class TeamsManagementClient:
         if sort_by:
             params["sort_by"] = sort_by
 
-        response: Final = requests.get(url, headers=self._get_headers(), params=params)
+        response: Final = requests.get(url, headers=self._get_headers(), params=params, timeout=self._timeout)
 
         if response.status_code == 401:
             raise UnauthorizedError("Authentication failed. Check your API key.")
@@ -138,7 +140,7 @@ class TeamsManagementClient:
         """
         url: Final = f"{self._base_url}/team/available"
 
-        response: Final = requests.get(url, headers=self._get_headers())
+        response: Final = requests.get(url, headers=self._get_headers(), timeout=self._timeout)
 
         if response.status_code == 401:
             raise UnauthorizedError("Authentication failed. Check your API key.")

litellm/proxy/client/users.py

@@ -6,9 +6,10 @@ from .exceptions import NotFoundError, UnauthorizedError
 
 
 class UsersManagementClient:
-    def __init__(self, base_url: str, api_key: str | None = None):
+    def __init__(self, base_url: str, api_key: str | None = None, timeout: int = 30):
         self.base_url = base_url.rstrip("/")
         self.api_key = api_key
+        self.timeout = timeout
 
     def _get_headers(self) -> dict[str, str]:
         headers: Final = {"Content-Type": "application/json"}
@@ -19,7 +20,7 @@ class UsersManagementClient:
     def list_users(self, params: dict[str, Any] | None = None) -> list[dict[str, Any]]:
         """List users (GET /user/list)"""
         url: Final = f"{self.base_url}/user/list"
-        response: Final = requests.get(url, headers=self._get_headers(), params=params)
+        response: Final = requests.get(url, headers=self._get_headers(), params=params, timeout=self.timeout)
         if response.status_code == 401:
             raise UnauthorizedError(response.text)
         response.raise_for_status()
@@ -29,7 +30,7 @@ class UsersManagementClient:
         """Get user info (GET /user/info)"""
         url: Final = f"{self.base_url}/user/info"
         params: Final = {"user_id": user_id} if user_id else {}
-        response: Final = requests.get(url, headers=self._get_headers(), params=params)
+        response: Final = requests.get(url, headers=self._get_headers(), params=params, timeout=self.timeout)
         if response.status_code == 401:
             raise UnauthorizedError(response.text)
         if response.status_code == 404:
@@ -41,7 +42,7 @@ class UsersManagementClient:
         """Get user info v2 - lightweight, returns only user object (GET /v2/user/info)"""
         url: Final = f"{self.base_url}/v2/user/info"
         params: Final = {"user_id": user_id} if user_id else {}
-        response: Final = requests.get(url, headers=self._get_headers(), params=params)
+        response: Final = requests.get(url, headers=self._get_headers(), params=params, timeout=self.timeout)
         if response.status_code == 401:
             raise UnauthorizedError(response.text)
         if response.status_code == 404:
@@ -52,7 +53,7 @@ class UsersManagementClient:
     def create_user(self, user_data: dict[str, Any]) -> dict[str, Any]:
         """Create a new user (POST /user/new)"""
         url: Final = f"{self.base_url}/user/new"
-        response: Final = requests.post(url, headers=self._get_headers(), json=user_data)
+        response: Final = requests.post(url, headers=self._get_headers(), json=user_data, timeout=self.timeout)
         if response.status_code == 401:
             raise UnauthorizedError(response.text)
         response.raise_for_status()
@@ -61,7 +62,9 @@ class UsersManagementClient:
     def delete_user(self, user_ids: list[str]) -> dict[str, Any]:
         """Delete users (POST /user/delete)"""
         url: Final = f"{self.base_url}/user/delete"
-        response: Final = requests.post(url, headers=self._get_headers(), json={"user_ids": user_ids})
+        response: Final = requests.post(
+            url, headers=self._get_headers(), json={"user_ids": user_ids}, timeout=self.timeout
+        )
         if response.status_code == 401:
             raise UnauthorizedError(response.text)
         response.raise_for_status()

litellm/proxy/common_request_processing.py

@@ -533,6 +533,21 @@ def _serialize_http_exception_detail(
     return str(detail), None
 
 
+def proxy_exception_from_http_exception(exc: HTTPException, headers: dict[str, str]) -> ProxyException:
+    raw_detail: Final = _getattr_object(exc, "detail", str(exc))
+    message, structured_fields = _serialize_http_exception_detail(raw_detail)
+    existing_fields: Final = getattr(exc, "provider_specific_fields", None) or {}
+    merged_fields: Final = {**existing_fields, **structured_fields} if structured_fields else (existing_fields or None)
+    return ProxyException(
+        message=message,
+        type=getattr(exc, "type", "None"),
+        param=getattr(exc, "param", "None"),
+        code=getattr(exc, "status_code", status.HTTP_400_BAD_REQUEST),
+        provider_specific_fields=merged_fields,
+        headers=headers,
+    )
+
+
 def _collect_response_file_search_vector_store_ids(data: Mapping[str, object]) -> set[str]:
     vector_store_ids: Final[set[str]] = set()
     tools: Final = data.get("tools")
@@ -1362,6 +1377,8 @@ def _classifier_cost_from_request_data(request_data: Mapping[str, object] | None
     routes and in `metadata` on chat-style routes, so both buckets are consulted, in the same
     precedence `get_or_create_metadata_bucket` writes them.
     """
+    from litellm.proxy.spend_tracking.savings import classifier_cost_from_decision
+
     data: Final = request_data or {}
     for metadata_key in ("litellm_metadata", "metadata"):
         metadata = data.get(metadata_key)
@@ -1370,10 +1387,10 @@ def _classifier_cost_from_request_data(request_data: Mapping[str, object] | None
         decision = metadata.get("routing_decision")
         if not isinstance(decision, dict):
             continue
-        cost = decision.get("classifier_cost")
-        if isinstance(cost, bool) or not isinstance(cost, (int, float)):
+        cost = classifier_cost_from_decision(decision)
+        if cost is None:
             continue
-        return float(cost)
+        return cost
     return None
 
 
@@ -3244,21 +3261,7 @@ class ProxyBaseLLMRequestProcessing:
             raise e
 
         if isinstance(e, HTTPException):
-            raw_detail: Final = _getattr_object(e, "detail", str(e))
-            message, structured_fields = _serialize_http_exception_detail(raw_detail)
-            existing_fields: Final = getattr(e, "provider_specific_fields", None) or {}
-            if structured_fields:
-                merged_fields: dict | None = {**existing_fields, **structured_fields}
-            else:
-                merged_fields = existing_fields or None
-            raise ProxyException(
-                message=message,
-                type=getattr(e, "type", "None"),
-                param=getattr(e, "param", "None"),
-                code=getattr(e, "status_code", status.HTTP_400_BAD_REQUEST),
-                provider_specific_fields=merged_fields,
-                headers=safe_headers,
-            )
+            raise proxy_exception_from_http_exception(e, safe_headers)
         elif isinstance(e, httpx.HTTPStatusError):
             # Handle httpx.HTTPStatusError - extract actual error from response
             # This matches the original behavior before the refactor in commit 511d435f6f

litellm/proxy/common_utils/reset_budget_job.py

@@ -4,7 +4,7 @@ import math
 import time
 from collections.abc import Awaitable, Callable, Iterable, Mapping, Sequence
 from dataclasses import dataclass, field
-from datetime import datetime, timezone
+from datetime import datetime, timedelta, timezone
 from enum import Enum
 from types import MappingProxyType
 from typing import Final, Literal, Protocol, TypeVar, assert_never
@@ -20,10 +20,12 @@ from litellm.constants import (
     RESET_BUDGET_JOB_MAX_CHUNKS_PER_RUN,
     RESET_BUDGET_JOB_NAME,
 )
+from litellm.litellm_core_utils.duration_parser import duration_in_seconds
 from litellm.proxy._types import (
     DB_RETRY_SAFE_ERROR_TYPES,
     LiteLLM_BudgetTableFull,
     LiteLLM_EndUserTable,
+    Litellm_EntityType,
     LiteLLM_TeamTable,
     LiteLLM_UserTable,
     LiteLLM_VerificationToken,
@@ -33,7 +35,12 @@ from litellm.proxy.common_utils.timezone_utils import (
     compute_budget_reset_at,
     get_budget_reset_settings,
 )
-from litellm.proxy.common_utils.user_api_key_cache import tag_cache_key
+from litellm.proxy.common_utils.user_api_key_cache import (
+    model_access_group_cache_key,
+    model_access_group_spend_counter_key,
+    tag_cache_key,
+)
+from litellm.proxy.db.budget_window_spend_writer import roll_window_spend_row
 from litellm.proxy.db.db_transaction_queue.pod_lock_manager import PodLockManager
 from litellm.proxy.db.exception_handler import call_with_db_reconnect_retry
 from litellm.proxy.utils import PrismaClient, ProxyLogging
@@ -41,6 +48,7 @@ from litellm.repositories.organization_repository import OrganizationRepository
 from litellm.repositories.prisma_protocols import SpendLinkedTable
 from litellm.repositories.table_repositories import (
     EndUserRepository,
+    ModelAccessGroupBudgetRepository,
     TagRepository,
     TeamMembershipRepository,
 )
@@ -92,6 +100,11 @@ class _TagRow(_BudgetLinkedRow, Protocol):
     def tag_name(self) -> str: ...
 
 
+class _ModelAccessGroupRow(_BudgetLinkedRow, Protocol):
+    @property
+    def access_group_name(self) -> str: ...
+
+
 class _EndUserRow(_BudgetLinkedRow, Protocol):
     @property
     def user_id(self) -> str: ...
@@ -154,6 +167,14 @@ def _tag_cache_keys(row: _TagRow) -> tuple[str, ...]:
     return (tag_cache_key(row.tag_name),)
 
 
+def _model_access_group_counter_key(row: _ModelAccessGroupRow) -> str:
+    return model_access_group_spend_counter_key(row.access_group_name)
+
+
+def _model_access_group_cache_keys(row: _ModelAccessGroupRow) -> tuple[str, ...]:
+    return (model_access_group_cache_key(row.access_group_name),)
+
+
 def _budget_link_where(
     budget_ids: Sequence[str],
     extra: Mapping[str, object] = MappingProxyType({}),
@@ -329,6 +350,7 @@ class _WindowSource:
 
     table: str
     id_column: str
+    entity_type: Litellm_EntityType
     counter_prefix: str
     log_subject: str
     retry_subject: str
@@ -353,6 +375,7 @@ _WINDOW_SOURCES: Final[tuple[_WindowSource, ...]] = (
     _WindowSource(
         table="LiteLLM_VerificationToken",
         id_column="token",
+        entity_type=Litellm_EntityType.KEY,
         counter_prefix="spend:key",
         log_subject="keys",
         retry_subject="key",
@@ -361,6 +384,7 @@ _WINDOW_SOURCES: Final[tuple[_WindowSource, ...]] = (
     _WindowSource(
         table="LiteLLM_TeamTable",
         id_column="team_id",
+        entity_type=Litellm_EntityType.TEAM,
         counter_prefix="spend:team",
         log_subject="teams",
         retry_subject="team",
@@ -610,6 +634,11 @@ class ResetBudgetJob:
             where=_budget_link_where(budget_ids, _SPENT_ROWS_WHERE),
             log_subject="tags",
         )
+        model_access_groups: Final[tuple[_ModelAccessGroupRow, ...]] = await self._fetch_linked_rows(
+            table=ModelAccessGroupBudgetRepository(self.prisma_client).table,
+            where=_budget_link_where(budget_ids, _SPENT_ROWS_WHERE),
+            log_subject="model access groups",
+        )
         rollover_caps: Final[Mapping[str, float]] = MappingProxyType(
             {  # mutable-ok: MappingProxyType wraps a one-shot dict comprehension
                 b.budget_id: cap
@@ -639,6 +668,10 @@ class ResetBudgetJob:
                 *((_key_counter_key(row), _row_carried_spend(row, rollover_caps)) for row in keys),
                 *((_org_counter_key(row), _row_carried_spend(row, rollover_caps)) for row in orgs),
                 *((_tag_counter_key(row), _row_carried_spend(row, rollover_caps)) for row in tags),
+                *(
+                    (_model_access_group_counter_key(row), _row_carried_spend(row, rollover_caps))
+                    for row in model_access_groups
+                ),
             ),
             rollover_caps=rollover_caps,
             cache_keys=(
@@ -646,6 +679,7 @@ class ResetBudgetJob:
                 *(key for row in keys for key in _key_cache_keys(row)),
                 *(key for row in orgs for key in _org_cache_keys(row)),
                 *(key for row in tags for key in _tag_cache_keys(row)),
+                *(key for row in model_access_groups for key in _model_access_group_cache_keys(row)),
             ),
         )
 
@@ -671,6 +705,7 @@ class ResetBudgetJob:
             _queue_budget_linked_resets(uow.keys, cascade, extra=_LINKED_KEYS_WHERE)
             _queue_budget_linked_resets(uow.organizations, cascade, extra=_SPENT_ROWS_WHERE)
             _queue_budget_linked_resets(uow.tags, cascade, extra=_SPENT_ROWS_WHERE)
+            _queue_budget_linked_resets(uow.model_access_groups, cascade, extra=_SPENT_ROWS_WHERE)
             _queue_enduser_resets(uow.endusers, cascade)
             for budget_id, budget_reset_at in cascade.budget_resets:
                 uow.budgets.queue_window_advance(budget_id=budget_id, budget_reset_at=budget_reset_at)
@@ -714,7 +749,8 @@ class ResetBudgetJob:
     async def reset_budget_for_litellm_budget_table(self) -> None:
         """
         Resets the spend a budget tier gates (end users, team members, keys,
-        orgs, tags) and advances the tier's budget_reset_at, atomically.
+        orgs, tags, model access groups) and advances the tier's
+        budget_reset_at, atomically.
 
         Caches are invalidated only after the transaction commits, so a failed
         run cannot leave a zeroed counter in front of an un-reset DB row.
@@ -745,8 +781,9 @@ class ResetBudgetJob:
                 return _ChunkOutcome(fetched=len(cascade.budgets), advanced=advanced)
             case _BudgetCascadeFailed(cascade=cascade, error=error):
                 verbose_proxy_logger.exception(
-                    "Failed to reset the budget table cascade (team member, enduser, org and tag spend, plus "
-                    "budget_reset_at); nothing was committed and the budgets stay due for the next run: %s",
+                    "Failed to reset the budget table cascade (team member, enduser, org, tag and model access "
+                    "group spend, plus budget_reset_at); nothing was committed and the budgets stay due for the "
+                    "next run: %s",
                     error,
                     exc_info=error,
                 )
@@ -1210,6 +1247,9 @@ class ResetBudgetJob:
         spend_counter_cache: DualCache,
         now: datetime,
         reset_settings: BudgetResetSettings,
+        prisma_client: PrismaClient,
+        entity_type: Litellm_EntityType,
+        entity_id: str,
     ) -> bool:
         """Reset a single budget window if expired. Returns True if the window was reset."""
         reset_at_str: Final = window.get("reset_at")
@@ -1225,11 +1265,56 @@ class ResetBudgetJob:
                 await spend_counter_cache.redis_cache.async_set_cache(key=counter_key, value=new_value)
             except Exception as redis_err:
                 verbose_proxy_logger.warning("Failed to reset Redis counter %s: %s", counter_key, redis_err)
-        window["reset_at"] = compute_budget_reset_at(
-            budget_duration=window["budget_duration"], settings=reset_settings
-        ).isoformat()
+        budget_duration: Final = window["budget_duration"]
+        next_reset_at: Final = compute_budget_reset_at(budget_duration=budget_duration, settings=reset_settings)
+        window["reset_at"] = next_reset_at.isoformat()
+        await ResetBudgetJob._roll_window_spend_row(
+            prisma_client=prisma_client,
+            entity_type=entity_type,
+            entity_id=entity_id,
+            budget_duration=budget_duration,
+            next_reset_at=next_reset_at,
+        )
         return True
 
+    @staticmethod
+    async def _roll_window_spend_row(
+        prisma_client: PrismaClient,
+        entity_type: Litellm_EntityType,
+        entity_id: str,
+        budget_duration: str,
+        next_reset_at: datetime,
+    ) -> None:
+        """Move this window's LiteLLM_BudgetWindowSpend row onto the window
+        that just started, so the maintained total the read path uses starts
+        from zero alongside the counter.
+
+        Best effort: the row is an optimization over aggregating
+        LiteLLM_SpendLogs, so a failure here must not stop the remaining
+        windows from having their counters reset.
+        """
+        try:
+            window_start: Final = next_reset_at - timedelta(seconds=duration_in_seconds(budget_duration))
+        except Exception as e:  # noqa: BLE001  # duration_in_seconds raises bare exceptions on bad input
+            verbose_proxy_logger.warning("Unparseable budget_duration %s: %s", budget_duration, e)
+            return
+        try:
+            await roll_window_spend_row(
+                prisma_client=prisma_client,
+                entity_type=entity_type.value,
+                entity_id=entity_id,
+                window_duration=budget_duration,
+                new_window_start=window_start,
+            )
+        except Exception as e:  # noqa: BLE001  # the row is best effort; counter resets must still land
+            verbose_proxy_logger.warning(
+                "Failed to roll budget window spend row for %s=%s window=%s: %s",
+                entity_type.value,
+                entity_id,
+                budget_duration,
+                e,
+            )
+
     @staticmethod
     async def _window_carried_spend(
         window: Mapping[str, object], counter_key: str, spend_counter_cache: DualCache
@@ -1325,6 +1410,9 @@ class ResetBudgetJob:
                     spend_counter_cache,
                     now,
                     self.reset_settings,
+                    prisma_client=self.prisma_client,
+                    entity_type=source.entity_type,
+                    entity_id=row_id,
                 ):
                     changed = True
             if changed:

litellm/proxy/common_utils/user_api_key_cache.py

@@ -185,6 +185,32 @@ def tag_registry_cache_key() -> str:
     return "tag_registry"
 
 
+#: Cached under ``model_access_group_registry_cache_key`` when the table exceeds
+#: ``MODEL_ACCESS_GROUP_REGISTRY_MAX_SIZE``: registry unusable, fall back to the per-group lookup.
+MODEL_ACCESS_GROUP_REGISTRY_OVERFLOW_SENTINEL: Final = "__model_access_group_registry_overflow__"
+
+
+def model_access_group_cache_key(access_group_name: str) -> str:
+    """Cache key one model access group budget row is stored under; shared so auth, spend tracking and the management endpoints cannot drift."""
+    return f"model_access_group:{access_group_name}"
+
+
+def model_access_group_registry_cache_key() -> str:
+    """Cache key for the set of model access group names that have a budget row."""
+    return "model_access_group_registry"
+
+
+def model_access_group_spend_counter_key(access_group_name: str) -> str:
+    """Spend counter key for one model access group; shared so its four owners cannot drift.
+
+    The reservation path writes it up front, the cost callback writes it after the call, auth
+    reads it to enforce ``max_budget``, and the reset job clears it on rollover. A copy that
+    drifts in any one of them silently resets or reads a counter nobody else touches, which shows
+    up as a budget that never trips or never resets.
+    """
+    return f"spend:model_access_group:{access_group_name}"
+
+
 #: Cached under ``end_user_restricted_registry_cache_key`` when the restricted set exceeds
 #: ``END_USER_RESTRICTED_REGISTRY_MAX_SIZE``: registry unusable, fall back to the per-id fetch.
 END_USER_RESTRICTED_REGISTRY_OVERFLOW_SENTINEL: Final = "__end_user_restricted_registry_overflow__"

litellm/proxy/db/autorouter_session_rollup.py

@@ -185,8 +185,10 @@ def build_autorouter_turn_transaction(
     of a request through the router) are excluded by their internal_call_origin stamp:
     they are not traffic a user sent, so counting them would manufacture sessions and
     savings in the adoption metrics. Failed requests served nothing and are excluded.
-    Cache facts are derived from the payload's own usage record through the savings
-    owner, never handed in beside it.
+    The classifier's charge still lands here exactly once, via the decision's own
+    classifier_cost folded into this turn's spend: the excluded classifier row is how
+    it was billed, the decision is how it is attributed. Cache facts are derived from
+    the payload's own usage record through the savings owner, never handed in beside it.
     """
     if payload.get("status") != "success":
         return None
@@ -204,9 +206,12 @@ def build_autorouter_turn_transaction(
     turn_at: Final = _turn_time_utc(str(payload.get("startTime") or ""))
     if turn_at is None:
         return None
+    from litellm.proxy.spend_tracking.savings import classifier_cost_from_decision
+
     usage_object_raw: Final = metadata.get("usage_object")
     cache: Final = turn_cache_facts(usage_object_raw if isinstance(usage_object_raw, Mapping) else None)
     tier_raw: Final = routing_decision.get("tier")
+    classifier_cost: Final = classifier_cost_from_decision(routing_decision)
     return AutoRouterTurnTransaction(
         api_key=api_key,
         session_id=_bounded_session_id(session_id),
@@ -216,7 +221,7 @@ def build_autorouter_turn_transaction(
         model=model,
         turn_at=turn_at,
         total_tokens=int(payload.get("prompt_tokens") or 0) + int(payload.get("completion_tokens") or 0),
-        spend=float(payload.get("spend") or 0.0),
+        spend=float(payload.get("spend") or 0.0) + (classifier_cost or 0.0),
         saved_spend=saved_spend,
         covered=cache.covered,
         cache_hit=cache.read_tokens > 0,

litellm/proxy/db/budget_window_spend_writer.py

@@ -0,0 +1,313 @@
+"""
+Writer for LiteLLM_BudgetWindowSpend.
+
+The table holds one row per configured budget window whose window_start rolls
+forward in place, so budget enforcement can read a maintained running total
+instead of aggregating LiteLLM_SpendLogs every time a window counter goes cold
+(issue #35766). Raw SQL rather than the Prisma upsert helper because the
+conditional roll cannot be expressed through the query builder.
+
+Seeding a row that does not exist yet reads LiteLLM_SpendLogs once, excluding
+the requests whose increments are in the same batch so neither source counts
+them twice. One gap survives that exclusion: without the Redis transaction
+buffer every pod flushes its own increments, so a row seeded by one pod can
+include spend logs whose increments are still queued on another pod, and those
+increments are added again when that pod flushes. That is bounded by a single
+flush interval, happens at most once per window row, and only ever over-counts:
+the seed never omits spend, because every increment not yet in the row still
+reaches it on its own pod's next flush. A row therefore lags real spend by at
+most one flush interval of queued increments, the same lag the SpendLogs
+aggregate it replaces (and every other spend column) already has.
+"""
+
+from collections.abc import Sequence
+from datetime import datetime, timedelta, timezone
+from typing import TYPE_CHECKING, Final, Protocol
+
+from litellm._logging import verbose_proxy_logger
+from litellm.proxy._types import Litellm_EntityType
+from litellm.proxy.db.db_transaction_queue.window_spend_update_queue import (
+    WindowSpendTransaction,
+    to_naive_utc,
+    window_spend_group_key,
+)
+
+if TYPE_CHECKING:
+    from litellm.proxy.utils import PrismaClient
+
+
+_SELECT_EXISTING_ROWS_SQL: Final = (
+    'SELECT entity_type, entity_id, window_duration FROM "LiteLLM_BudgetWindowSpend" '
+    "WHERE (entity_type, entity_id, window_duration) "
+    "IN (SELECT * FROM unnest($1::text[], $2::text[], $3::text[]))"
+)
+
+_UPSERT_WINDOW_SPEND_SQL: Final = (
+    'INSERT INTO "LiteLLM_BudgetWindowSpend" '
+    "(entity_type, entity_id, window_duration, window_start, spend, created_at, updated_at) "
+    "VALUES ($1, $2, $3, ($4::timestamptz AT TIME ZONE 'UTC'), $5, "
+    "($7::timestamptz AT TIME ZONE 'UTC'), ($7::timestamptz AT TIME ZONE 'UTC')) "
+    "ON CONFLICT (entity_type, entity_id, window_duration) DO UPDATE SET "
+    "spend = CASE "
+    'WHEN "LiteLLM_BudgetWindowSpend".window_start >= EXCLUDED.window_start '
+    'THEN "LiteLLM_BudgetWindowSpend".spend + $6 '
+    "ELSE EXCLUDED.spend "
+    "END, "
+    'window_start = GREATEST("LiteLLM_BudgetWindowSpend".window_start, EXCLUDED.window_start), '
+    "updated_at = ($7::timestamptz AT TIME ZONE 'UTC')"
+)
+
+_ROLL_WINDOW_SPEND_SQL: Final = (
+    'UPDATE "LiteLLM_BudgetWindowSpend" SET '
+    "window_start = ($4::timestamptz AT TIME ZONE 'UTC'), "
+    "spend = 0, "
+    "updated_at = ($5::timestamptz AT TIME ZONE 'UTC') "
+    "WHERE entity_type = $1 AND entity_id = $2 AND window_duration = $3 "
+    "AND window_start < ($4::timestamptz AT TIME ZONE 'UTC')"
+)
+
+_SEED_FROM_SPEND_LOGS_KEY_SQL: Final = (
+    'SELECT COALESCE(SUM(spend), 0.0) AS total FROM "LiteLLM_SpendLogs" '
+    "WHERE api_key = $1 AND \"startTime\" >= ($2::timestamptz AT TIME ZONE 'UTC') "
+    "AND NOT (request_id = ANY($3::text[]) AND \"startTime\" >= ($4::timestamptz AT TIME ZONE 'UTC'))"
+)
+
+_SEED_FROM_SPEND_LOGS_TEAM_SQL: Final = (
+    'SELECT COALESCE(SUM(spend), 0.0) AS total FROM "LiteLLM_SpendLogs" '
+    "WHERE team_id = $1 AND \"startTime\" >= ($2::timestamptz AT TIME ZONE 'UTC') "
+    "AND NOT (request_id = ANY($3::text[]) AND \"startTime\" >= ($4::timestamptz AT TIME ZONE 'UTC'))"
+)
+
+_SEED_FROM_SPEND_LOGS_KEY_UNBOUNDED_SQL: Final = (
+    'SELECT COALESCE(SUM(spend), 0.0) AS total FROM "LiteLLM_SpendLogs" '
+    "WHERE api_key = $1 AND \"startTime\" >= ($2::timestamptz AT TIME ZONE 'UTC')"
+)
+
+_SEED_FROM_SPEND_LOGS_TEAM_UNBOUNDED_SQL: Final = (
+    'SELECT COALESCE(SUM(spend), 0.0) AS total FROM "LiteLLM_SpendLogs" '
+    "WHERE team_id = $1 AND \"startTime\" >= ($2::timestamptz AT TIME ZONE 'UTC')"
+)
+
+_UPSERT_TRANSACTION_TIMEOUT: Final = timedelta(seconds=60)
+
+
+class WindowSpendLogsAggregate(Protocol):
+    """Sums LiteLLM_SpendLogs for one entity since window_start, ignoring the
+    requests whose ids are handed in.
+
+    Injected so the flush can be exercised without a database and so the
+    expensive aggregate stays swappable.
+    """
+
+    async def __call__(
+        self,
+        prisma_client: "PrismaClient",
+        entity_type: str,
+        entity_id: str,
+        window_start: datetime,
+        exclude_request_ids: Sequence[str],
+        exclude_started_at: datetime | None,
+    ) -> float | None: ...
+
+
+async def spend_logs_total_excluding(
+    prisma_client: "PrismaClient",
+    entity_type: str,
+    entity_id: str,
+    window_start: datetime,
+    exclude_request_ids: Sequence[str],
+    exclude_started_at: datetime | None,
+) -> float | None:
+    """LiteLLM_SpendLogs spend for one entity since window_start, minus the
+    requests already accounted for by the increments being flushed.
+
+    The spend log writer drains its own queue on a ~2s poll whenever anything
+    is queued, while window increments flush on the much slower batch tick, so
+    by the time a window row is seeded its batch's log rows are normally
+    already in the table. Counting them in the seed and again in the increment
+    is what made a fresh row land at twice the true spend.
+
+    The exclusion is bounded to rows that started at or after the batch's
+    earliest request. request_id can be chosen by the client
+    (x-litellm-call-id), so an unbounded exclusion would let a replayed old id
+    erase a historical row from the seed while its increment still lands.
+    Without a known start the batch's ids are not excluded at all: that can
+    only over-count once, which enforcement tolerates, whereas under-counting
+    is a budget bypass.
+    """
+    if entity_type == Litellm_EntityType.KEY.value:
+        bounded_sql, unbounded_sql = _SEED_FROM_SPEND_LOGS_KEY_SQL, _SEED_FROM_SPEND_LOGS_KEY_UNBOUNDED_SQL
+    elif entity_type == Litellm_EntityType.TEAM.value:
+        bounded_sql, unbounded_sql = _SEED_FROM_SPEND_LOGS_TEAM_SQL, _SEED_FROM_SPEND_LOGS_TEAM_UNBOUNDED_SQL
+    else:
+        return None
+    rows: Final = (
+        await prisma_client.db.query_raw(unbounded_sql, entity_id, window_start)
+        if exclude_started_at is None or not exclude_request_ids
+        else await prisma_client.db.query_raw(
+            bounded_sql,
+            entity_id,
+            window_start,
+            tuple(exclude_request_ids),
+            _exclusion_lower_bound(exclude_started_at),
+        )
+    )
+    if not rows:
+        return 0.0
+    return float(rows[0].get("total") or 0.0)
+
+
+def _exclusion_lower_bound(started_at: datetime) -> datetime:
+    """LiteLLM_SpendLogs.startTime is TIMESTAMP(3); floor to the second so a
+    millisecond rounding of the batch's own earliest row cannot slip under it."""
+    return to_naive_utc(started_at).replace(microsecond=0)
+
+
+def _primary_key(transaction: WindowSpendTransaction) -> tuple[str, str, str]:
+    return (
+        transaction["entity_type"],
+        transaction["entity_id"],
+        transaction["window_duration"],
+    )
+
+
+async def _existing_primary_keys(
+    prisma_client: "PrismaClient",
+    transactions: tuple[WindowSpendTransaction, ...],
+) -> frozenset[tuple[str, str, str]]:
+    rows: Final = await prisma_client.db.query_raw(
+        _SELECT_EXISTING_ROWS_SQL,
+        tuple(transaction["entity_type"] for transaction in transactions),
+        tuple(transaction["entity_id"] for transaction in transactions),
+        tuple(transaction["window_duration"] for transaction in transactions),
+    )
+    return frozenset((row["entity_type"], row["entity_id"], row["window_duration"]) for row in rows or ())
+
+
+async def _seed_base_for_missing_row(
+    prisma_client: "PrismaClient",
+    transaction: WindowSpendTransaction,
+    existing_primary_keys: frozenset[tuple[str, str, str]],
+    spend_logs_aggregate: WindowSpendLogsAggregate,
+) -> float:
+    """Spend already recorded for a window that has no row yet.
+
+    This is the LiteLLM_SpendLogs aggregate the window counter reseed runs on
+    every cold counter today, but here it runs once per window lifetime and off
+    the request path, and it excludes this batch's own requests so they are
+    counted by their increments alone.
+    """
+    if _primary_key(transaction) in existing_primary_keys:
+        return 0.0
+    base: Final = await spend_logs_aggregate(
+        prisma_client=prisma_client,
+        entity_type=transaction["entity_type"],
+        entity_id=transaction["entity_id"],
+        window_start=datetime.fromisoformat(transaction["window_start"]).replace(tzinfo=timezone.utc),
+        exclude_request_ids=transaction["request_ids"],
+        exclude_started_at=_transaction_started_at(transaction),
+    )
+    return float(base or 0.0)
+
+
+def _transaction_started_at(transaction: WindowSpendTransaction) -> datetime | None:
+    started_at: Final = transaction.get("started_at")
+    if started_at is None:
+        return None
+    return datetime.fromisoformat(started_at).replace(tzinfo=timezone.utc)
+
+
+def _upsert_params(
+    transaction: WindowSpendTransaction,
+    seed_base: float,
+    now: datetime,
+) -> tuple[str, str, str, datetime, float, float, datetime]:
+    """$5 is what a brand new row starts at (pre-existing spend plus this
+    increment); $6 is the increment alone, which is all an already-current row
+    may add. They are equal for every row that already existed, so a row is
+    never seeded twice when two pods flush the same new window."""
+    increment: Final = float(transaction["spend"])
+    return (
+        transaction["entity_type"],
+        transaction["entity_id"],
+        transaction["window_duration"],
+        datetime.fromisoformat(transaction["window_start"]),
+        seed_base + increment,
+        increment,
+        now,
+    )
+
+
+async def commit_window_spend_updates(
+    prisma_client: "PrismaClient",
+    transactions: Sequence[WindowSpendTransaction],
+    spend_logs_aggregate: WindowSpendLogsAggregate = spend_logs_total_excluding,
+) -> None:
+    """Apply aggregated window increments to LiteLLM_BudgetWindowSpend.
+
+    An increment at or behind the row's window_start adds into the row (this is
+    how in-flight requests that raced a reset carry into the new window); an
+    increment ahead of it rolls the window and starts from that increment.
+
+    Statements are ordered by primary key so concurrent pods take row locks in
+    the same order, with window_start breaking ties so an older window is
+    applied before the roll that supersedes it.
+    """
+    if not transactions:
+        return
+
+    ordered: Final = tuple(sorted(transactions, key=window_spend_group_key))
+    existing_primary_keys: Final = await _existing_primary_keys(
+        prisma_client=prisma_client,
+        transactions=ordered,
+    )
+    seed_bases: Final = tuple(
+        [
+            await _seed_base_for_missing_row(
+                prisma_client=prisma_client,
+                transaction=transaction,
+                existing_primary_keys=existing_primary_keys,
+                spend_logs_aggregate=spend_logs_aggregate,
+            )
+            for transaction in ordered
+        ]
+    )
+
+    now: Final = to_naive_utc(datetime.now(timezone.utc))
+    verbose_proxy_logger.debug(
+        "Spend tracking - committing %d budget window spend upserts over %d existing rows",
+        len(ordered),
+        len(existing_primary_keys),
+    )
+    async with (
+        prisma_client.db.tx(timeout=_UPSERT_TRANSACTION_TIMEOUT) as db_transaction,
+        db_transaction.batch_() as batcher,
+    ):
+        for transaction, seed_base in zip(ordered, seed_bases):
+            batcher.execute_raw(
+                _UPSERT_WINDOW_SPEND_SQL,
+                *_upsert_params(transaction=transaction, seed_base=seed_base, now=now),
+            )
+
+
+async def roll_window_spend_row(
+    prisma_client: "PrismaClient",
+    entity_type: str,
+    entity_id: str,
+    window_duration: str,
+    new_window_start: datetime,
+) -> None:
+    """Move a row onto the window that just started and zero its spend.
+
+    Conditional on the stored window_start still being behind the new one so a
+    pod that already rolled the row (or increments that arrived under the new
+    window) are not clobbered.
+    """
+    await prisma_client.db.execute_raw(
+        _ROLL_WINDOW_SPEND_SQL,
+        entity_type,
+        entity_id,
+        window_duration,
+        to_naive_utc(new_window_start),
+        to_naive_utc(datetime.now(timezone.utc)),
+    )

litellm/proxy/db/db_spend_update_writer.py

@@ -12,6 +12,7 @@ import os
 import random
 import time
 import traceback
+from collections.abc import Mapping, Sequence
 from datetime import datetime, timedelta, timezone
 from typing import TYPE_CHECKING, Any, Final, Literal, Protocol, cast, overload
 
@@ -23,6 +24,7 @@ from litellm.constants import (
     DB_SPEND_UPDATE_JOB_NAME,
     INTERNAL_CALL_ORIGIN_METADATA_KEY,
 )
+from litellm.litellm_core_utils.litellm_logging import coerce_model_access_groups
 from litellm.litellm_core_utils.safe_json_loads import safe_json_loads
 from litellm.proxy._types import (
     DB_RETRY_SAFE_ERROR_TYPES,
@@ -54,6 +56,10 @@ from litellm.proxy.db.db_transaction_queue.spend_update_queue import SpendUpdate
 from litellm.proxy.db.db_transaction_queue.tool_discovery_queue import (
     ToolDiscoveryQueue,
 )
+from litellm.proxy.db.db_transaction_queue.window_spend_update_queue import (
+    WindowSpendTransaction,
+    WindowSpendUpdateQueue,
+)
 from litellm.proxy.route_llm_request import ROUTE_ENDPOINT_MAPPING
 from litellm.proxy.spend_tracking.compression_savings import (
     extract_compression_saved_tokens,
@@ -86,6 +92,7 @@ class _SpendBatch(Protocol):
     litellm_organizationtable: BatchTable
     litellm_tagtable: BatchTable
     litellm_agentstable: BatchTable
+    litellm_modelaccessgroupbudgettable: BatchTable
 
 
 class _SpendBatchManager(Protocol):
@@ -109,7 +116,7 @@ def _spend_update_tx(prisma_client: PrismaClient) -> _SpendTransactionManager:
     return tx
 
 
-def _get_llm_router():
+def get_llm_router():
     """The proxy's router, or None outside a running proxy.
 
     Injected rather than imported where it is used, so the savings computation stays
@@ -123,6 +130,52 @@ def _get_llm_router():
         return None
 
 
+class _DeploymentLookup(Protocol):
+    def get_model_info(self, id: str) -> Mapping[str, object] | None: ...
+
+
+def _served_model_access_groups(
+    router: _DeploymentLookup | None,
+    served_model_id: str | None,
+) -> frozenset[str] | None:
+    """Access groups declared by the deployment that actually served the request.
+
+    None when the served deployment cannot be identified, in which case the set
+    attributed at auth time stands unchanged.
+    """
+    if router is None or not served_model_id:
+        return None
+    deployment: Final = router.get_model_info(id=served_model_id)
+    if deployment is None:
+        return None
+    model_info: Final = deployment.get("model_info")
+    if not isinstance(model_info, Mapping):
+        return None
+    declared: Final = model_info.get("access_groups")
+    if not isinstance(declared, (list, tuple)):
+        return frozenset()
+    return frozenset(group for group in declared if isinstance(group, str))
+
+
+def debitable_model_access_groups(
+    attributed: Sequence[str] | None,
+    served_model_id: str | None,
+    router: _DeploymentLookup | None,
+) -> tuple[str, ...]:
+    """Groups to debit: the set attributed at auth time, narrowed to those the served model belongs to.
+
+    The router may fall back to a model outside the pool auth reserved against, so the
+    attributed set is the hard upper bound: a group absent from it is never debited.
+    """
+    ordered: Final = coerce_model_access_groups(attributed)
+    if not ordered:
+        return ()
+    served: Final = _served_model_access_groups(router=router, served_model_id=served_model_id)
+    if served is None:
+        return ordered
+    return tuple(group for group in ordered if group in served)
+
+
 class DBSpendUpdateWriter:
     """
     Module responsible for
@@ -146,6 +199,7 @@ class DBSpendUpdateWriter:
         self.daily_agent_spend_update_queue = DailySpendUpdateQueue()
         self.daily_org_spend_update_queue = DailySpendUpdateQueue()
         self.daily_tag_spend_update_queue = DailySpendUpdateQueue()
+        self.window_spend_update_queue = WindowSpendUpdateQueue()
 
     async def update_database(
         # LiteLLM management object fields
@@ -161,7 +215,11 @@ class DBSpendUpdateWriter:
         start_time: datetime | None,
         end_time: datetime | None,
         response_cost: float | None,
-    ):
+    ) -> str | None:
+        """Returns the LiteLLM_SpendLogs request_id this call was recorded
+        under, so the caller can tell the budget-window writer which log rows
+        its increments already cover. None when the payload could not be built.
+        """
         from litellm.proxy.proxy_server import (
             disable_spend_logs,
             litellm_proxy_budget_name,
@@ -178,7 +236,7 @@ class DBSpendUpdateWriter:
                 team_id,
             )
             if ProxyUpdateSpend.disable_spend_updates() is True:
-                return
+                return None
             if token is not None and isinstance(token, str) and token.startswith("sk-"):
                 hashed_token = hash_token(token=token)
             else:
@@ -187,6 +245,7 @@ class DBSpendUpdateWriter:
             ## CREATE SPEND LOG PAYLOAD ##
             from litellm.proxy.spend_tracking.spend_tracking_utils import (
                 get_logging_payload,
+                get_request_model_access_groups,
             )
 
             payload: Final = get_logging_payload(
@@ -239,6 +298,7 @@ class DBSpendUpdateWriter:
                     prisma_client=prisma_client,
                     litellm_proxy_budget_name=litellm_proxy_budget_name,
                     payload=payload,
+                    request_model_access_groups=get_request_model_access_groups(kwargs),
                 )
             )
 
@@ -250,6 +310,7 @@ class DBSpendUpdateWriter:
             )
 
             verbose_proxy_logger.debug("Runs spend update on all tables")
+            return payload.get("request_id")
         except Exception:
             spend_log_error(
                 "Spend tracking - update_database failed. Spend log insertion or daily transaction enqueue "
@@ -262,6 +323,7 @@ class DBSpendUpdateWriter:
                 org_id,
                 end_user_id,
             )
+            return None
 
     async def _enqueue_tool_usage_transaction(
         self,
@@ -320,7 +382,7 @@ class DBSpendUpdateWriter:
                 routing_decision=metadata.get("routing_decision"),
                 usage_object=usage_object_raw if isinstance(usage_object_raw, dict) else None,
                 model_id=payload.get("model_id"),
-                llm_router=_get_llm_router,
+                llm_router=get_llm_router,
                 cost_breakdown=metadata.get("cost_breakdown"),
                 recorded_autorouter_savings=metadata.get("autorouter_savings"),
             )
@@ -431,9 +493,10 @@ class DBSpendUpdateWriter:
         prisma_client: PrismaClient | None,
         litellm_proxy_budget_name: str | None,
         payload: SpendLogsPayload,
+        request_model_access_groups: Sequence[str] = (),
     ):
         """
-        Runs all 11 spend-update helpers sequentially inside a single asyncio task.
+        Runs all 13 spend-update helpers sequentially inside a single asyncio task.
 
         Each helper is wrapped in try/except so one failure doesn't prevent the others.
 
@@ -505,6 +568,14 @@ class DBSpendUpdateWriter:
                 traceback.format_exc(),
             )
 
+        await self._update_model_access_group_db(
+            response_cost=response_cost,
+            request_model_access_groups=request_model_access_groups,
+            served_model_id=payload_copy.get("model_id"),
+            prisma_client=prisma_client,
+            router=get_llm_router(),
+        )
+
         _agent_id_for_spend: Final = payload_copy.get("agent_id")
         try:
             await self._update_agent_db(
@@ -814,6 +885,50 @@ class DBSpendUpdateWriter:
             )
             raise e
 
+    async def _update_model_access_group_db(
+        self,
+        response_cost: float | None,
+        request_model_access_groups: Sequence[str] | None,
+        served_model_id: str | None,
+        prisma_client: PrismaClient | None,
+        router: _DeploymentLookup | None = None,
+    ) -> None:
+        """
+        Update spend for every model access group this request is billed against.
+
+        Args:
+            response_cost: Cost of the request, charged in full to each group
+            request_model_access_groups: Groups attributed at auth time, the upper bound on what may be debited
+            served_model_id: Deployment id actually served, used to narrow the attributed set
+            prisma_client: Prisma client instance
+            router: Deployment lookup used to re-resolve groups after a fallback
+        """
+        try:
+            if prisma_client is None:
+                return
+
+            for model_access_group in debitable_model_access_groups(
+                attributed=request_model_access_groups,
+                served_model_id=served_model_id,
+                router=router,
+            ):
+                await self.spend_update_queue.add_update(
+                    update=SpendUpdateQueueItem(
+                        entity_type=Litellm_EntityType.MODEL_ACCESS_GROUP,
+                        entity_id=model_access_group,
+                        response_cost=response_cost,
+                    )
+                )
+        except Exception as e:  # noqa: BLE001  # isolation: a helper failure must not stop the batch
+            spend_log_error(
+                "Spend tracking - failed to enqueue model access group spend update. "
+                "model_access_groups=%s, response_cost=%s - %s",
+                request_model_access_groups,
+                response_cost,
+                str(e),
+                exc=e,
+            )
+
     async def _insert_spend_log_to_db(
         self,
         payload: dict | SpendLogsPayload,
@@ -895,6 +1010,7 @@ class DBSpendUpdateWriter:
             daily_org_spend_update_queue=self.daily_org_spend_update_queue,
             daily_end_user_spend_update_queue=self.daily_end_user_spend_update_queue,
             daily_agent_spend_update_queue=self.daily_agent_spend_update_queue,
+            window_spend_update_queue=self.window_spend_update_queue,
         )
 
         # Only commit from redis to db if this pod is the leader
@@ -913,6 +1029,7 @@ class DBSpendUpdateWriter:
                     daily_org_spend_update_transactions,
                     daily_end_user_spend_update_transactions,
                     daily_agent_spend_update_transactions,
+                    window_spend_update_transactions,
                 ) = await self.redis_update_buffer.get_all_transactions_from_redis_buffer_pipeline()
 
                 uncommitted = {  # mutable-ok: drives which popped categories still need re-queuing
@@ -922,12 +1039,14 @@ class DBSpendUpdateWriter:
                     "daily_org_spend_update_transactions": daily_org_spend_update_transactions,
                     "daily_end_user_spend_update_transactions": daily_end_user_spend_update_transactions,
                     "daily_agent_spend_update_transactions": daily_agent_spend_update_transactions,
+                    "window_spend_update_transactions": window_spend_update_transactions,
                 }
 
                 if db_spend_update_transactions is not None:
                     verbose_proxy_logger.info(
                         "Spend tracking - committing spend updates from Redis to DB: "
-                        "keys=%d, users=%d, teams=%d, orgs=%d, end_users=%d, team_members=%d, tags=%d, agents=%d",
+                        "keys=%d, users=%d, teams=%d, orgs=%d, end_users=%d, team_members=%d, tags=%d, agents=%d, "
+                        "model_access_groups=%d",
                         len(db_spend_update_transactions.get("key_list_transactions") or {}),
                         len(db_spend_update_transactions.get("user_list_transactions") or {}),
                         len(db_spend_update_transactions.get("team_list_transactions") or {}),
@@ -936,6 +1055,7 @@ class DBSpendUpdateWriter:
                         len(db_spend_update_transactions.get("team_member_list_transactions") or {}),
                         len(db_spend_update_transactions.get("tag_list_transactions") or {}),
                         len(db_spend_update_transactions.get("agent_list_transactions") or {}),
+                        len(db_spend_update_transactions.get("model_access_group_list_transactions") or {}),
                     )
                     await self._commit_spend_updates_to_db(
                         prisma_client=prisma_client,
@@ -989,6 +1109,12 @@ class DBSpendUpdateWriter:
                         daily_spend_transactions=daily_agent_spend_update_transactions,
                     )
                 uncommitted.pop("daily_agent_spend_update_transactions", None)
+                if window_spend_update_transactions is not None:
+                    await DBSpendUpdateWriter._commit_window_spend_updates(
+                        prisma_client=prisma_client,
+                        window_spend_transactions=window_spend_update_transactions,
+                    )
+                uncommitted.pop("window_spend_update_transactions", None)
             except Exception as e:
                 spend_log_error(
                     "Spend tracking - failed to commit spend updates from Redis to DB. "
@@ -1104,6 +1230,27 @@ class DBSpendUpdateWriter:
             daily_spend_transactions=daily_agent_spend_update_transactions,
         )
 
+        ################## Budget Window Spend Update Transactions ##################
+        # Aggregate all in memory budget window spend transactions and commit to db
+        window_spend_update_transactions: Final = (
+            await self.window_spend_update_queue.flush_and_get_aggregated_window_spend_transactions()
+        )
+
+        try:
+            await DBSpendUpdateWriter._commit_window_spend_updates(
+                prisma_client=prisma_client,
+                window_spend_transactions=window_spend_update_transactions,
+            )
+        except Exception as e:  # noqa: BLE001  # the increments go back on the queue; the rest of the flush must run
+            spend_log_error(
+                "Spend tracking - failed to commit budget window spend updates. "
+                "Re-queued %d window increments for retry on next tick. Error: %s",
+                len(window_spend_update_transactions),
+                str(e),
+                exc=e,
+            )
+            await self.window_spend_update_queue.update_queue.put(window_spend_update_transactions)
+
         ################## Tool Registry Upserts ##################
         await self._flush_tool_discovery_queue(prisma_client=prisma_client)
 
@@ -1168,6 +1315,28 @@ class DBSpendUpdateWriter:
                     cronjob_id=DB_DAILY_TAG_SPEND_UPDATE_JOB_NAME,
                 )
 
+    @staticmethod
+    async def _commit_window_spend_updates(
+        prisma_client: PrismaClient,
+        window_spend_transactions: Sequence[WindowSpendTransaction],
+    ) -> None:
+        """
+        Commit per-budget-window spend increments to LiteLLM_BudgetWindowSpend.
+
+        Raises on failure so the caller re-queues the increments: budget
+        enforcement trusts a current row without reconciling it against
+        LiteLLM_SpendLogs, so a dropped increment would let the entity spend
+        past its window limit after the next counter reseed.
+        """
+        from litellm.proxy.db.budget_window_spend_writer import (
+            commit_window_spend_updates,
+        )
+
+        await commit_window_spend_updates(
+            prisma_client=prisma_client,
+            transactions=window_spend_transactions,
+        )
+
     async def _drain_and_commit_daily_tag_spend_from_redis(
         self,
         prisma_client: PrismaClient,
@@ -1433,6 +1602,20 @@ class DBSpendUpdateWriter:
             proxy_logging_obj=proxy_logging_obj,
         )
 
+        ### UPDATE MODEL ACCESS GROUP TABLE ###
+        model_access_group_list_transactions: Final = db_spend_update_transactions.get(
+            "model_access_group_list_transactions"
+        )
+        await DBSpendUpdateWriter._update_entity_spend_in_db(
+            entity_name="Model access group",
+            transactions=model_access_group_list_transactions,
+            table_accessor="litellm_modelaccessgroupbudgettable",
+            where_field="access_group_name",
+            n_retry_times=n_retry_times,
+            prisma_client=prisma_client,
+            proxy_logging_obj=proxy_logging_obj,
+        )
+
         ### UPDATE AGENT TABLE ###
         agent_list_transactions: Final = db_spend_update_transactions["agent_list_transactions"]
         await DBSpendUpdateWriter._update_entity_spend_in_db(
@@ -1449,7 +1632,7 @@ class DBSpendUpdateWriter:
     async def _update_entity_spend_in_db(
         entity_name: str,
         transactions: dict[str, float] | None,
-        table_accessor: Literal["litellm_tagtable", "litellm_agentstable"],
+        table_accessor: Literal["litellm_tagtable", "litellm_agentstable", "litellm_modelaccessgroupbudgettable"],
         where_field: str,
         n_retry_times: int,
         prisma_client: PrismaClient,
@@ -1884,7 +2067,7 @@ class DBSpendUpdateWriter:
                 gateway_injected_cache=marks_gateway_injection(_metadata, payload.get("model_id")),
                 routing_decision=_metadata.get("routing_decision"),
                 model_id=payload.get("model_id"),
-                llm_router=_get_llm_router,
+                llm_router=get_llm_router,
                 usage_object=usage_obj,
                 cost_breakdown=_metadata.get("cost_breakdown"),
                 recorded_autorouter_savings=_metadata.get("autorouter_savings"),

litellm/proxy/db/db_transaction_queue/redis_update_buffer.py

@@ -23,6 +23,7 @@ from litellm.constants import (
     REDIS_DAILY_TAG_SPEND_UPDATE_BUFFER_KEY,
     REDIS_DAILY_TEAM_SPEND_UPDATE_BUFFER_KEY,
     REDIS_UPDATE_BUFFER_KEY,
+    REDIS_WINDOW_SPEND_UPDATE_BUFFER_KEY,
 )
 from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
 from litellm.proxy._types import (
@@ -42,6 +43,10 @@ from litellm.proxy.db.db_transaction_queue.daily_spend_update_queue import (
     DailySpendUpdateQueue,
 )
 from litellm.proxy.db.db_transaction_queue.spend_update_queue import SpendUpdateQueue
+from litellm.proxy.db.db_transaction_queue.window_spend_update_queue import (
+    WindowSpendTransaction,
+    WindowSpendUpdateQueue,
+)
 from litellm.secret_managers.main import str_to_bool
 from litellm.types.caching import (
     RedisPipelineLpopOperation,
@@ -65,6 +70,7 @@ _SpendTransactionField: TypeAlias = Literal[
     "org_list_transactions",
     "tag_list_transactions",
     "agent_list_transactions",
+    "model_access_group_list_transactions",
 ]
 
 _SPEND_TRANSACTION_FIELDS: Final[tuple[_SpendTransactionField, ...]] = (
@@ -76,6 +82,7 @@ _SPEND_TRANSACTION_FIELDS: Final[tuple[_SpendTransactionField, ...]] = (
     "org_list_transactions",
     "tag_list_transactions",
     "agent_list_transactions",
+    "model_access_group_list_transactions",
 )
 
 _ValueT = TypeVar("_ValueT")
@@ -180,6 +187,7 @@ class RedisUpdateBuffer:
         daily_org_spend_update_queue: DailySpendUpdateQueue,
         daily_end_user_spend_update_queue: DailySpendUpdateQueue,
         daily_agent_spend_update_queue: DailySpendUpdateQueue,
+        window_spend_update_queue: WindowSpendUpdateQueue | None = None,
     ):
         """
         Stores the in-memory spend updates to Redis
@@ -248,6 +256,11 @@ class RedisUpdateBuffer:
         daily_agent_spend_update_transactions: Final = (
             await daily_agent_spend_update_queue.flush_and_get_aggregated_daily_spend_update_transactions()
         )
+        window_spend_update_transactions: Final = (
+            await window_spend_update_queue.flush_and_get_aggregated_window_spend_transactions()
+            if window_spend_update_queue is not None
+            else ()
+        )
 
         verbose_proxy_logger.debug("ALL DB SPEND UPDATE TRANSACTIONS: %s", db_spend_update_transactions)
         verbose_proxy_logger.debug("ALL DAILY SPEND UPDATE TRANSACTIONS: %s", daily_spend_update_transactions)
@@ -284,6 +297,11 @@ class RedisUpdateBuffer:
                 REDIS_DAILY_AGENT_SPEND_UPDATE_BUFFER_KEY,
                 ServiceTypes.REDIS_DAILY_AGENT_SPEND_UPDATE_QUEUE,
             ),
+            (
+                window_spend_update_transactions,
+                REDIS_WINDOW_SPEND_UPDATE_BUFFER_KEY,
+                ServiceTypes.REDIS_WINDOW_SPEND_UPDATE_QUEUE,
+            ),
         ]
 
         rpush_list: Final[list[RedisPipelineRpushOperation]] = []
@@ -324,12 +342,14 @@ class RedisUpdateBuffer:
                 daily_org_spend_update_transactions=daily_org_spend_update_transactions,
                 daily_end_user_spend_update_transactions=daily_end_user_spend_update_transactions,
                 daily_agent_spend_update_transactions=daily_agent_spend_update_transactions,
+                window_spend_update_transactions=window_spend_update_transactions,
                 spend_update_queue=spend_update_queue,
                 daily_spend_update_queue=daily_spend_update_queue,
                 daily_team_spend_update_queue=daily_team_spend_update_queue,
                 daily_org_spend_update_queue=daily_org_spend_update_queue,
                 daily_end_user_spend_update_queue=daily_end_user_spend_update_queue,
                 daily_agent_spend_update_queue=daily_agent_spend_update_queue,
+                window_spend_update_queue=window_spend_update_queue,
             )
             return
 
@@ -349,12 +369,14 @@ class RedisUpdateBuffer:
         daily_org_spend_update_transactions: dict[str, BaseDailySpendTransaction] | None,
         daily_end_user_spend_update_transactions: dict[str, BaseDailySpendTransaction] | None,
         daily_agent_spend_update_transactions: dict[str, BaseDailySpendTransaction] | None,
+        window_spend_update_transactions: tuple[WindowSpendTransaction, ...] | None,
         spend_update_queue: SpendUpdateQueue,
         daily_spend_update_queue: DailySpendUpdateQueue,
         daily_team_spend_update_queue: DailySpendUpdateQueue,
         daily_org_spend_update_queue: DailySpendUpdateQueue,
         daily_end_user_spend_update_queue: DailySpendUpdateQueue,
         daily_agent_spend_update_queue: DailySpendUpdateQueue,
+        window_spend_update_queue: WindowSpendUpdateQueue | None,
     ) -> None:
         """
         Put drained-but-unpushed transactions back into in-memory queues.
@@ -397,6 +419,10 @@ class RedisUpdateBuffer:
                     Litellm_EntityType.AGENT,
                     db_spend_update_transactions.get("agent_list_transactions"),
                 ),
+                (
+                    Litellm_EntityType.MODEL_ACCESS_GROUP,
+                    db_spend_update_transactions.get("model_access_group_list_transactions"),
+                ),
             ]
             for entity_type, entities in entity_entries:
                 if not entities:
@@ -424,6 +450,9 @@ class RedisUpdateBuffer:
             if daily_txns:
                 await daily_queue.update_queue.put(daily_txns)
 
+        if window_spend_update_transactions and window_spend_update_queue is not None:
+            await window_spend_update_queue.update_queue.put(window_spend_update_transactions)
+
     async def restore_transactions_to_redis(
         self,
         db_spend_update_transactions: DBSpendUpdateTransactions | None = None,
@@ -433,6 +462,7 @@ class RedisUpdateBuffer:
         daily_end_user_spend_update_transactions: Mapping[str, BaseDailySpendTransaction] | None = None,
         daily_agent_spend_update_transactions: Mapping[str, BaseDailySpendTransaction] | None = None,
         daily_tag_spend_update_transactions: Mapping[str, BaseDailySpendTransaction] | None = None,
+        window_spend_update_transactions: Sequence[WindowSpendTransaction] | None = None,
     ) -> None:
         """
         Re-push transactions that were popped from Redis but not committed to the DB.
@@ -454,6 +484,7 @@ class RedisUpdateBuffer:
             (daily_end_user_spend_update_transactions, REDIS_DAILY_END_USER_SPEND_UPDATE_BUFFER_KEY),
             (daily_agent_spend_update_transactions, REDIS_DAILY_AGENT_SPEND_UPDATE_BUFFER_KEY),
             (daily_tag_spend_update_transactions, REDIS_DAILY_TAG_SPEND_UPDATE_BUFFER_KEY),
+            (window_spend_update_transactions, REDIS_WINDOW_SPEND_UPDATE_BUFFER_KEY),
         )
 
         rpush_list: Final = tuple(
@@ -571,20 +602,22 @@ class RedisUpdateBuffer:
         dict[str, DailyOrganizationSpendTransaction] | None,
         dict[str, DailyEndUserSpendTransaction] | None,
         dict[str, DailyAgentSpendTransaction] | None,
+        tuple[WindowSpendTransaction, ...] | None,
     ]:
         """
-        Drains the main 6 Redis buffer queues in a single pipeline round-trip.
+        Drains the main 7 Redis buffer queues in a single pipeline round-trip.
 
-        Returns a 6-tuple of parsed results in this order:
+        Returns a 7-tuple of parsed results in this order:
             0: DBSpendUpdateTransactions
             1: daily user spend
             2: daily team spend
             3: daily org spend
             4: daily end-user spend
             5: daily agent spend
+            6: budget window spend
         """
         if self.redis_cache is None:
-            return None, None, None, None, None, None
+            return None, None, None, None, None, None, None
 
         lpop_list: Final[list[RedisPipelineLpopOperation]] = [
             RedisPipelineLpopOperation(key=REDIS_UPDATE_BUFFER_KEY, count=MAX_REDIS_BUFFER_DEQUEUE_COUNT),
@@ -608,12 +641,16 @@ class RedisUpdateBuffer:
                 key=REDIS_DAILY_AGENT_SPEND_UPDATE_BUFFER_KEY,
                 count=MAX_REDIS_BUFFER_DEQUEUE_COUNT,
             ),
+            RedisPipelineLpopOperation(
+                key=REDIS_WINDOW_SPEND_UPDATE_BUFFER_KEY,
+                count=MAX_REDIS_BUFFER_DEQUEUE_COUNT,
+            ),
         ]
 
         raw_results: Final = await self.redis_cache.async_lpop_pipeline(lpop_list=lpop_list)
 
         # Pad with None if pipeline returned fewer results than expected
-        while len(raw_results) < 6:
+        while len(raw_results) < 7:
             raw_results.append(None)
 
         # Slot 0: DBSpendUpdateTransactions
@@ -634,6 +671,14 @@ class RedisUpdateBuffer:
                 aggregated = DailySpendUpdateQueue.get_aggregated_daily_spend_update_transactions(list_of_daily)
                 daily_results.append(aggregated)
 
+        window_spend: Final = (
+            WindowSpendUpdateQueue.get_aggregated_window_spend_transactions(
+                tuple(json.loads(transaction) for transaction in raw_results[6])
+            )
+            if raw_results[6] is not None
+            else None
+        )
+
         return (
             db_spend,
             cast(dict[str, DailyUserSpendTransaction] | None, daily_results[0]),
@@ -641,6 +686,7 @@ class RedisUpdateBuffer:
             cast(dict[str, DailyOrganizationSpendTransaction] | None, daily_results[2]),
             cast(dict[str, DailyEndUserSpendTransaction] | None, daily_results[3]),
             cast(dict[str, DailyAgentSpendTransaction] | None, daily_results[4]),
+            window_spend,
         )
 
     async def store_in_memory_daily_tag_spend_updates_in_redis(
@@ -826,6 +872,9 @@ class RedisUpdateBuffer:
             org_list_transactions=_merged_entity_transactions(list_of_transactions, "org_list_transactions"),
             tag_list_transactions=_merged_entity_transactions(list_of_transactions, "tag_list_transactions"),
             agent_list_transactions=_merged_entity_transactions(list_of_transactions, "agent_list_transactions"),
+            model_access_group_list_transactions=_merged_entity_transactions(
+                list_of_transactions, "model_access_group_list_transactions"
+            ),
         )
 
     async def _emit_new_item_added_to_redis_buffer_event(

litellm/proxy/db/db_transaction_queue/spend_update_queue.py

@@ -139,6 +139,7 @@ class SpendUpdateQueue(BaseUpdateQueue):
             org_list_transactions={},
             tag_list_transactions={},
             agent_list_transactions={},
+            model_access_group_list_transactions={},
         )
 
         # Map entity types to their corresponding transaction dictionary keys
@@ -151,6 +152,7 @@ class SpendUpdateQueue(BaseUpdateQueue):
             Litellm_EntityType.ORGANIZATION: "org_list_transactions",
             Litellm_EntityType.TAG: "tag_list_transactions",
             Litellm_EntityType.AGENT: "agent_list_transactions",
+            Litellm_EntityType.MODEL_ACCESS_GROUP: "model_access_group_list_transactions",
         }
 
         for update in updates:
@@ -190,6 +192,8 @@ class SpendUpdateQueue(BaseUpdateQueue):
                 transactions_dict = db_spend_update_transactions["tag_list_transactions"]
             elif dict_key == "agent_list_transactions":
                 transactions_dict = db_spend_update_transactions["agent_list_transactions"]
+            elif dict_key == "model_access_group_list_transactions":
+                transactions_dict = db_spend_update_transactions["model_access_group_list_transactions"]
             else:
                 continue
 

litellm/proxy/db/db_transaction_queue/window_spend_update_queue.py

@@ -0,0 +1,176 @@
+"""
+In memory buffer for per-budget-window spend increments.
+
+Kept separate from SpendUpdateQueue: an increment is only meaningful together
+with the window it landed in, so two increments for the same entity must not be
+merged when their window_start differs.
+"""
+
+import asyncio
+import math
+from collections.abc import Sequence
+from datetime import datetime, timezone
+from itertools import chain, groupby
+from typing import Final, TypedDict
+
+from typing_extensions import ReadOnly
+
+from litellm._logging import verbose_proxy_logger
+from litellm.constants import LITELLM_ASYNCIO_QUEUE_MAXSIZE
+from litellm.proxy.db.db_transaction_queue.base_update_queue import BaseUpdateQueue
+
+
+class WindowSpendTransaction(TypedDict):
+    """One increment for a single (entity, budget window) pair.
+
+    window_start is an ISO-8601 string rather than a datetime so the
+    transaction survives the JSON round trip through the Redis buffer.
+
+    request_ids carries the LiteLLM_SpendLogs ids this spend came from. The
+    one-time seed for a window that has no row yet subtracts them from its
+    LiteLLM_SpendLogs aggregate, because the spend log writer flushes on its
+    own ~2s poll and will usually have persisted these rows before the window
+    queue flushes; without the exclusion the seed and the increment would each
+    count them.
+
+    started_at is the earliest request start in the batch. The seed only
+    subtracts a request_id whose LiteLLM_SpendLogs.startTime is at or after it,
+    so a client that replays an old id through x-litellm-call-id cannot make the
+    seed drop the historical row that id already paid for.
+    """
+
+    entity_type: ReadOnly[str]
+    entity_id: ReadOnly[str]
+    window_duration: ReadOnly[str]
+    window_start: ReadOnly[str]
+    spend: ReadOnly[float]
+    request_ids: ReadOnly[Sequence[str]]
+    started_at: ReadOnly[str | None]
+
+
+def to_naive_utc(value: datetime) -> datetime:
+    """LiteLLM_BudgetWindowSpend.window_start is TIMESTAMP(3), which holds naive UTC."""
+    if value.tzinfo is None:
+        return value
+    return value.astimezone(timezone.utc).replace(tzinfo=None)
+
+
+def window_spend_group_key(transaction: WindowSpendTransaction) -> tuple[str, str, str, str]:
+    """Identity of a window increment: the row's primary key plus the window it
+    belongs to. Two increments only aggregate when all four match."""
+    return (
+        transaction["entity_type"],
+        transaction["entity_id"],
+        transaction["window_duration"],
+        transaction["window_start"],
+    )
+
+
+def build_window_spend_transaction(
+    entity_type: str,
+    entity_id: str,
+    window_duration: str,
+    window_start: datetime,
+    spend: float,
+    request_id: str | None = None,
+    started_at: datetime | None = None,
+) -> WindowSpendTransaction:
+    return WindowSpendTransaction(
+        entity_type=entity_type,
+        entity_id=entity_id,
+        window_duration=window_duration,
+        window_start=to_naive_utc(window_start).isoformat(timespec="microseconds"),
+        spend=spend,
+        request_ids=() if request_id is None else (request_id,),
+        started_at=None
+        if started_at is None
+        else to_naive_utc(started_at.astimezone(timezone.utc)).isoformat(timespec="microseconds"),
+    )
+
+
+def _merge_window_spend_transactions(
+    payloads: tuple[WindowSpendTransaction, ...],
+) -> WindowSpendTransaction:
+    first: Final = payloads[0]
+    started_ats: Final = tuple(
+        started_at for payload in payloads if (started_at := payload.get("started_at")) is not None
+    )
+    return WindowSpendTransaction(
+        entity_type=first["entity_type"],
+        entity_id=first["entity_id"],
+        window_duration=first["window_duration"],
+        window_start=first["window_start"],
+        spend=math.fsum(payload["spend"] for payload in payloads),
+        request_ids=tuple(sorted(frozenset(chain.from_iterable(payload["request_ids"] for payload in payloads)))),
+        started_at=min(started_ats) if started_ats else None,
+    )
+
+
+class WindowSpendUpdateQueue(BaseUpdateQueue):
+    """
+    In memory buffer for budget-window spend increments committed to
+    LiteLLM_BudgetWindowSpend.
+
+    Add an update with the payload built by build_window_spend_transaction:
+        window_spend_update_queue.add_update(
+            build_window_spend_transaction(
+                entity_type="key",
+                entity_id="<hashed token>",
+                window_duration="30d",
+                window_start=datetime(2026, 8, 1, tzinfo=timezone.utc),
+                spend=0.02,
+            )
+        )
+    """
+
+    def __init__(self) -> None:
+        super().__init__()
+        self.update_queue: asyncio.Queue[tuple[WindowSpendTransaction, ...]] = asyncio.Queue(
+            maxsize=LITELLM_ASYNCIO_QUEUE_MAXSIZE
+        )
+
+    async def add_update(self, update: WindowSpendTransaction) -> None:
+        """Enqueue an update."""
+        verbose_proxy_logger.debug("Adding budget window spend update to queue: %s", update)
+        await self.update_queue.put((update,))
+        if self.update_queue.qsize() >= self.MAX_SIZE_IN_MEMORY_QUEUE:
+            verbose_proxy_logger.warning(
+                "Budget window spend update queue is full. Aggregating all entries in queue to concatenate entries."
+            )
+            await self.aggregate_queue_updates()
+
+    async def aggregate_queue_updates(self) -> None:
+        """Collapse everything currently queued into a single aggregated update."""
+        updates: Final = await self.flush_all_updates_from_in_memory_queue()
+        await self.update_queue.put(WindowSpendUpdateQueue.get_aggregated_window_spend_transactions(updates))
+
+    async def flush_and_get_aggregated_window_spend_transactions(
+        self,
+    ) -> tuple[WindowSpendTransaction, ...]:
+        """Drain the queue and return the increments aggregated per window."""
+        updates: Final = await self.flush_all_updates_from_in_memory_queue()
+        if len(updates) > 0:
+            verbose_proxy_logger.info(
+                "Spend tracking - flushed %d budget window spend update batches from in-memory queue",
+                len(updates),
+            )
+        return WindowSpendUpdateQueue.get_aggregated_window_spend_transactions(updates)
+
+    @staticmethod
+    def get_aggregated_window_spend_transactions(
+        updates: Sequence[Sequence[WindowSpendTransaction]],
+    ) -> tuple[WindowSpendTransaction, ...]:
+        """Sum spend per (entity_type, entity_id, window_duration, window_start).
+
+        Increments belonging to different windows stay separate even when they
+        share a primary key, so a window boundary crossed mid-tick does not fold
+        the new window's spend into the previous window's total.
+
+        The result is ordered by that same key, which is the order the flush
+        needs: primary key first for cross-pod lock ordering, then window_start
+        so an older window is applied before the roll that supersedes it.
+        """
+        ordered: Final = tuple(sorted(chain.from_iterable(updates), key=window_spend_group_key))
+        return tuple(
+            _merge_window_spend_transactions(tuple(group)) for _, group in groupby(ordered, key=window_spend_group_key)
+        )

litellm/proxy/db/routing_prisma_wrapper.py

@@ -61,6 +61,26 @@ class _RoutedActions:
         return getattr(self._writer_actions, name)
 
 
+class WriterPinnedClient:
+    """PrismaClient-shaped view whose `.db` resolves to the writer while it is available.
+
+    Read-after-write paths (e.g. the model reconcile a /model/new triggers to
+    verify its own just-committed row) must not read through a lagging read
+    replica: the row is not replayed there yet, so the reconcile concludes the
+    write is missing and fails the request even though it is durable (#38556).
+
+    While the writer is degraded (`writer_unavailable`), the pin yields to the
+    routed wrapper so reconcile reads keep working from the replica: a proxy
+    that starts during a primary outage must still load DB-backed models, and
+    no read-after-write hazard exists then because writes are failing anyway.
+    """
+
+    __slots__ = ("db",)
+
+    def __init__(self, db: "PrismaWrapper | RoutingPrismaWrapper") -> None:
+        self.db: Final = db.writer if isinstance(db, RoutingPrismaWrapper) and not db.writer_unavailable else db
+
+
 class RoutingPrismaWrapper:
     """
     Routes Prisma operations between a writer and a reader Prisma client.

litellm/proxy/db/spend_counter_reseed.py

@@ -14,14 +14,18 @@ memory in long-lived deployments.
 
 import asyncio
 from collections import OrderedDict
-from datetime import datetime
+from collections.abc import Mapping
+from datetime import datetime, timezone
+from types import MappingProxyType
 from typing import TYPE_CHECKING, ClassVar, Final, Optional
 
 from litellm._logging import verbose_proxy_logger
 from litellm.constants import SPEND_COUNTER_RESEED_LOCKS_MAX_SIZE
 from litellm.litellm_core_utils.duration_parser import duration_in_seconds
+from litellm.proxy._types import Litellm_EntityType
 from litellm.repositories.organization_repository import OrganizationRepository
 from litellm.repositories.table_repositories import (
+    BudgetWindowSpendRepository,
     SpendLogsRepository,
     TeamMembershipRepository,
 )
@@ -36,6 +40,25 @@ if TYPE_CHECKING:
     from litellm.proxy.utils import PrismaClient
 
 
+_WINDOW_SPEND_ENTITY_TYPES: Final[Mapping[str, str]] = MappingProxyType(
+    {
+        "Key": Litellm_EntityType.KEY.value,
+        "Team": Litellm_EntityType.TEAM.value,
+    }
+)
+
+_WINDOW_SPEND_LOG_FIELDS: Final[Mapping[str, str]] = MappingProxyType(
+    {
+        "Key": "api_key",
+        "Team": "team_id",
+    }
+)
+
+
+def _as_utc(value: datetime) -> datetime:
+    return value if value.tzinfo is not None else value.replace(tzinfo=timezone.utc)
+
+
 class SpendCounterReseed:
     """
     Reseeds spend counters from the authoritative DB and warms the cache,
@@ -205,6 +228,92 @@ class SpendCounterReseed:
                     raise
             return current_value
 
+    @staticmethod
+    async def window_from_table(
+        prisma_client: Optional["PrismaClient"],
+        entity_type: str,
+        entity_id: str,
+        window_duration: str,
+        expected_window_start: datetime,
+    ) -> float | None:
+        """
+        Read the maintained per-window spend row by primary key.
+
+        Returns the row's spend only when the row belongs to the window the
+        caller is enforcing, i.e. ``row.window_start >= expected_window_start``.
+        A row at or past the expected start was rolled by a pod whose reset_at
+        was at least as fresh as this caller's, so it is trusted; an older row
+        means the window boundary was crossed and nothing has rolled the row
+        yet, so its spend belongs to a previous window.
+
+        Returns None for a missing, stale or unreadable row so the caller falls
+        back to the spend-logs aggregate. ``entity_type`` is the counter-facing
+        label ("Key"/"Team"); anything else has no row and returns None.
+        """
+        if prisma_client is None:
+            return None
+        row_entity_type: Final = _WINDOW_SPEND_ENTITY_TYPES.get(entity_type)
+        if row_entity_type is None:
+            return None
+
+        try:
+            row: Final = await BudgetWindowSpendRepository(prisma_client).table.find_unique(
+                where={
+                    "entity_type_entity_id_window_duration": {
+                        "entity_type": row_entity_type,
+                        "entity_id": entity_id,
+                        "window_duration": window_duration,
+                    }
+                }
+            )
+        except Exception:  # noqa: BLE001  # any read failure (DB, stale prisma client) must degrade to the aggregate path
+            verbose_proxy_logger.exception(
+                "SpendCounterReseed.window_from_table: failed for %s=%s window=%s",
+                entity_type,
+                entity_id,
+                window_duration,
+            )
+            return None
+
+        if row is None:
+            return None
+        if _as_utc(row.window_start) < _as_utc(expected_window_start):
+            return None
+        return float(row.spend or 0.0)
+
+    @staticmethod
+    async def window_from_db(
+        prisma_client: Optional["PrismaClient"],
+        entity_type: str,
+        entity_id: str,
+        window_duration: str | None,
+        window_start: datetime,
+    ) -> float | None:
+        """
+        Authoritative window spend: the maintained row first, falling back to
+        the spend-logs aggregate only when no current row exists.
+
+        The aggregate range-scans an unindexed table, so it must stay a
+        transitional path (window configured before the row existed) rather
+        than a steady-state read.
+        """
+        if window_duration is not None:
+            from_table: Final = await SpendCounterReseed.window_from_table(
+                prisma_client=prisma_client,
+                entity_type=entity_type,
+                entity_id=entity_id,
+                window_duration=window_duration,
+                expected_window_start=window_start,
+            )
+            if from_table is not None:
+                return from_table
+        return await SpendCounterReseed.window_from_spend_logs(
+            prisma_client=prisma_client,
+            entity_type=entity_type,
+            entity_id=entity_id,
+            window_start=window_start,
+        )
+
     @staticmethod
     async def window_from_spend_logs(
         prisma_client: Optional["PrismaClient"],
@@ -215,20 +324,13 @@ class SpendCounterReseed:
         if prisma_client is None:
             return None
 
-        if entity_type == "Key":
-            group_field = "api_key"
-            where = {
-                "api_key": entity_id,
-                "startTime": {"gte": window_start},
-            }
-        elif entity_type == "Team":
-            group_field = "team_id"
-            where = {
-                "team_id": entity_id,
-                "startTime": {"gte": window_start},
-            }
-        else:
+        group_field: Final = _WINDOW_SPEND_LOG_FIELDS.get(entity_type)
+        if group_field is None:
             return None
+        where: Final = {
+            group_field: entity_id,
+            "startTime": {"gte": window_start},
+        }
 
         try:
             response: Final = await SpendLogsRepository(prisma_client).table.group_by(
@@ -258,6 +360,7 @@ class SpendCounterReseed:
         counter_key: str,
         entity_type: str,
         entity_id: str,
+        window_duration: str | None,
         window_start: datetime,
     ) -> float | None:
         lock: Final = await SpendCounterReseed._get_lock(counter_key)
@@ -276,10 +379,11 @@ class SpendCounterReseed:
                 if val is not None:
                     return float(val)
 
-            window_spend: Final = await SpendCounterReseed.window_from_spend_logs(
+            window_spend: Final = await SpendCounterReseed.window_from_db(
                 prisma_client=prisma_client,
                 entity_type=entity_type,
                 entity_id=entity_id,
+                window_duration=window_duration,
                 window_start=window_start,
             )
             if window_spend is None:

litellm/proxy/guardrails/guardrail_hooks/headroom/headroom.py

@@ -37,8 +37,12 @@ from litellm.proxy.guardrails.guardrail_hooks.content_text import (
 from litellm.proxy.spend_tracking.compression_savings import HEADROOM_GUARDRAIL_PROVIDER
 from litellm.secret_managers.main import get_secret_str
 from litellm.types.guardrails import GuardrailEventHooks, Mode
-from litellm.types.integrations.custom_logger import AgenticLoopPlan, AgenticLoopRequestPatch
-from litellm.types.utils import GenericGuardrailAPIInputs
+from litellm.types.integrations.custom_logger import (
+    HEADROOM_CONVERTED_STREAM_KEY,
+    AgenticLoopPlan,
+    AgenticLoopRequestPatch,
+)
+from litellm.types.utils import CallTypes, GenericGuardrailAPIInputs
 
 if TYPE_CHECKING:
     from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
@@ -713,6 +717,25 @@ class HeadroomGuardrail(CustomGuardrail):
 
         return {**inputs, "structured_messages": compressed, "tools": merged_tools}  # pyright: ignore[reportReturnType]
 
+    async def async_pre_call_deployment_hook(
+        self,
+        kwargs: dict[str, Any],
+        call_type: CallTypes | None,
+    ) -> dict[str, Any] | None:  # mutable-ok: overrides CustomLogger hook whose contract is a plain dict
+        base_result: Final = await super().async_pre_call_deployment_hook(kwargs, call_type)
+        effective: Final = base_result if base_result is not None else kwargs
+        if call_type not in (CallTypes.completion, CallTypes.acompletion):
+            return base_result
+        if not effective.get("stream"):
+            return base_result
+        if not has_headroom_retrieve_tool(effective.get("tools")):
+            return base_result
+        return {  # mutable-ok: the hook contract is a plain dict the router merges into the request kwargs
+            **effective,
+            "stream": False,
+            HEADROOM_CONVERTED_STREAM_KEY: True,
+        }
+
     async def async_should_run_agentic_loop(
         self,
         response: Any,

litellm/proxy/guardrails/guardrail_hooks/hiddenlayer/hiddenlayer.py

@@ -37,6 +37,9 @@ if TYPE_CHECKING:
     from litellm.types.proxy.guardrails.guardrail_hooks.base import GuardrailConfigModel
 
 
+_AUTH_TIMEOUT_SECONDS: Final[float] = 30.0
+
+
 class _HiddenlayerEvaluation(TypedDict, total=False):
     action: str
     threat_level: str
@@ -157,10 +160,10 @@ def is_saas(host: str) -> bool:
     return False
 
 
-def _get_jwt(auth_url, api_id, api_key) -> str:
+def _get_jwt(auth_url, api_id, api_key, timeout: float = _AUTH_TIMEOUT_SECONDS) -> str:
     token_url: Final = f"{auth_url}/oauth2/token?grant_type=client_credentials"
 
-    resp: Final = requests.post(token_url, auth=HTTPBasicAuth(api_id, api_key))
+    resp: Final = requests.post(token_url, auth=HTTPBasicAuth(api_id, api_key), timeout=timeout)
 
     if not resp.ok:
         raise RuntimeError(

litellm/proxy/guardrails/guardrail_hooks/lakera_ai_v2.py

@@ -301,12 +301,12 @@ class LakeraAIGuardrail(CustomGuardrail):
         explicit sync below a hot reload that changes mode would pass validation but
         keep dispatching on the stale event_hook.
         """
-        new_event_hook: Final = getattr(litellm_params, "mode", None) or self.event_hook
-        prospective_payload: Final = getattr(litellm_params, "payload", None)
-        prospective_breakdown: Final = getattr(litellm_params, "breakdown", None)
+        new_event_hook: Final = litellm_params.mode or self.event_hook
+        prospective_payload: Final = litellm_params.payload
+        prospective_breakdown: Final = litellm_params.breakdown
         self._validate_advisory_config(
-            on_flagged=getattr(litellm_params, "on_flagged", None) or self.on_flagged,
-            advisory_system_message=getattr(litellm_params, "advisory_system_message", None),
+            on_flagged=litellm_params.on_flagged or self.on_flagged,
+            advisory_system_message=litellm_params.advisory_system_message,
             payload=self.payload if prospective_payload is None else prospective_payload,
             breakdown=self.breakdown if prospective_breakdown is None else prospective_breakdown,
         )

litellm/proxy/guardrails/guardrail_hooks/qualifire/qualifire.py

@@ -121,7 +121,7 @@ class QualifireGuardrail(CustomGuardrail):
         the live instance untouched instead of raising after it's already been
         corrupted. Mirrors LakeraAIGuardrail's own override of this same method.
         """
-        prospective_on_flagged: Final = getattr(litellm_params, "on_flagged", None) or self.on_flagged
+        prospective_on_flagged: Final = litellm_params.on_flagged or self.on_flagged
         self._validate_on_flagged(prospective_on_flagged)
         super().update_in_memory_litellm_params(litellm_params=litellm_params)
 

litellm/proxy/guardrails/guardrail_registry.py

@@ -413,14 +413,15 @@ class GuardrailRegistry:
             raise Exception(f"Error getting guardrail from DB: {e}")
 
 
-def _apply_configured_bool_override(instance: CustomGuardrail, litellm_params: LitellmParams, param_name: str) -> None:
-    """Override ``instance.<param_name>`` only when ``litellm_params`` explicitly
-    sets it, preserving whatever default the guardrail's own constructor chose
+def _apply_configured_bool_overrides(instance: CustomGuardrail, litellm_params: LitellmParams) -> None:
+    """Override the parallel/raw-scan flags only when ``litellm_params`` explicitly
+    sets them, preserving whatever default the guardrail's own constructor chose
     otherwise (its constructor default may be True, so blindly copying an
     absent/None config value would silently clobber it back to False)."""
-    configured: Final = getattr(litellm_params, param_name, None)
-    if configured is not None:
-        setattr(instance, param_name, bool(configured))
+    if litellm_params.run_in_parallel is not None:
+        instance.run_in_parallel = bool(litellm_params.run_in_parallel)
+    if litellm_params.scan_raw_request is not None:
+        instance.scan_raw_request = bool(litellm_params.scan_raw_request)
 
 
 class InMemoryGuardrailHandler:
@@ -544,8 +545,7 @@ class InMemoryGuardrailHandler:
                     "skip_tool_message_in_guardrail are enabled together, which excludes every message from "
                     "scanning, so no request content would ever be scanned. Remove one of the two."
                 )
-            for override_param in ("run_in_parallel", "scan_raw_request"):
-                _apply_configured_bool_override(custom_guardrail_callback, litellm_params, override_param)
+            _apply_configured_bool_overrides(custom_guardrail_callback, litellm_params)
 
         parsed_guardrail: Final = Guardrail(
             guardrail_id=guardrail.get("guardrail_id"),
@@ -803,7 +803,6 @@ class InMemoryGuardrailHandler:
         previous_guardrail: Final = self.IN_MEMORY_GUARDRAILS.get(guardrail_id)
         previous_source: Final = self._sources.get(guardrail_id, source)
 
-        # Remove from memory if exists (also removes from callbacks)
         if guardrail_id in self.IN_MEMORY_GUARDRAILS:
             self.delete_in_memory_guardrail(guardrail_id)
 

litellm/proxy/hooks/proxy_track_cost_callback.py

@@ -1,5 +1,6 @@
 import asyncio
 import traceback
+from collections.abc import Sequence
 from datetime import datetime
 from typing import TYPE_CHECKING, Any, Final, cast
 
@@ -20,6 +21,10 @@ from litellm.proxy.auth.auth_checks import (
     log_db_metrics,
 )
 from litellm.proxy.auth.route_checks import RouteChecks
+from litellm.proxy.db.db_spend_update_writer import (
+    debitable_model_access_groups,
+    get_llm_router,
+)
 from litellm.proxy.litellm_pre_call_utils import LiteLLMProxyRequestSetup
 from litellm.proxy.spend_tracking.spend_log_error_logger import (
     should_suppress_spend_log_tracebacks,
@@ -27,6 +32,7 @@ from litellm.proxy.spend_tracking.spend_log_error_logger import (
 )
 from litellm.proxy.spend_tracking.spend_tracking_utils import (
     _sanitize_error_information_for_spend_logs,
+    get_request_model_access_groups,
 )
 from litellm.proxy.utils import ProxyUpdateSpend
 from litellm.types.utils import (
@@ -258,6 +264,11 @@ class _ProxyDBLogger(CustomLogger):
                 sl_object=sl_object,
                 metadata=metadata,
             )
+            model_access_groups: Final = debitable_model_access_groups(
+                attributed=get_request_model_access_groups(kwargs),
+                served_model_id=sl_object.get("model_id") if sl_object is not None else None,
+                router=get_llm_router(),
+            )
 
             if response_cost is not None:
                 user_api_key: Final = metadata.get("user_api_key", None)
@@ -296,6 +307,7 @@ class _ProxyDBLogger(CustomLogger):
                         response_cost=response_cost,
                         budget_reservation=budget_reservation,
                         request_tags=tags,
+                        model_access_groups=model_access_groups,
                     )
 
                     # update cache (fire-and-forget for backward compat:
@@ -572,9 +584,10 @@ async def _update_database_and_spend_counters(
     response_cost: float,
     budget_reservation: dict | None,
     request_tags: list[str] | None = None,
+    model_access_groups: Sequence[str] | None = None,
 ) -> None:
     try:
-        await proxy_logging_obj.db_spend_update_writer.update_database(
+        spend_log_request_id = await proxy_logging_obj.db_spend_update_writer.update_database(
             token=user_api_key,
             response_cost=response_cost,
             user_id=user_id,
@@ -610,6 +623,9 @@ async def _update_database_and_spend_counters(
             budget_reservation=budget_reservation,
             end_user_id=end_user_id,
             tags=request_tags,
+            request_id=spend_log_request_id,
+            request_started_at=start_time,
+            model_access_groups=model_access_groups,
         )
     except Exception:
         if budget_reservation is not None:

litellm/proxy/litellm_pre_call_utils.py

@@ -29,6 +29,7 @@ from litellm.litellm_core_utils.initialize_dynamic_callback_params import (
     _request_blocked_callback_params,
     iter_client_callback_metadata_dicts,
 )
+from litellm.litellm_core_utils.internal_call_metadata import MODEL_ACCESS_GROUP_METADATA_KEY
 from litellm.litellm_core_utils.safe_json_loads import safe_json_loads
 from litellm.litellm_core_utils.url_utils import (
     is_url_destination_allowed_by_host,
@@ -253,6 +254,7 @@ _UNTRUSTED_ROOT_CONTROL_FIELDS: Final = (
     "_code_interpreter_interception_converted_stream",
     "_code_interpreter_interception_sandbox_key",
     "_code_interpreter_interception_session_scoped",
+    "_headroom_interception_converted_stream",
     "max_agentic_loops",
     # Recomputed below from the actual caller-controlled timeout sources (headers and
     # body fields); a client-forged value here would let a request either dodge cooldown
@@ -1377,6 +1379,10 @@ class LiteLLMProxyRequestSetup:
         )
         if user_api_key_dict.budget_reservation is not None:
             data[_metadata_variable_name]["user_api_key_budget_reservation"] = user_api_key_dict.budget_reservation
+        if user_api_key_dict.matched_model_access_groups:
+            data[_metadata_variable_name][MODEL_ACCESS_GROUP_METADATA_KEY] = (
+                user_api_key_dict.matched_model_access_groups
+            )
         # UserAPIKeyAuth object for MCP server access control
         data[_metadata_variable_name]["user_api_key_auth"] = user_api_key_dict.model_copy(
             update={

litellm/proxy/management_endpoints/config_override_endpoints.py

@@ -3,7 +3,7 @@ import json
 import os
 from collections.abc import Mapping, Sequence
 from datetime import datetime, timezone
-from typing import TYPE_CHECKING, Any, Final, Protocol
+from typing import TYPE_CHECKING, Final, Protocol
 
 from fastapi import APIRouter, Depends, Header, HTTPException
 from pydantic import BaseModel, TypeAdapter
@@ -36,10 +36,12 @@ from litellm.repositories.table_repositories import ConfigOverridesRepository
 from litellm.types.llms.custom_http import httpxSpecialProvider
 from litellm.types.proxy.management_endpoints.config_overrides import (
     ConfigOverrideSettingsResponse,
+    CyberArkConfig,
     HashicorpVaultConfig,
 )
 
 if TYPE_CHECKING:
+    from litellm.proxy.proxy_server import ProxyConfig
     from litellm.proxy.utils import PrismaClient
 
 router: Final = APIRouter()
@@ -83,18 +85,19 @@ def _log_audit_task_exception(task: "asyncio.Task[None]") -> None:
         return
     exc: Final = task.exception()
     if exc is not None:
-        verbose_proxy_logger.warning("Failed to write hashicorp-vault config audit log: %s", exc)
+        verbose_proxy_logger.warning("Failed to write config override audit log: %s", exc)
 
 
-async def _emit_hashicorp_vault_audit_log(
+async def _emit_config_override_audit_log(
     *,
+    object_id: str,
     action: AUDIT_ACTIONS,
     before_config: Mapping[str, object] | None,
     after_config: Mapping[str, object] | None,
     user_api_key_dict: UserAPIKeyAuth,
     litellm_changed_by: str | None,
 ) -> None:
-    """Emit an audit-log row for a /config_overrides/hashicorp_vault mutation.
+    """Emit an audit-log row for a /config_overrides/{object_id} mutation.
 
     Mirrors the ``store_audit_logs``-gated pattern from
     ``team_callback_endpoints.py``.  Captured under
@@ -118,7 +121,7 @@ async def _emit_hashicorp_vault_audit_log(
                 changed_by=litellm_changed_by or user_api_key_dict.user_id or litellm_proxy_admin_name,
                 changed_by_api_key=user_api_key_dict.api_key,
                 table_name=LitellmTableNames.CONFIG_OVERRIDES_TABLE_NAME,
-                object_id="hashicorp_vault",
+                object_id=object_id,
                 action=action,
                 updated_values=json.dumps({"config": _redact_config(after_config)}, default=str),
                 before_value=json.dumps({"config": _redact_config(before_config)}, default=str),
@@ -150,6 +153,24 @@ HASHICORP_SENSITIVE_FIELDS: Final[set[str]] = {
     "client_key",
 }
 
+# --- CyberArk Conjur constants ---
+
+CYBERARK_ENV_VAR_MAPPING: Final[dict[str, str]] = {  # mutable-ok: module-level env mapping
+    "cyberark_api_base": "CYBERARK_API_BASE",
+    "cyberark_account": "CYBERARK_ACCOUNT",
+    "cyberark_username": "CYBERARK_USERNAME",
+    "cyberark_api_key": "CYBERARK_API_KEY",
+    "client_cert": "CYBERARK_CLIENT_CERT",
+    "client_key": "CYBERARK_CLIENT_KEY",
+    "ssl_verify": "CYBERARK_SSL_VERIFY",
+    "refresh_interval": "CYBERARK_REFRESH_INTERVAL",
+}
+
+CYBERARK_SENSITIVE_FIELDS: Final[set[str]] = {  # mutable-ok: module-level constant, mirrors HASHICORP_SENSITIVE_FIELDS
+    "cyberark_api_key",
+    "client_key",
+}
+
 _sensitive_masker: Final = SensitiveDataMasker()
 
 
@@ -215,9 +236,12 @@ def _parse_config_value(raw: str | Mapping[str, object]) -> dict[str, object]:
     return dict(raw)
 
 
-def _set_env_vars(config_data: Mapping[str, object]) -> None:
-    """Set HCP_VAULT_* env vars from config data. Unsets vars for missing/None/empty fields."""
-    for field_name, env_var_name in HASHICORP_ENV_VAR_MAPPING.items():
+def _set_env_vars(
+    config_data: Mapping[str, object],
+    env_var_mapping: Mapping[str, str] = HASHICORP_ENV_VAR_MAPPING,
+) -> None:
+    """Set mapped env vars from config data. Unsets vars for missing/None/empty fields."""
+    for field_name, env_var_name in env_var_mapping.items():
         value = config_data.get(field_name)
         if value is not None and value != "":
             os.environ[env_var_name] = str(value)
@@ -225,13 +249,74 @@ def _set_env_vars(config_data: Mapping[str, object]) -> None:
             os.environ.pop(env_var_name, None)
 
 
-def _clear_hashicorp_vault_state(proxy_config: Any) -> None:
+def _clear_hashicorp_vault_state(proxy_config: "ProxyConfig") -> None:
     """Clear all Hashicorp Vault state: env vars, secret manager, and change-detection cache."""
     _set_env_vars({})
     if litellm._key_management_system == KeyManagementSystem.HASHICORP_VAULT:
         litellm.secret_manager_client = None
         litellm._key_management_system = None
-    proxy_config._last_hashicorp_vault_config = None
+    proxy_config._last_hashicorp_vault_config = None  # pyright: ignore[reportPrivateUsage]  # proxy-internal change-detection cache
+
+
+def _snapshot_cyberark_boot_env(proxy_config: "ProxyConfig") -> None:
+    """Capture deployment-provided CYBERARK_* env vars once, before the first DB-driven overwrite."""
+    if proxy_config._cyberark_boot_env is None:  # pyright: ignore[reportPrivateUsage]  # proxy-internal boot snapshot
+        proxy_config._cyberark_boot_env = _get_current_env_values(CYBERARK_ENV_VAR_MAPPING)  # pyright: ignore[reportPrivateUsage]  # proxy-internal boot snapshot
+
+
+def _restore_cyberark_runtime(proxy_config: "ProxyConfig", env_values: Mapping[str, str | None]) -> None:
+    """Restore CYBERARK_* env vars and reinitialize (or drop) the secret manager to match them."""
+    _set_env_vars(env_values, CYBERARK_ENV_VAR_MAPPING)
+    if env_values.get("cyberark_api_base"):
+        try:
+            proxy_config.initialize_secret_manager(key_management_system="cyberark")
+        except Exception:  # noqa: BLE001  # restore is best-effort; fall through to dropping the manager
+            verbose_proxy_logger.exception("Failed to restore previous CyberArk configuration")
+        else:
+            return
+    if litellm._key_management_system != KeyManagementSystem.CYBERARK:  # pyright: ignore[reportPrivateUsage]  # proxy-internal helper, mirrors hashicorp endpoint usage
+        return
+    litellm.secret_manager_client = None
+    litellm._key_management_system = None  # pyright: ignore[reportPrivateUsage]  # proxy-internal helper, mirrors hashicorp endpoint usage
+    # Force the vault reload to re-init from its own row so no manager is stranded inactive
+    proxy_config._last_hashicorp_vault_config = None  # pyright: ignore[reportPrivateUsage]  # proxy-internal change-detection cache
+    if os.environ.get("HCP_VAULT_ADDR"):
+        try:
+            proxy_config.initialize_secret_manager(key_management_system="hashicorp_vault")
+        except Exception:  # noqa: BLE001  # restore is best-effort; the vault reload loop retries from its own row
+            verbose_proxy_logger.exception("Failed to reinitialize Hashicorp Vault after CyberArk rollback")
+
+
+def _clear_cyberark_state(proxy_config: "ProxyConfig") -> None:
+    """Drop DB-driven CyberArk state, restoring deployment-provided env vars if any."""
+    boot_env: Final[Mapping[str, str | None]] = (
+        proxy_config._cyberark_boot_env or {}  # pyright: ignore[reportPrivateUsage]  # proxy-internal boot snapshot
+    )
+    _restore_cyberark_runtime(proxy_config, boot_env)
+    proxy_config._last_cyberark_config = None  # pyright: ignore[reportPrivateUsage]  # proxy-internal helper, mirrors hashicorp endpoint usage
+
+
+async def _persist_cyberark_config(
+    prisma_client: "PrismaClient",
+    proxy_config: "ProxyConfig",
+    config_data: Mapping[str, object],
+) -> dict[str, object]:
+    """Encrypt and upsert the CyberArk config row; returns the stored (encrypted) payload."""
+    encrypted_data: Final = proxy_config._encrypt_env_variables(dict(config_data))  # pyright: ignore[reportPrivateUsage]  # proxy-internal helper, mirrors hashicorp endpoint usage
+    config_value: Final = safe_dumps(encrypted_data)
+    await _config_overrides_table(prisma_client).upsert(
+        where={"config_type": "cyberark"},  # mutable-ok: prisma upsert payload
+        data={  # mutable-ok: prisma upsert payload
+            "create": {  # mutable-ok: prisma upsert payload
+                "config_type": "cyberark",
+                "config_value": config_value,
+            },
+            "update": {  # mutable-ok: prisma upsert payload
+                "config_value": config_value,
+            },
+        },
+    )
+    return safe_json_loads(config_value)
 
 
 # --- Hashicorp Vault endpoints ---
@@ -358,7 +443,8 @@ async def update_hashicorp_vault_config(
     # row was absent or its ``config_value`` was NULL.
     before_config: Final = existing_decrypted if existing_decrypted is not None else env_values
     action: Final[AUDIT_ACTIONS] = "updated" if existing_record is not None else "created"
-    await _emit_hashicorp_vault_audit_log(
+    await _emit_config_override_audit_log(
+        object_id="hashicorp_vault",
         action=action,
         before_config=before_config,
         after_config=config_data,
@@ -484,7 +570,8 @@ async def delete_hashicorp_vault_config(
     # Only emit audit log if a row was actually removed; an idempotent
     # delete on a non-existent row produces no security-relevant change.
     if deleted:
-        await _emit_hashicorp_vault_audit_log(
+        await _emit_config_override_audit_log(
+            object_id="hashicorp_vault",
             action="deleted",
             before_config=before_config,
             after_config=None,
@@ -529,7 +616,7 @@ async def test_hashicorp_vault_connection(
 
     # Step 1: Authenticate (exercises AppRole login, TLS cert login, or direct token)
     try:
-        headers: Final[dict[str, str]] = await asyncio.to_thread(client._get_request_headers)
+        headers: Final[Mapping[str, str]] = await asyncio.to_thread(client._get_request_headers)
     except Exception as e:
         raise HTTPException(
             status_code=502,
@@ -554,3 +641,298 @@ async def test_hashicorp_vault_connection(
         "status": "success",
         "message": f"Successfully connected to Vault at {client.vault_addr}",
     }
+
+
+# --- CyberArk Conjur endpoints ---
+
+
+@router.post(
+    "/config_overrides/cyberark",
+    tags=["Config Overrides"],  # mutable-ok: FastAPI route decorator metadata
+    dependencies=[Depends(user_api_key_auth)],  # mutable-ok: FastAPI route decorator metadata
+)
+async def update_cyberark_config(
+    config: CyberArkConfig,
+    user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),  # noqa: B008  # FastAPI dependency injection
+    litellm_changed_by: str | None = Header(
+        None,
+        description="The litellm-changed-by header enables tracking of actions performed by authorized users on behalf of other users, providing an audit trail for accountability",
+    ),
+) -> dict[str, str]:
+    """
+    Update CyberArk Conjur secret manager configuration.
+    Sets environment variables, encrypts sensitive fields, and stores in DB.
+    Reinitializes the secret manager on this pod.
+    """
+    from litellm.proxy.proxy_server import prisma_client, proxy_config
+
+    if user_api_key_dict.user_role != LitellmUserRoles.PROXY_ADMIN:
+        raise HTTPException(
+            status_code=403,
+            detail="Only admin users can update config overrides",
+        )
+
+    if prisma_client is None:
+        raise HTTPException(
+            status_code=500,
+            detail=CommonProxyErrors.db_not_connected_error.value,
+        )
+
+    config_data: dict[str, object] = config.model_dump(exclude_none=True)  # mutable-ok: merged  # rebind-ok: stripped
+
+    # Merge ALL fields the user didn't send: try DB first, fall back to env vars.
+    # Omitted field = keep existing; empty string = clear/remove the field.
+    existing_record: Final = await _config_overrides_table(prisma_client).find_unique(
+        where={"config_type": "cyberark"}  # mutable-ok: prisma where clause
+    )
+    existing_decrypted: dict[str, object] | None = None  # mutable-ok: DB payload  # rebind-ok: set when record exists
+    env_values: dict[str, str | None] = {}  # mutable-ok: env snapshot  # rebind-ok: populated when no DB record exists
+    if existing_record is not None and existing_record.config_value is not None:
+        existing_data: Final = _parse_config_value(existing_record.config_value)
+        existing_decrypted = proxy_config._decrypt_db_variables(existing_data)  # pyright: ignore[reportPrivateUsage]  # rebind-ok: populated when a prior record decrypts
+        for field in CYBERARK_ENV_VAR_MAPPING:
+            if field not in config_data and existing_decrypted.get(field):
+                config_data[field] = existing_decrypted[field]
+    else:
+        env_values = _get_current_env_values(CYBERARK_ENV_VAR_MAPPING)  # rebind-ok: populated when no DB record exists
+        for field in CYBERARK_ENV_VAR_MAPPING:
+            if field not in config_data and env_values.get(field):
+                config_data[field] = env_values[field]
+
+    config_data = {k: v for k, v in config_data.items() if v != ""}  # mutable-ok: dict  # rebind-ok: "" means clear
+
+    has_api_base: Final = bool(config_data.get("cyberark_api_base"))
+    has_api_key_auth: Final = bool(config_data.get("cyberark_api_key"))
+    has_tls_cert_auth: Final = bool(config_data.get("client_cert") and config_data.get("client_key"))
+
+    if not has_api_base:
+        raise HTTPException(
+            status_code=400,
+            detail="CyberArk API Base is required",
+        )
+
+    if not has_api_key_auth and not has_tls_cert_auth:
+        raise HTTPException(
+            status_code=400,
+            detail="At least one authentication method is required: "
+            "provide an API Key, or both Client Certificate and Client Key",
+        )
+
+    _snapshot_cyberark_boot_env(proxy_config)
+    previous_env: Final = _get_current_env_values(CYBERARK_ENV_VAR_MAPPING)
+    _set_env_vars(config_data, CYBERARK_ENV_VAR_MAPPING)
+
+    try:
+        proxy_config.initialize_secret_manager(key_management_system="cyberark")
+    except Exception as e:  # noqa: BLE001  # any init failure must roll back env vars
+        _set_env_vars(previous_env, CYBERARK_ENV_VAR_MAPPING)
+        verbose_proxy_logger.exception("Error reinitializing CyberArk secret manager: %s", str(e))
+        raise HTTPException(
+            status_code=500,
+            detail=f"Failed to initialize secret manager: {e}",
+        )
+
+    try:
+        proxy_config._last_cyberark_config = await _persist_cyberark_config(  # pyright: ignore[reportPrivateUsage]  # proxy-internal helper, mirrors hashicorp endpoint usage
+            prisma_client, proxy_config, config_data
+        )
+    except Exception as e:  # noqa: BLE001  # persistence failure must roll back the runtime state set above
+        _restore_cyberark_runtime(proxy_config, previous_env)
+        verbose_proxy_logger.exception("Error persisting CyberArk configuration: %s", str(e))
+        raise HTTPException(
+            status_code=500,
+            detail=f"Failed to persist CyberArk configuration: {e}",
+        )
+
+    before_config: Final = existing_decrypted if existing_decrypted is not None else env_values
+    action: Final[AUDIT_ACTIONS] = "updated" if existing_record is not None else "created"
+    await _emit_config_override_audit_log(
+        object_id="cyberark",
+        action=action,
+        before_config=before_config,
+        after_config=config_data,
+        user_api_key_dict=user_api_key_dict,
+        litellm_changed_by=litellm_changed_by,
+    )
+
+    return {  # mutable-ok: JSON response payload
+        "message": "CyberArk configuration updated successfully",
+        "status": "success",
+    }
+
+
+@router.get(
+    "/config_overrides/cyberark",
+    tags=["Config Overrides"],  # mutable-ok: FastAPI route decorator metadata
+    dependencies=[Depends(user_api_key_auth)],  # mutable-ok: FastAPI route decorator metadata
+    response_model=ConfigOverrideSettingsResponse,
+)
+async def get_cyberark_config(
+    user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),  # noqa: B008  # FastAPI dependency injection
+) -> ConfigOverrideSettingsResponse:
+    """
+    Get current CyberArk Conjur configuration.
+    Returns decrypted values from DB, or falls back to current env vars.
+    Sensitive fields are masked before leaving the server.
+    """
+    from litellm.proxy.management_endpoints.common_utils import (
+        _user_has_admin_view,  # pyright: ignore[reportPrivateUsage]  # proxy-internal helper, mirrors hashicorp endpoint usage
+    )
+    from litellm.proxy.proxy_server import prisma_client, proxy_config
+
+    if not _user_has_admin_view(user_api_key_dict):
+        raise HTTPException(
+            status_code=403,
+            detail="Only admin users can view config overrides",
+        )
+
+    if prisma_client is None:
+        raise HTTPException(
+            status_code=500,
+            detail=CommonProxyErrors.db_not_connected_error.value,
+        )
+
+    field_schema: Final = _build_field_schema(CyberArkConfig)
+
+    db_record: Final = await _config_overrides_table(prisma_client).find_unique(
+        where={"config_type": "cyberark"}
+    )  # mutable-ok: prisma where clause
+
+    if db_record is not None and db_record.config_value is not None:
+        config_data: Final = _parse_config_value(db_record.config_value)
+        decrypted_data: Final[Mapping[str, object]] = proxy_config._decrypt_db_variables(config_data)  # pyright: ignore[reportPrivateUsage]  # proxy-internal helper, mirrors hashicorp endpoint usage
+        masked_data: Final = _mask_sensitive_fields(decrypted_data, CYBERARK_SENSITIVE_FIELDS)
+
+        return ConfigOverrideSettingsResponse(
+            config_type="cyberark",
+            values=masked_data,
+            field_schema=field_schema,
+        )
+
+    env_values: Final = _get_current_env_values(CYBERARK_ENV_VAR_MAPPING)
+    masked_env_values: Final = _mask_sensitive_fields(env_values, CYBERARK_SENSITIVE_FIELDS)
+
+    return ConfigOverrideSettingsResponse(
+        config_type="cyberark",
+        values=masked_env_values,
+        field_schema=field_schema,
+    )
+
+
+@router.delete(
+    "/config_overrides/cyberark",
+    tags=["Config Overrides"],  # mutable-ok: FastAPI route decorator metadata
+    dependencies=[Depends(user_api_key_auth)],  # mutable-ok: FastAPI route decorator metadata
+)
+async def delete_cyberark_config(
+    user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),  # noqa: B008  # FastAPI dependency injection
+    litellm_changed_by: str | None = Header(
+        None,
+        description="The litellm-changed-by header enables tracking of actions performed by authorized users on behalf of other users, providing an audit trail for accountability",
+    ),
+) -> dict[str, str]:
+    """Delete CyberArk Conjur configuration. Idempotent."""
+    from litellm.proxy.proxy_server import prisma_client, proxy_config
+
+    if user_api_key_dict.user_role != LitellmUserRoles.PROXY_ADMIN:
+        raise HTTPException(
+            status_code=403,
+            detail="Only admin users can delete config overrides",
+        )
+
+    if prisma_client is None:
+        raise HTTPException(
+            status_code=500,
+            detail=CommonProxyErrors.db_not_connected_error.value,
+        )
+
+    existing_record: Final = await _config_overrides_table(prisma_client).find_unique(
+        where={"config_type": "cyberark"}  # mutable-ok: prisma where clause
+    )
+    before_config: dict[str, object] | None = None  # mutable-ok: audit snapshot  # rebind-ok: set when decrypts
+    if existing_record is not None and existing_record.config_value is not None:
+        try:
+            before_config = proxy_config._decrypt_db_variables(_parse_config_value(existing_record.config_value))  # pyright: ignore[reportPrivateUsage]  # rebind-ok: populated when the prior record decrypts
+        except Exception:  # noqa: BLE001  # undecryptable prior config must not block deletion
+            before_config = None  # rebind-ok: reset when decryption fails
+
+    deleted = False  # rebind-ok: set true once the DB row is removed
+    try:
+        await _config_overrides_table(prisma_client).delete(
+            where={"config_type": "cyberark"}
+        )  # mutable-ok: prisma where clause
+        deleted = True  # rebind-ok: set true once the DB row is removed
+    except RecordNotFoundError:
+        verbose_proxy_logger.debug("No existing CyberArk config record to delete")
+
+    _clear_cyberark_state(proxy_config)
+
+    if deleted:
+        await _emit_config_override_audit_log(
+            object_id="cyberark",
+            action="deleted",
+            before_config=before_config,
+            after_config=None,
+            user_api_key_dict=user_api_key_dict,
+            litellm_changed_by=litellm_changed_by,
+        )
+
+    return {  # mutable-ok: JSON response payload
+        "message": "CyberArk configuration deleted successfully",
+        "status": "success",
+    }
+
+
+@router.post(
+    "/config_overrides/cyberark/test_connection",
+    tags=["Config Overrides"],  # mutable-ok: FastAPI route decorator metadata
+    dependencies=[Depends(user_api_key_auth)],  # mutable-ok: FastAPI route decorator metadata
+)
+async def test_cyberark_connection(
+    user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),  # noqa: B008  # FastAPI dependency injection
+) -> dict[str, str]:
+    """
+    Test the connection to the currently configured CyberArk Conjur server.
+    Uses the already-initialized secret manager client. Does not modify any state.
+    """
+    from litellm.secret_managers.cyberark_secret_manager import CyberArkSecretManager
+
+    if user_api_key_dict.user_role != LitellmUserRoles.PROXY_ADMIN:
+        raise HTTPException(
+            status_code=403,
+            detail="Only admin users can test CyberArk connection",
+        )
+
+    client: Final = litellm.secret_manager_client
+    if not isinstance(client, CyberArkSecretManager):
+        raise HTTPException(
+            status_code=400,
+            detail="CyberArk is not configured. Save a configuration first.",
+        )
+
+    try:
+        headers: Final[Mapping[str, str]] = await asyncio.to_thread(client._get_request_headers)  # pyright: ignore[reportPrivateUsage]  # proxy-internal helper, mirrors hashicorp endpoint usage
+    except Exception as e:  # noqa: BLE001  # surface any auth failure as a 502 with detail
+        raise HTTPException(
+            status_code=502,
+            detail=f"CyberArk authentication failed: {e}",
+        )
+
+    try:
+        async_client: Final = get_async_httpx_client(
+            llm_provider=httpxSpecialProvider.SecretManager,
+            params={"ssl_verify": client.ssl_verify},  # mutable-ok: httpx client params
+        )
+        whoami_url: Final = f"{client.conjur_addr}/whoami"
+        response: Final = await async_client.get(whoami_url, headers=headers)
+        response.raise_for_status()
+    except Exception as e:  # noqa: BLE001  # surface any connectivity/TLS failure as a 502 with detail
+        raise HTTPException(
+            status_code=502,
+            detail=f"CyberArk token validation failed: {e}",
+        )
+
+    return {  # mutable-ok: JSON response payload
+        "status": "success",
+        "message": f"Successfully connected to CyberArk Conjur at {client.conjur_addr}",
+    }

litellm/proxy/management_endpoints/model_access_group_management_endpoints.py

@@ -2,18 +2,33 @@
 Allow proxy admin to manage model access groups
 
 Endpoints here:
-- POST /model_group/new - Create a new access group with multiple model names
+- POST /access_group/new - Create a new access group with multiple model names
+- GET /access_group/list - List every access group
+- GET /access_group/{access_group}/info - Read one access group, including its budget
+- PUT /access_group/{access_group}/update - Replace an access group's deployments
+- DELETE /access_group/{access_group}/delete - Delete an access group and its budget
+- GET /access_group/{access_group}/budget - Read an access group's shared budget and spend
+- PUT /access_group/{access_group}/budget - Set or replace an access group's shared budget
+- DELETE /access_group/{access_group}/budget - Clear an access group's shared budget
 """
 
 import json
 from collections.abc import Mapping, Sequence
-from typing import TYPE_CHECKING, Any, Final, Protocol
+from datetime import datetime
+from typing import TYPE_CHECKING, Annotated, Any, Final, Protocol
 
 from fastapi import APIRouter, Depends, HTTPException
+from typing_extensions import ReadOnly, TypedDict
 
 from litellm._logging import verbose_proxy_logger
 from litellm.proxy._types import UserAPIKeyAuth
 from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
+from litellm.proxy.common_utils.user_api_key_cache import (
+    UserApiKeyCache,
+    model_access_group_cache_key,
+    model_access_group_registry_cache_key,
+)
+from litellm.proxy.management_endpoints.common_utils import validate_budget_duration
 
 # Clear cache and reload models to pick up the access group changes
 from litellm.proxy.management_endpoints.model_management_endpoints import (
@@ -22,10 +37,16 @@ from litellm.proxy.management_endpoints.model_management_endpoints import (
     model_info_as_mapping,
     reload_serving_verdict,
 )
+from litellm.proxy.management_helpers.utils import handle_budget_for_entity
 from litellm.proxy.utils import PrismaClient
 from litellm.repositories.model_repository import ModelRepository
+from litellm.repositories.table_repositories import ModelAccessGroupBudgetRepository
 from litellm.types.proxy.management_endpoints.model_management_endpoints import (
+    AccessGroupBudget,
+    AccessGroupBudgetRequest,
+    AccessGroupBudgetResponse,
     AccessGroupInfo,
+    DeleteAccessGroupBudgetResponse,
     DeleteModelGroupResponse,
     ListAccessGroupsResponse,
     NewModelGroupRequest,
@@ -36,7 +57,43 @@ from litellm.types.proxy.management_endpoints.model_management_endpoints import
 if TYPE_CHECKING:
     from litellm import Router
 
-router: Final = APIRouter()
+router: Final = APIRouter(tags=["model management"])
+
+_AUTH_DEPENDENCIES: Final = (Depends(user_api_key_auth),)
+
+
+class _ErrorDetail(TypedDict):
+    error: ReadOnly[str]
+
+
+class _ModelAccessGroupWhere(TypedDict):
+    access_group_name: ReadOnly[str]
+
+
+class _BudgetInclude(TypedDict):
+    litellm_budget_table: ReadOnly[bool]
+
+
+class _ModelAccessGroupBudgetCreate(TypedDict):
+    access_group_name: ReadOnly[str]
+    budget_id: ReadOnly[str | None]
+    created_by: ReadOnly[str]
+    updated_by: ReadOnly[str]
+
+
+class _ModelAccessGroupBudgetUpdate(TypedDict):
+    budget_id: ReadOnly[str | None]
+    updated_by: ReadOnly[str]
+
+
+class _ModelAccessGroupBudgetUpsert(TypedDict):
+    create: ReadOnly[_ModelAccessGroupBudgetCreate]
+    update: ReadOnly[_ModelAccessGroupBudgetUpdate]
+
+
+def _http_error(status_code: int, message: str) -> HTTPException:
+    detail: Final[_ErrorDetail] = {"error": message}
+    return HTTPException(status_code=status_code, detail=detail)
 
 
 class _DeploymentRow(Protocol):
@@ -58,10 +115,140 @@ class _ModelTableClient(Protocol):
     async def update(self, *, where: Mapping[str, object], data: Mapping[str, object]) -> object: ...
 
 
+class _BudgetRow(Protocol):
+    @property
+    def budget_id(self) -> str: ...
+
+    @property
+    def max_budget(self) -> float | None: ...
+
+    @property
+    def soft_budget(self) -> float | None: ...
+
+    @property
+    def budget_duration(self) -> str | None: ...
+
+    @property
+    def budget_reset_at(self) -> datetime | None: ...
+
+
+class _ModelAccessGroupBudgetRow(Protocol):
+    @property
+    def spend(self) -> float: ...
+
+    @property
+    def budget_id(self) -> str | None: ...
+
+    @property
+    def litellm_budget_table(self) -> _BudgetRow | None: ...
+
+
+class _ModelAccessGroupBudgetTableClient(Protocol):
+    async def find_unique(
+        self, *, where: Mapping[str, object], include: Mapping[str, object] | None = None
+    ) -> _ModelAccessGroupBudgetRow | None: ...
+
+    async def upsert(
+        self,
+        *,
+        where: Mapping[str, object],
+        data: Mapping[str, object],
+        include: Mapping[str, object] | None = None,
+    ) -> _ModelAccessGroupBudgetRow: ...
+
+    async def delete(self, *, where: Mapping[str, object]) -> _ModelAccessGroupBudgetRow | None: ...
+
+
 def _model_table(prisma_client: PrismaClient) -> _ModelTableClient:
     return ModelRepository(prisma_client).table
 
 
+def _model_access_group_budget_table(prisma_client: PrismaClient) -> _ModelAccessGroupBudgetTableClient:
+    return ModelAccessGroupBudgetRepository(prisma_client).table
+
+
+def _prisma_client_or_500() -> PrismaClient:
+    from litellm.proxy.proxy_server import prisma_client
+
+    if prisma_client is None:
+        raise _http_error(500, "Database not connected.")
+    return prisma_client
+
+
+def _auth_cache() -> UserApiKeyCache:
+    from litellm.proxy.proxy_server import user_api_key_cache
+
+    return user_api_key_cache
+
+
+async def _evict_model_access_group_cache_keys(access_group: str, auth_cache: UserApiKeyCache) -> None:
+    """
+    Every endpoint that writes an access group budget row must call this, or the budget stays
+    unenforced until the TTL expires: auth gates the feature on a cached registry of the groups
+    that have a budget row, read cache-first with no freshness check.
+    """
+    from litellm.proxy.common_utils.auth_cache_invalidation_pubsub import (
+        evict_and_broadcast,
+    )
+
+    await evict_and_broadcast(
+        cache_keys=(model_access_group_cache_key(access_group), model_access_group_registry_cache_key()),
+        user_api_key_cache=auth_cache,
+    )
+
+
+async def _model_access_group_budget_row(
+    access_group: str, prisma_client: PrismaClient
+) -> _ModelAccessGroupBudgetRow | None:
+    where: Final[_ModelAccessGroupWhere] = {"access_group_name": access_group}
+    include: Final[_BudgetInclude] = {"litellm_budget_table": True}
+    return await _model_access_group_budget_table(prisma_client).find_unique(where=where, include=include)
+
+
+def _budget_or_none(row: _ModelAccessGroupBudgetRow | None) -> AccessGroupBudget | None:
+    budget: Final = row.litellm_budget_table if row is not None else None
+    if budget is None:
+        return None
+    return AccessGroupBudget(
+        budget_id=budget.budget_id,
+        max_budget=budget.max_budget,
+        soft_budget=budget.soft_budget,
+        budget_duration=budget.budget_duration,
+        budget_reset_at=budget.budget_reset_at,
+    )
+
+
+def _budget_response(access_group: str, row: _ModelAccessGroupBudgetRow | None) -> AccessGroupBudgetResponse:
+    return AccessGroupBudgetResponse(
+        access_group=access_group,
+        spend=row.spend if row is not None else 0.0,
+        budget=_budget_or_none(row),
+    )
+
+
+async def _delete_model_access_group_budget_row(
+    access_group: str, prisma_client: PrismaClient, auth_cache: UserApiKeyCache
+) -> bool:
+    """
+    Drop the group's budget row only, matching /tag/delete: the LiteLLM_BudgetTable row survives
+    because the link is ON DELETE SET NULL and a budget_id an admin passed in may be shared with
+    other entities.
+
+    Evicts unconditionally: a group with no row of its own can still be sitting in the cached
+    registry, so skipping the eviction when nothing was deleted would leave that stale.
+    """
+    where: Final[_ModelAccessGroupWhere] = {"access_group_name": access_group}
+    row: Final = await _model_access_group_budget_table(prisma_client).delete(where=where)
+    await _evict_model_access_group_cache_keys(access_group, auth_cache)
+    return row is not None
+
+
+async def _raise_404_if_model_access_group_missing(access_group: str, prisma_client: PrismaClient) -> None:
+    access_groups_map: Final = await get_all_access_groups_from_db(prisma_client=prisma_client)
+    if access_group not in access_groups_map:
+        raise _http_error(404, f"Access group '{access_group}' not found")
+
+
 def validate_models_exist(model_names: Sequence[str], llm_router: "Router | None") -> tuple[bool, Sequence[str]]:
     """
     Validate that all requested model names exist in the router.
@@ -356,13 +543,12 @@ async def get_all_access_groups_from_db(
 
 @router.post(
     "/access_group/new",
-    tags=["model management"],
-    dependencies=[Depends(user_api_key_auth)],
+    dependencies=_AUTH_DEPENDENCIES,
     response_model=NewModelGroupResponse,
 )
 async def create_model_group(
     data: NewModelGroupRequest,
-    user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),
+    user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)],
 ):
     """
     Create a new access group containing multiple model names.
@@ -503,12 +689,11 @@ async def create_model_group(
 
 @router.get(
     "/access_group/list",
-    tags=["model management"],
-    dependencies=[Depends(user_api_key_auth)],
+    dependencies=_AUTH_DEPENDENCIES,
     response_model=ListAccessGroupsResponse,
 )
 async def list_access_groups(
-    user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),
+    user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)],
 ):
     """
     List all access groups.
@@ -553,13 +738,12 @@ async def list_access_groups(
 
 @router.get(
     "/access_group/{access_group}/info",
-    tags=["model management"],
-    dependencies=[Depends(user_api_key_auth)],
+    dependencies=_AUTH_DEPENDENCIES,
     response_model=AccessGroupInfo,
 )
 async def get_access_group_info(
     access_group: str,
-    user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),
+    user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)],
 ):
     """
     Get information about a specific access group.
@@ -574,7 +758,7 @@ async def get_access_group_info(
     - access_group: str - The access group name (URL path parameter)
     
     Returns:
-    - AccessGroupInfo with the access group details
+    - AccessGroupInfo with the access group details, its shared budget and its spend
     
     Raises:
     - HTTPException 404: If access group not found
@@ -596,7 +780,15 @@ async def get_access_group_info(
                 detail={"error": f"Access group '{access_group}' not found"},
             )
 
-        return access_groups_map[access_group]
+        info: Final = access_groups_map[access_group]
+        budget_row: Final = await _model_access_group_budget_row(access_group, prisma_client)
+        return AccessGroupInfo(
+            access_group=info.access_group,
+            model_names=info.model_names,
+            deployment_count=info.deployment_count,
+            spend=budget_row.spend if budget_row is not None else 0.0,
+            budget=_budget_or_none(budget_row),
+        )
 
     except HTTPException:
         raise
@@ -610,14 +802,13 @@ async def get_access_group_info(
 
 @router.put(
     "/access_group/{access_group}/update",
-    tags=["model management"],
-    dependencies=[Depends(user_api_key_auth)],
+    dependencies=_AUTH_DEPENDENCIES,
     response_model=NewModelGroupResponse,
 )
 async def update_access_group(
     access_group: str,
     data: UpdateModelGroupRequest,
-    user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),
+    user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)],
 ):
     """
     Update an access group's model names.
@@ -765,13 +956,13 @@ async def update_access_group(
 
 @router.delete(
     "/access_group/{access_group}/delete",
-    tags=["model management"],
-    dependencies=[Depends(user_api_key_auth)],
+    dependencies=_AUTH_DEPENDENCIES,
     response_model=DeleteModelGroupResponse,
 )
 async def delete_access_group(
     access_group: str,
-    user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),
+    user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)],
+    auth_cache: Annotated[UserApiKeyCache, Depends(_auth_cache)],
 ):
     """
     Delete an access group.
@@ -835,6 +1026,13 @@ async def delete_access_group(
         removed_pairs: Final = tuple(pair for pair in removed if pair is not None)
         models_updated: Final = len(removed_pairs)
 
+        # Budget last, deliberately: failing here strands a budget row for a group already on no
+        # deployment (clutter), where the reverse order can leave a live group enforcing nothing.
+        # The LiteLLM_BudgetTable row it linked is left alone, as /tag/delete leaves a tag's.
+        await _delete_model_access_group_budget_row(
+            access_group=access_group, prisma_client=prisma_client, auth_cache=auth_cache
+        )
+
         # Clear cache and reload models to pick up the access group changes
         live_before_reload: Final = live_model_ids_snapshot()
         reload_outcome: Final = await clear_cache()
@@ -864,3 +1062,162 @@ async def delete_access_group(
             status_code=500,
             detail={"error": f"Failed to delete access group: {e}"},
         )
+
+
+@router.get(
+    "/access_group/{access_group}/budget",
+    dependencies=_AUTH_DEPENDENCIES,
+    response_model=AccessGroupBudgetResponse,
+)
+async def get_access_group_budget(
+    access_group: str,
+) -> AccessGroupBudgetResponse:
+    """
+    Get the shared budget of an access group, and the spend drawn against it.
+
+    Example:
+    ```bash
+    curl -X GET 'http://localhost:4000/access_group/production-models/budget' \\
+      -H 'Authorization: Bearer sk-1234'
+    ```
+
+    Parameters:
+    - access_group: str - The access group name (URL path parameter)
+
+    Returns:
+    - AccessGroupBudgetResponse; budget is null when the group has no budget set
+
+    Raises:
+    - HTTPException 404: If access group not found
+    """
+    prisma_client: Final = _prisma_client_or_500()
+    await _raise_404_if_model_access_group_missing(access_group=access_group, prisma_client=prisma_client)
+
+    return _budget_response(
+        access_group=access_group,
+        row=await _model_access_group_budget_row(access_group, prisma_client),
+    )
+
+
+@router.put(
+    "/access_group/{access_group}/budget",
+    dependencies=_AUTH_DEPENDENCIES,
+    response_model=AccessGroupBudgetResponse,
+)
+async def set_access_group_budget(
+    access_group: str,
+    data: AccessGroupBudgetRequest,
+    user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)],
+    auth_cache: Annotated[UserApiKeyCache, Depends(_auth_cache)],
+) -> AccessGroupBudgetResponse:
+    """
+    Set or replace the shared budget of an access group. Idempotent.
+
+    Every key that can reach a model in the group draws from this one budget.
+
+    Example:
+    ```bash
+    curl -X PUT 'http://localhost:4000/access_group/production-models/budget' \\
+      -H 'Authorization: Bearer sk-1234' \\
+      -H 'Content-Type: application/json' \\
+      -d '{
+        "max_budget": 100.0,
+        "budget_duration": "30d"
+      }'
+    ```
+
+    Parameters:
+    - access_group: str - The access group name (URL path parameter)
+    - max_budget: Optional[float] - Requests fail once the group's shared spend exceeds this
+    - soft_budget: Optional[float] - Fires an alert when reached; requests still succeed
+    - budget_duration: Optional[str] - Frequency of resetting the group's spend (e.g. '30d')
+    - budget_id: Optional[str] - Link an existing budget instead of creating one
+
+    Returns:
+    - AccessGroupBudgetResponse with the stored budget and current spend
+
+    Raises:
+    - HTTPException 400: If no budget field is given, or budget_duration cannot be parsed
+    - HTTPException 404: If access group not found
+    """
+    from litellm.proxy.proxy_server import litellm_proxy_admin_name
+
+    prisma_client: Final = _prisma_client_or_500()
+    if not data.model_dump(exclude_none=True):
+        raise _http_error(400, "One of max_budget, soft_budget, budget_duration or budget_id is required")
+    validate_budget_duration(data.budget_duration)
+    await _raise_404_if_model_access_group_missing(access_group=access_group, prisma_client=prisma_client)
+
+    existing_row: Final = await _model_access_group_budget_row(access_group, prisma_client)
+    budget_id: Final = await handle_budget_for_entity(
+        data=data,
+        existing_budget_id=existing_row.budget_id if existing_row is not None else None,
+        user_api_key_dict=user_api_key_dict,
+        prisma_client=prisma_client,
+        litellm_proxy_admin_name=litellm_proxy_admin_name,
+    )
+    actor: Final = user_api_key_dict.user_id or litellm_proxy_admin_name
+    upsert_data: Final[_ModelAccessGroupBudgetUpsert] = {
+        "create": {
+            "access_group_name": access_group,
+            "budget_id": budget_id,
+            "created_by": actor,
+            "updated_by": actor,
+        },
+        "update": {"budget_id": budget_id, "updated_by": actor},
+    }
+    where: Final[_ModelAccessGroupWhere] = {"access_group_name": access_group}
+    include: Final[_BudgetInclude] = {"litellm_budget_table": True}
+    row: Final = await _model_access_group_budget_table(prisma_client).upsert(
+        where=where, data=upsert_data, include=include
+    )
+    await _evict_model_access_group_cache_keys(access_group, auth_cache)
+
+    verbose_proxy_logger.info("Set budget %s on access group '%s'", budget_id, access_group)
+    return _budget_response(access_group=access_group, row=row)
+
+
+@router.delete(
+    "/access_group/{access_group}/budget",
+    dependencies=_AUTH_DEPENDENCIES,
+    response_model=DeleteAccessGroupBudgetResponse,
+)
+async def delete_access_group_budget(
+    access_group: str,
+    auth_cache: Annotated[UserApiKeyCache, Depends(_auth_cache)],
+) -> DeleteAccessGroupBudgetResponse:
+    """
+    Clear the shared budget of an access group, leaving the group itself in place.
+
+    Example:
+    ```bash
+    curl -X DELETE 'http://localhost:4000/access_group/production-models/budget' \\
+      -H 'Authorization: Bearer sk-1234'
+    ```
+
+    Parameters:
+    - access_group: str - The access group name (URL path parameter)
+
+    Returns:
+    - DeleteAccessGroupBudgetResponse; budget_deleted is false when there was nothing to clear
+
+    Raises:
+    - HTTPException 404: If access group not found
+    """
+    prisma_client: Final = _prisma_client_or_500()
+    await _raise_404_if_model_access_group_missing(access_group=access_group, prisma_client=prisma_client)
+
+    budget_deleted: Final = await _delete_model_access_group_budget_row(
+        access_group=access_group,
+        prisma_client=prisma_client,
+        auth_cache=auth_cache,
+    )
+    return DeleteAccessGroupBudgetResponse(
+        access_group=access_group,
+        budget_deleted=budget_deleted,
+        message=(
+            f"Budget for access group '{access_group}' deleted successfully"
+            if budget_deleted
+            else f"Access group '{access_group}' has no budget to delete"
+        ),
+    )

litellm/proxy/pass_through_endpoints/managed_id_rewriter.py

@@ -49,6 +49,7 @@ from litellm._logging import verbose_proxy_logger
 from litellm.llms.base_llm.managed_resources.isolation import (
     build_owner_filter,
     can_access_resource,
+    resolve_resource_owner_id,
 )
 from litellm.proxy._types import UserAPIKeyAuth
 from litellm.proxy.batches_endpoints.common_utils import validate_batch_list_limit
@@ -686,7 +687,7 @@ async def _mint_or_reuse_object(
                     "file_object": json.dumps(body_snapshot),
                     "model_object_id": namespaced_model_object_id,
                     "file_purpose": file_purpose,
-                    "created_by": user_api_key_dict.user_id,
+                    "created_by": resolve_resource_owner_id(user_api_key_dict),
                     "team_id": user_api_key_dict.team_id,
                     "updated_by": user_api_key_dict.user_id,
                 },

litellm/proxy/pass_through_endpoints/pass_through_endpoints.py

@@ -47,6 +47,7 @@ from litellm.litellm_core_utils.core_helpers import (
     get_metadata_variable_name_from_kwargs,
     get_or_create_metadata_bucket,
 )
+from litellm.litellm_core_utils.internal_call_metadata import MODEL_ACCESS_GROUP_METADATA_KEY
 from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
 from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER
 from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
@@ -577,6 +578,7 @@ class HttpPassThroughEndpointHelpers(BasePassthroughUtils):
         _metadata["user_api_key"] = user_api_key_dict.api_key
         _metadata["litellm_parent_otel_span"] = user_api_key_dict.parent_otel_span
         _metadata["user_api_key_budget_reservation"] = user_api_key_dict.budget_reservation
+        _metadata[MODEL_ACCESS_GROUP_METADATA_KEY] = user_api_key_dict.matched_model_access_groups
         # The per-model budget counters are keyed off these. get_sanitized_user_information_from_key
         # returns StandardLoggingUserAPIKeyMetadata, which carries no budget field, so without this
         # the post-call increment finds nothing and every passthrough request goes untracked and

litellm/proxy/policy_engine/pipeline_executor.py

@@ -185,7 +185,7 @@ class PipelineExecutor:
             # snapshot instead of `data` (which earlier pass_data steps in
             # this same pipeline may have already rewritten), same reason
             # the normal sequential/parallel guardrail loops do this.
-            scans_raw_request: Final = getattr(callback, "scan_raw_request", False)
+            scans_raw_request: Final = callback.scan_raw_request
             hook_input: Final[dict] = (  # mutable-ok: same request-payload shape as data
                 independent_snapshot(raw_request_snapshot)
                 if scans_raw_request and raw_request_snapshot is not None

litellm/proxy/proxy_server.py

@@ -382,6 +382,8 @@ from litellm.proxy.common_utils.user_api_key_cache import (
     UserApiKeyCache,
     end_user_cache_key,
     get_management_object_ttl,
+    model_access_group_cache_key,
+    model_access_group_spend_counter_key,
     tag_cache_key,
 )
 from litellm.proxy.config_resolvers import resolve_fields
@@ -396,6 +398,9 @@ from litellm.proxy.db.db_transaction_queue.spend_log_cleanup import (
     SPEND_LOG_CLEANUP_BOUND_SETTINGS,
     SpendLogCleanup,
 )
+from litellm.proxy.db.db_transaction_queue.window_spend_update_queue import (
+    build_window_spend_transaction,
+)
 from litellm.proxy.db.exception_handler import (
     PrismaDBExceptionHandler,
     call_with_db_reconnect_retry,
@@ -2431,6 +2436,7 @@ async def get_current_spend(
     max_budget: float | None = None,
     window_entity_type: str | None = None,
     window_entity_id: str | None = None,
+    window_duration: str | None = None,
     window_start: datetime | None = None,
     fallback_authoritative: bool = False,
 ) -> float:
@@ -2455,7 +2461,8 @@ async def get_current_spend(
     runs and a key can leak spend past ``max_budget`` indefinitely. The
     authoritative source depends on the counter: primary key/team/user/org
     counters read the DB row; per-window counters (``window_start`` supplied)
-    aggregate spend logs; end-user/tag counters have no DB row, so the caller's
+    read the maintained window-spend row and only aggregate spend logs when
+    that row is missing or stale; end-user/tag counters have no DB row, so the caller's
     ``fallback_spend`` (loaded fresh in auth) is authoritative. The DB read is
     skipped for healthy primary counters (counter at or above recorded spend)
     and cached in-process for a few seconds, so a persistently stale counter
@@ -2480,6 +2487,7 @@ async def get_current_spend(
             counter_key=counter_key,
             window_entity_type=window_entity_type,
             window_entity_id=window_entity_id,
+            window_duration=window_duration,
             window_start=window_start,
         )
         if authoritative is not None:
@@ -2561,6 +2569,7 @@ async def _authoritative_floor_spend(
     counter_key: str,
     window_entity_type: str | None = None,
     window_entity_id: str | None = None,
+    window_duration: str | None = None,
     window_start: datetime | None = None,
 ) -> float | None:
     marker_key: Final = f"spend_db_floor:{counter_key}"
@@ -2575,10 +2584,11 @@ async def _authoritative_floor_spend(
         and window_entity_id is not None
         and window_start is not None
     ):
-        db_spend = await SpendCounterReseed.window_from_spend_logs(
+        db_spend = await SpendCounterReseed.window_from_db(
             prisma_client=prisma_client,
             entity_type=window_entity_type,
             entity_id=window_entity_id,
+            window_duration=window_duration,
             window_start=window_start,
         )
     if db_spend is None:
@@ -2648,6 +2658,9 @@ async def increment_spend_counters(
     budget_reservation: dict | None = None,
     end_user_id: str | None = None,
     tags: list[str] | None = None,
+    request_id: str | None = None,
+    request_started_at: datetime | None = None,
+    model_access_groups: Sequence[str] | None = None,
 ):
     """
     Atomically increment spend counters for budget enforcement.
@@ -2701,15 +2714,28 @@ async def increment_spend_counters(
             return
         for window in key_budget_limits:
             duration = window["budget_duration"] if isinstance(window, dict) else window.budget_duration
+            key_window_reset_at = window.get("reset_at") if isinstance(window, dict) else window.reset_at
             key_window_counter = f"spend:key:{hashed_token}:window:{duration}"
+            key_window_start = get_budget_window_start(window)
             if key_window_counter not in reserved_counter_keys:
                 await _init_and_increment_window_spend_counter(
                     counter_key=key_window_counter,
                     entity_type="Key",
                     entity_id=hashed_token,
-                    window_start=get_budget_window_start(window),
+                    window_duration=duration,
+                    window_start=key_window_start,
                     increment=cost,
                 )
+            await _enqueue_window_spend_row_update(
+                entity_type=Litellm_EntityType.KEY,
+                entity_id=hashed_token,
+                reset_at=key_window_reset_at,
+                window_duration=duration,
+                window_start=key_window_start,
+                increment=cost,
+                request_id=request_id,
+                request_started_at=request_started_at,
+            )
 
     async def _team_scope(scope_team_id: str) -> None:
         team_counter_key: Final = f"spend:team:{scope_team_id}"
@@ -2732,15 +2758,28 @@ async def increment_spend_counters(
             return
         for window in team_budget_limits:
             duration = window["budget_duration"] if isinstance(window, dict) else window.budget_duration
+            team_window_reset_at = window.get("reset_at") if isinstance(window, dict) else window.reset_at
             team_window_counter = f"spend:team:{scope_team_id}:window:{duration}"
+            team_window_start = get_budget_window_start(window)
             if team_window_counter not in reserved_counter_keys:
                 await _init_and_increment_window_spend_counter(
                     counter_key=team_window_counter,
                     entity_type="Team",
                     entity_id=scope_team_id,
-                    window_start=get_budget_window_start(window),
+                    window_duration=duration,
+                    window_start=team_window_start,
                     increment=cost,
                 )
+            await _enqueue_window_spend_row_update(
+                entity_type=Litellm_EntityType.TEAM,
+                entity_id=scope_team_id,
+                reset_at=team_window_reset_at,
+                window_duration=duration,
+                window_start=team_window_start,
+                increment=cost,
+                request_id=request_id,
+                request_started_at=request_started_at,
+            )
 
     async def _team_member_scope(scope_user_id: str, scope_team_id: str) -> None:
         team_member_counter_key: Final = f"spend:team_member:{scope_user_id}:{scope_team_id}"
@@ -2777,6 +2816,13 @@ async def increment_spend_counters(
             )
             if end_user_id is not None or tags is not None
             else None,
+            _increment_model_access_group_spend_counters(
+                model_access_groups=model_access_groups,
+                response_cost=cost,
+                reserved_counter_keys=reserved_counter_keys,
+            )
+            if model_access_groups
+            else None,
             _increment_org_spend_counter(
                 org_id=org_id,
                 response_cost=cost,
@@ -2865,6 +2911,33 @@ async def _increment_end_user_and_tag_spend_counters(
         )
 
 
+async def _increment_model_access_group_spend_counters(
+    model_access_groups: Sequence[object],
+    response_cost: float,
+    reserved_counter_keys: set[str],
+) -> None:
+    """Charge the model access groups that authorized this request.
+
+    Without this the counter auth reads is written only by the reservation path, so
+    ``disable_budget_reservation`` would leave ``_model_access_group_max_budget_check`` enforcing
+    against the DB row's spend, which lags by up to the cache TTL.
+
+    Typed ``object`` rather than ``str`` because the names reach the cost callback out of request
+    metadata, which the coercion upstream filters to a list but not to strings. A non-string that
+    slipped through would build a counter key nothing else ever reads.
+    """
+    unique_groups: Final = tuple(
+        dict.fromkeys(group for group in model_access_groups if group and isinstance(group, str))
+    )
+    for group in unique_groups:
+        await _init_and_increment_unreserved_spend_counter(
+            counter_key=model_access_group_spend_counter_key(group),
+            source_cache_key=model_access_group_cache_key(group),
+            increment=response_cost,
+            reserved_counter_keys=reserved_counter_keys,
+        )
+
+
 async def _increment_org_spend_counter(
     org_id: str | None,
     response_cost: float,
@@ -2925,10 +2998,62 @@ async def _init_and_increment_spend_counter(
     await _increment_spend_counter_cache(counter_key=counter_key, increment=increment)
 
 
+async def _enqueue_window_spend_row_update(
+    entity_type: Litellm_EntityType,
+    entity_id: str,
+    reset_at: datetime | str | None,
+    window_duration: str,
+    window_start: datetime | None,
+    increment: float,
+    request_id: str | None,
+    request_started_at: datetime | None,
+) -> None:
+    """Queue this request's cost against the LiteLLM_BudgetWindowSpend row for
+    the window, so enforcement can read a maintained total instead of
+    aggregating LiteLLM_SpendLogs.
+
+    request_id is the LiteLLM_SpendLogs id this cost was recorded under and
+    request_started_at its startTime; the flush uses them to keep the one-time
+    seed from counting a request that its increment already covers.
+
+    Enqueued even when the cache increment was skipped for a reserved counter:
+    the reservation only pre-charged the counter, and the row still owes the
+    actual cost.
+
+    Windows with no reset_at slide with wall clock, so their window_start moves
+    on every request and no single row can represent them. Those are left to
+    the read path's LiteLLM_SpendLogs fallback rather than rewritten per
+    request.
+    """
+    if window_start is None or not reset_at:
+        return
+    try:
+        await proxy_logging_obj.db_spend_update_writer.window_spend_update_queue.add_update(
+            build_window_spend_transaction(
+                entity_type=entity_type.value,
+                entity_id=entity_id,
+                window_duration=window_duration,
+                window_start=window_start,
+                spend=increment,
+                request_id=request_id,
+                started_at=request_started_at,
+            )
+        )
+    except Exception as e:  # noqa: BLE001  # spend tracking must never fail the cost callback
+        verbose_proxy_logger.debug(
+            "Unable to enqueue budget window spend update for %s=%s window=%s: %s",
+            entity_type.value,
+            entity_id,
+            window_duration,
+            e,
+        )
+
+
 async def _init_and_increment_window_spend_counter(
     counter_key: str,
     entity_type: str,
     entity_id: str,
+    window_duration: str | None,
     window_start: datetime | None,
     increment: float,
 ):
@@ -2943,6 +3068,7 @@ async def _init_and_increment_window_spend_counter(
         counter_key=counter_key,
         entity_type=entity_type,
         entity_id=entity_id,
+        window_duration=window_duration,
         window_start=window_start,
     )
     if initialized is False:
@@ -2988,6 +3114,7 @@ async def _ensure_window_spend_counter_initialized(
     counter_key: str,
     entity_type: str,
     entity_id: str,
+    window_duration: str | None,
     window_start: datetime,
 ) -> bool:
     is_warm: Final = await _is_spend_counter_cache_warm(counter_key=counter_key)
@@ -3000,6 +3127,7 @@ async def _ensure_window_spend_counter_initialized(
         counter_key=counter_key,
         entity_type=entity_type,
         entity_id=entity_id,
+        window_duration=window_duration,
         window_start=window_start,
     )
     if window_spend is None:
@@ -4281,6 +4409,8 @@ class ProxyConfig:
         self.config: dict[str, Any] = {}
         self._last_semantic_filter_config: dict[str, object] | None = None
         self._last_hashicorp_vault_config: dict[str, object] | None = None
+        self._last_cyberark_config: dict[str, object] | None = None  # mutable-ok: change-detection cache
+        self._cyberark_boot_env: dict[str, str | None] | None = None  # mutable-ok: deployment env snapshot, set once
         self.worker_registry: list[WorkerRegistryEntry] = []
         self.config_sync_subscriber: ConfigSyncSubscriber | None = None
         self.auth_cache_invalidation_subscriber: AuthCacheInvalidationSubscriber | None = None
@@ -6764,9 +6894,18 @@ class ProxyConfig:
         - list: the rows (may be empty if no models exist)
         - None: signals a DB fetch *failure* — callers must not treat this
           as "all models deleted" and must not evict existing router deployments.
+
+        Pinned to the writer DB: this read reconciles the router against the rows a
+        model write just committed, and reading it through a lagging read replica
+        makes the write-triggered reload report its own durable write as missing
+        (#38556). It also keeps a stale replica snapshot from evicting a deployment
+        another pod just added. While the writer is degraded the pin yields to the
+        replica so reader-only mode keeps loading DB-backed models.
         """
         try:
-            new_models: Final[Sequence[_ProxyModelRow]] = await ModelRepository(prisma_client).table.find_many()
+            new_models: Final[Sequence[_ProxyModelRow]] = await ModelRepository(
+                WriterPinnedClient(prisma_client.db)
+            ).table.find_many()
             return new_models
         except Exception as e:
             verbose_proxy_logger.exception(
@@ -6968,6 +7107,7 @@ class ProxyConfig:
 
         if self._should_load_db_object(object_type="config_overrides"):
             await self._init_hashicorp_vault_config_override(prisma_client=prisma_client)
+            await self._init_cyberark_config_override(prisma_client=prisma_client)
 
         await self._apply_safe_litellm_settings_overrides_from_db(prisma_client=prisma_client)
 
@@ -7132,6 +7272,64 @@ class ProxyConfig:
                 str(e),
             )
 
+    async def _init_cyberark_config_override(self, prisma_client: PrismaClient) -> None:
+        """
+        Load CyberArk Conjur config override from DB.
+        Decrypts sensitive fields, sets CYBERARK_* env vars, and reinitializes the secret manager.
+        Called periodically via _init_non_llm_objects_in_db to sync config across pods.
+        """
+        from litellm.proxy.management_endpoints.config_override_endpoints import (
+            CYBERARK_ENV_VAR_MAPPING,
+            _clear_cyberark_state,  # pyright: ignore[reportPrivateUsage]  # module-internal helper shared with the endpoint module
+            _get_current_env_values,  # pyright: ignore[reportPrivateUsage]  # module-internal helper shared with the endpoint module
+            _parse_config_value,  # pyright: ignore[reportPrivateUsage]  # module-internal helper shared with the endpoint module
+            _set_env_vars,  # pyright: ignore[reportPrivateUsage]  # module-internal helper shared with the endpoint module
+            _snapshot_cyberark_boot_env,  # pyright: ignore[reportPrivateUsage]  # module-internal helper shared with the endpoint module
+        )
+
+        try:
+            db_record: Final[_ConfigOverridesRow | None] = cast(  # cast-ok: prisma Json stub is `str`, runtime dict
+                "_ConfigOverridesRow | None",
+                await call_with_db_reconnect_retry(
+                    prisma_client,
+                    lambda: ConfigOverridesRepository(prisma_client).table.find_unique(
+                        where={"config_type": "cyberark"}  # mutable-ok: prisma where clause
+                    ),
+                    reason="init_cyberark_config_override_lookup_failure",
+                ),
+            )
+
+            if db_record is None or db_record.config_value is None:
+                if self._last_cyberark_config is not None:
+                    _clear_cyberark_state(self)
+                return
+
+            config_data: Final = _parse_config_value(db_record.config_value)
+
+            # Skip reinit if config hasn't changed since last poll
+            if self._last_cyberark_config == config_data:
+                return
+
+            decrypted_data: Final = self._decrypt_db_variables(config_data)
+
+            _snapshot_cyberark_boot_env(self)
+            previous_env: Final = _get_current_env_values(CYBERARK_ENV_VAR_MAPPING)
+            _set_env_vars(decrypted_data, CYBERARK_ENV_VAR_MAPPING)
+
+            try:
+                self.initialize_secret_manager(key_management_system="cyberark")
+            except Exception:
+                _set_env_vars(previous_env, CYBERARK_ENV_VAR_MAPPING)
+                raise
+
+            self._last_cyberark_config = config_data.copy()
+            verbose_proxy_logger.debug("CyberArk config override loaded from DB")
+        except Exception as e:  # noqa: BLE001  # any DB/decrypt/init failure must not break proxy boot
+            verbose_proxy_logger.exception(
+                "Error loading CyberArk config override from DB: %s",
+                str(e),
+            )
+
     async def check_periodic_reloads(self, prisma_client: PrismaClient):
         """
         Run the admin-configured periodic model cost map reload.
@@ -12013,6 +12211,7 @@ async def run_thread(
 # )
 # async def get_available_routes(user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth)):
 from litellm.llms.base_llm.base_utils import BaseTokenCounter
+from litellm.proxy.db.routing_prisma_wrapper import WriterPinnedClient
 from litellm.repositories.config_repository import ConfigRepository
 from litellm.repositories.model_repository import ModelRepository
 from litellm.repositories.table_repositories import (

litellm/proxy/schema.prisma

@@ -29,6 +29,7 @@ model LiteLLM_BudgetTable {
   keys LiteLLM_VerificationToken[] // multiple keys can have the same budget
   end_users LiteLLM_EndUserTable[] // multiple end-users can have the same budget
   tags LiteLLM_TagTable[] // multiple tags can have the same budget
+  model_access_groups LiteLLM_ModelAccessGroupBudgetTable[] // multiple model access groups can have the same budget
   team_membership LiteLLM_TeamMembership[] // budgets of Users within a Team 
   organization_membership LiteLLM_OrganizationMembership[] // budgets of Users within a Organization
 }
@@ -585,6 +586,20 @@ model LiteLLM_EndUserTable {
   blocked Boolean @default(false)
 }
 
+// Budget and shared spend for a model access group. The groups themselves are not rows anywhere:
+// they are free-text strings in LiteLLM_ProxyModelTable.model_info.access_groups, so a row here
+// exists only once someone gives that group a budget.
+model LiteLLM_ModelAccessGroupBudgetTable {
+  access_group_name String @id
+  spend Float @default(0.0)
+  budget_id String?
+  litellm_budget_table LiteLLM_BudgetTable? @relation(fields: [budget_id], references: [budget_id])
+  created_at DateTime @default(now()) @map("created_at")
+  created_by String?
+  updated_at DateTime @default(now()) @updatedAt
+  updated_by String?
+}
+
 // Track tags with budgets and spend
 model LiteLLM_TagTable {
   tag_name String @id
@@ -649,6 +664,18 @@ model LiteLLM_SpendLogs {
   @@index([session_id])
 }
 
+model LiteLLM_BudgetWindowSpend {
+  entity_type     String
+  entity_id       String
+  window_duration String
+  window_start    DateTime
+  spend           Float    @default(0.0)
+  created_at      DateTime @default(now())
+  updated_at      DateTime @updatedAt
+
+  @@id([entity_type, entity_id, window_duration])
+}
+
 // View spend, model, api_key per request
 model LiteLLM_ErrorLogs {
   request_id          String   @id @default(uuid())

litellm/proxy/spend_tracking/budget_reservation.py

@@ -12,7 +12,6 @@ from fastapi import HTTPException, status
 
 import litellm
 from litellm._logging import verbose_proxy_logger
-from litellm.caching import DualCache
 from litellm.litellm_core_utils.duration_parser import duration_in_seconds
 from litellm.litellm_core_utils.llm_cost_calc.tiered_pricing import select_tier_for_input, tier_rate
 from litellm.proxy._types import (
@@ -26,12 +25,16 @@ from litellm.proxy.auth.auth_utils import get_model_from_request
 from litellm.proxy.auth.budget_throttle import should_throttle_budget_exceeded
 from litellm.proxy.auth.route_checks import RouteChecks
 from litellm.proxy.common_utils.user_api_key_cache import (
+    UserApiKeyCache,
     end_user_cache_key,
+    model_access_group_cache_key,
+    model_access_group_spend_counter_key,
     tag_cache_key,
     team_membership_reservation_cache_key,
 )
 from litellm.proxy.utils import PrismaClient, ProxyLogging
 from litellm.router import Router
+from litellm.types.proxy.model_access_group_budget import ModelAccessGroupBudget
 
 
 @dataclass
@@ -43,6 +46,7 @@ class _BudgetCounter:
     entity_id: str
     source_cache_key: str | None = None
     spend_log_entity_id: str | None = None
+    window_duration: str | None = None
     window_start: datetime | None = None
 
 
@@ -53,6 +57,7 @@ _COUNTER_ENTITY_TYPES: Final[Mapping[str, str]] = {
     "User": Litellm_EntityType.USER.value,
     "EndUser": Litellm_EntityType.END_USER.value,
     "Tag": Litellm_EntityType.TAG.value,
+    "Model access group": Litellm_EntityType.MODEL_ACCESS_GROUP.value,
     "Organization": Litellm_EntityType.ORGANIZATION.value,
 }
 
@@ -158,7 +163,7 @@ async def reserve_budget_for_request(
     team_object: LiteLLM_TeamTable | None,
     user_object: LiteLLM_UserTable | None,
     prisma_client: PrismaClient | None,
-    user_api_key_cache: DualCache,
+    user_api_key_cache: UserApiKeyCache,
     proxy_logging_obj: ProxyLogging,
     end_user_id: str | None = None,
     end_user_object: object = None,
@@ -348,7 +353,7 @@ async def _get_budget_counters(
     team_object: LiteLLM_TeamTable | None,
     user_object: LiteLLM_UserTable | None,
     prisma_client: PrismaClient | None,
-    user_api_key_cache: DualCache,
+    user_api_key_cache: UserApiKeyCache,
     proxy_logging_obj: ProxyLogging,
     end_user_id: str | None = None,
     end_user_object: object = None,
@@ -437,6 +442,14 @@ async def _get_budget_counters(
         )
     )
 
+    counters.extend(
+        await _get_model_access_group_budget_counters(
+            valid_token=valid_token,
+            prisma_client=prisma_client,
+            user_api_key_cache=user_api_key_cache,
+        )
+    )
+
     team_member_counter: Final = await _get_team_member_budget_counter(
         valid_token=valid_token,
         team_object=team_object,
@@ -491,7 +504,7 @@ async def _get_end_user_budget_counter(
 async def _get_tag_budget_counters(
     request_body: dict,
     prisma_client: PrismaClient | None,
-    user_api_key_cache: DualCache,
+    user_api_key_cache: UserApiKeyCache,
     proxy_logging_obj: ProxyLogging,
 ) -> list[_BudgetCounter]:
     from litellm.proxy.auth.auth_checks import get_tag_objects_batch
@@ -530,6 +543,46 @@ async def _get_tag_budget_counters(
     return counters
 
 
+async def _get_model_access_group_budget_counters(
+    valid_token: UserAPIKeyAuth,
+    prisma_client: PrismaClient | None,
+    user_api_key_cache: UserApiKeyCache,
+) -> list[_BudgetCounter]:
+    """Reservation counters for the model access groups that authorized this request.
+
+    The names come off the auth object rather than the request body: ``common_checks`` already
+    resolved which granted groups serve the requested model, and re-deriving that here would both
+    duplicate the walk and risk disagreeing with what the spend writer attributes.
+    """
+    from litellm.proxy.auth.auth_checks import get_model_access_group_budgets_batch
+
+    group_names: Final = tuple(dict.fromkeys(valid_token.matched_model_access_groups or ()))
+    if not group_names:
+        return []
+
+    budgets: Final = await get_model_access_group_budgets_batch(
+        access_group_names=group_names,
+        prisma_client=prisma_client,
+        user_api_key_cache=user_api_key_cache,
+    )
+    candidates: Final = (_model_access_group_counter(group, budgets.get(group)) for group in group_names)
+    return [counter for counter in candidates if counter is not None]
+
+
+def _model_access_group_counter(group: str, budget: ModelAccessGroupBudget | None) -> _BudgetCounter | None:
+    """A counter for one group, or nothing when the group carries no budget to reserve against."""
+    if budget is None or budget.max_budget is None or budget.max_budget <= 0:
+        return None
+    return _BudgetCounter(
+        counter_key=model_access_group_spend_counter_key(group),
+        source_cache_key=model_access_group_cache_key(group),
+        max_budget=budget.max_budget,
+        fallback_spend=budget.spend,
+        entity_type="Model access group",
+        entity_id=group,
+    )
+
+
 def _dedupe_tags(tags: list[str]) -> list[str]:
     seen: Final = set()
     deduped_tags: Final = []
@@ -545,7 +598,7 @@ async def _get_team_member_budget_counter(
     valid_token: UserAPIKeyAuth,
     team_object: LiteLLM_TeamTable | None,
     user_object: LiteLLM_UserTable | None,
-    user_api_key_cache: DualCache,
+    user_api_key_cache: UserApiKeyCache,
 ) -> _BudgetCounter | None:
     if team_object is None or team_object.team_id is None or user_object is None or valid_token.user_id is None:
         return None
@@ -588,7 +641,7 @@ async def _get_team_member_budget_counter(
 async def _get_org_budget_counter(
     valid_token: UserAPIKeyAuth,
     team_object: LiteLLM_TeamTable | None,
-    user_api_key_cache: DualCache,
+    user_api_key_cache: UserApiKeyCache,
 ) -> _BudgetCounter | None:
     org_id: str | None = None
     if valid_token.org_id is not None:
@@ -657,6 +710,7 @@ def _get_budget_limit_counters(
                 entity_type=entity_type,
                 entity_id=f"{entity_id}:{budget_duration}",
                 spend_log_entity_id=entity_id,
+                window_duration=str(budget_duration),
                 window_start=window_start,
             )
         )
@@ -700,6 +754,7 @@ async def _reserve_counter(
                 counter_key=counter.counter_key,
                 entity_type=counter.entity_type,
                 entity_id=counter.spend_log_entity_id,
+                window_duration=counter.window_duration,
                 window_start=counter.window_start,
             )
             if initialized is False:

litellm/proxy/spend_tracking/savings.py

@@ -481,6 +481,20 @@ def _numeric_savings(value: object) -> float | None:
     return float(value)
 
 
+def classifier_cost_from_decision(routing_decision: Mapping[str, object] | None) -> float | None:
+    """The LLM-classifier cost a routing decision recorded, or ``None`` when it holds none.
+
+    ``None`` covers the decision-less request, the heuristic short-circuit that never
+    called a classifier, the unpriced classifier model, and a malformed value alike:
+    in every one of those cases there is no dollar figure to move, so callers treat
+    ``None`` as zero rather than as an error. The one owner of that reading, shared by
+    the savings netting, the session rollup and the response header, so the three can
+    never disagree about what counts as a classifier charge.
+    """
+    decision: Final = routing_decision if isinstance(routing_decision, Mapping) else {}
+    return _numeric_savings(decision.get("classifier_cost"))
+
+
 def autorouter_savings_for_request(
     model: str | None,
     custom_llm_provider: str | None,
@@ -490,7 +504,8 @@ def autorouter_savings_for_request(
     llm_router: "Callable[[], Router | None] | None" = None,
     cost_breakdown: Mapping[str, object] | None = None,
 ) -> float | None:
-    """Auto-router savings for one request, or ``None`` when the driver is off.
+    """Auto-router savings for one request, net of the classifier call that routed it,
+    or ``None`` when the driver is off.
 
     ``None`` and ``0.0`` are different facts: ``None`` means this request cannot carry a
     figure at all (no routing decision, no baseline, unusable usage), while ``0.0`` is a
@@ -498,6 +513,11 @@ def autorouter_savings_for_request(
     Never raises: pricing failures inside degrade to zero, and the driver-off cases
     return ``None``, so this is safe on the logging path where a raise would fail the
     request's logging.
+
+    The classifier deduction lives here, at the figure's one computation owner, rather
+    than in any reader: the stamped ``autorouter_savings`` is then already net, so the
+    session rollup, the daily tables and every logging consumer agree without each
+    re-deriving the deduction, and the recorded-figure-wins path cannot deduct twice.
     """
     usage: Final = _usage_from_spend_log(usage_object)
     if usage is None or not model:
@@ -510,7 +530,7 @@ def autorouter_savings_for_request(
     if not decision or not baseline_model:
         return None
     router_instance: Final = llm_router() if llm_router else None
-    return compute_autorouter_savings(
+    gross: Final = compute_autorouter_savings(
         baseline_model=baseline_model,
         selected_model=model,
         selected_provider=custom_llm_provider,
@@ -522,6 +542,8 @@ def autorouter_savings_for_request(
         baseline_info=_effective_model_info(router_instance, baseline_id, baseline_model or ""),
         cost_breakdown=cost_breakdown,
     )
+    classifier_cost: Final = classifier_cost_from_decision(decision)
+    return gross if classifier_cost is None else gross - classifier_cost
 
 
 def autorouter_savings_for_logging_payload(

litellm/proxy/spend_tracking/spend_tracking_utils.py

@@ -1,6 +1,7 @@
 import os
 import re
 import secrets
+from collections.abc import Mapping
 from datetime import datetime, timezone
 from datetime import datetime as dt
 from typing import Any, Final, Literal, cast
@@ -23,7 +24,11 @@ from litellm.litellm_core_utils.core_helpers import (
     reconstruct_model_name,
 )
 from litellm.litellm_core_utils.internal_call_metadata import is_unbilled_non_inference_call
-from litellm.litellm_core_utils.litellm_logging import is_valid_sha256_hash
+from litellm.litellm_core_utils.litellm_logging import (
+    coerce_model_access_groups,
+    is_valid_sha256_hash,
+    request_model_access_groups_from_litellm_params,
+)
 from litellm.litellm_core_utils.safe_json_dumps import safe_dumps, strip_null_bytes
 from litellm.proxy._types import SpendLogsMetadata, SpendLogsPayload
 from litellm.proxy.spend_tracking.spend_log_error_logger import spend_log_error
@@ -92,6 +97,8 @@ def _get_spend_logs_metadata(
     metadata: dict | None,
     applied_guardrails: list[str] | None = None,
     batch_models: list[str] | None = None,
+    batch_successful_requests: int | None = None,
+    batch_failed_requests: int | None = None,
     mcp_tool_call_metadata: StandardLoggingMCPToolCall | None = None,
     vector_store_request_metadata: list[StandardLoggingVectorStoreRequest] | None = None,
     guardrail_information: list[StandardLoggingGuardrailInformation] | None = None,
@@ -121,6 +128,8 @@ def _get_spend_logs_metadata(
             error_information=None,
             proxy_server_request=None,
             batch_models=None,
+            batch_successful_requests=None,
+            batch_failed_requests=None,
             mcp_tool_call_metadata=None,
             vector_store_request_metadata=None,
             model_map_information=None,
@@ -154,6 +163,8 @@ def _get_spend_logs_metadata(
     clean_metadata["user_api_key"] = _redact_logged_api_key(_raw_key, already_redacted=_already_redacted)
     clean_metadata["applied_guardrails"] = applied_guardrails
     clean_metadata["batch_models"] = batch_models
+    clean_metadata["batch_successful_requests"] = batch_successful_requests
+    clean_metadata["batch_failed_requests"] = batch_failed_requests
     clean_metadata["mcp_tool_call_metadata"] = mcp_tool_call_metadata
     clean_metadata["vector_store_request_metadata"] = _get_vector_store_request_for_spend_logs_payload(
         vector_store_request_metadata
@@ -243,6 +254,23 @@ def _extract_usage_for_ocr_call(response_obj: Any, response_obj_dict: dict) -> d
         return {}
 
 
+def get_request_model_access_groups(kwargs: Mapping[str, object] | None) -> tuple[str, ...]:
+    """Model access groups that authorized this request, as stamped onto request metadata at auth time."""
+    if kwargs is None:
+        return ()
+
+    standard_logging_payload: Final = kwargs.get("standard_logging_object")
+    if isinstance(standard_logging_payload, Mapping):
+        from_payload: Final = coerce_model_access_groups(standard_logging_payload.get("request_model_access_groups"))
+        if from_payload:
+            return from_payload
+
+    litellm_params: Final = kwargs.get("litellm_params")
+    if not isinstance(litellm_params, Mapping):
+        return ()
+    return request_model_access_groups_from_litellm_params(litellm_params)
+
+
 def _sl_attribution_fallback(
     standard_logging_payload: StandardLoggingPayload | None,
     field: Literal["model_id", "model_group", "api_base", "custom_llm_provider"],
@@ -360,6 +388,16 @@ def get_logging_payload(kwargs, response_obj, start_time, end_time) -> SpendLogs
             if standard_logging_payload is not None
             else None
         ),
+        batch_successful_requests=(
+            standard_logging_payload.get("hidden_params", {}).get("batch_successful_requests", None)
+            if standard_logging_payload is not None
+            else None
+        ),
+        batch_failed_requests=(
+            standard_logging_payload.get("hidden_params", {}).get("batch_failed_requests", None)
+            if standard_logging_payload is not None
+            else None
+        ),
         mcp_tool_call_metadata=(
             standard_logging_payload["metadata"].get("mcp_tool_call_metadata", None)
             if standard_logging_payload is not None

litellm/proxy/utils.py

@@ -1416,7 +1416,7 @@ class ProxyLogging:
         mutation is discarded and a warning is logged so the misconfiguration
         is visible instead of silently forwarding unredacted content.
         """
-        scans_raw_request: Final = getattr(callback, "scan_raw_request", False)
+        scans_raw_request: Final = callback.scan_raw_request
         should_use_raw_snapshot: Final = scans_raw_request and raw_request_snapshot is not None
         input_data: Final = (  # mutable-ok: same request-payload shape as data
             independent_snapshot(raw_request_snapshot) if should_use_raw_snapshot else data
@@ -1453,7 +1453,7 @@ class ProxyLogging:
                 "scan_raw_request is for block-only guardrails and this mutation is being "
                 "discarded. Remove scan_raw_request from this guardrail's config if it needs "
                 "to mask/rewrite content.",
-                getattr(callback, "guardrail_name", None) or callback.__class__.__name__,
+                callback.guardrail_name or callback.__class__.__name__,
             )
         if scans_raw_request:
             if result is not None:
@@ -1778,7 +1778,7 @@ class ProxyLogging:
         # guarantee must hold even under litellm.safe_memory_mode, which
         # otherwise makes deep copies return the original object.
         needs_raw_request_snapshot: Final = any(
-            isinstance(cb, CustomGuardrail) and getattr(cb, "scan_raw_request", False)
+            isinstance(cb, CustomGuardrail) and cb.scan_raw_request
             for cb in ProxyLogging._callback_capabilities().resolved_callbacks
         )
         raw_request_snapshot: Final[dict | None] = (  # mutable-ok: same request-payload shape as data
@@ -1938,7 +1938,7 @@ class ProxyLogging:
         """
 
         def _input_for(callback: CustomGuardrail) -> dict:  # mutable-ok: same request-payload shape as data
-            if not getattr(callback, "scan_raw_request", False) or raw_request_snapshot is None:
+            if not callback.scan_raw_request or raw_request_snapshot is None:
                 return data
             return independent_snapshot(raw_request_snapshot)
 
@@ -1962,11 +1962,7 @@ class ProxyLogging:
             # deployment-level guardrail sharing this name would see no marker
             # via _pre_call_hook_already_ran and re-run it a second time on
             # live kwargs.
-            if (
-                getattr(callback, "scan_raw_request", False)
-                and not isinstance(result, BaseException)
-                and result is not None
-            ):
+            if callback.scan_raw_request and not isinstance(result, BaseException) and result is not None:
                 callback.mark_pre_call_hook_ran(data)
         raised: Final = tuple(result for result in results if isinstance(result, BaseException))
         blocking: Final = next((exc for exc in raised if not _exception_changes_request_flow(exc)), None)
@@ -6035,10 +6031,42 @@ def _should_use_smtp_ssl(smtp_port: int) -> bool:
     return os.getenv("SMTP_USE_SSL", "False") == "True" or smtp_port == 465
 
 
-def _create_smtp_connection(smtp_host: str, smtp_port: int) -> smtplib.SMTP:
+def _create_smtp_connection(smtp_host: str, smtp_port: int, timeout: float) -> smtplib.SMTP:
     if _should_use_smtp_ssl(smtp_port=smtp_port):
-        return smtplib.SMTP_SSL(host=smtp_host, port=smtp_port, context=ssl.create_default_context())
-    return smtplib.SMTP(host=smtp_host, port=smtp_port)
+        return smtplib.SMTP_SSL(host=smtp_host, port=smtp_port, context=ssl.create_default_context(), timeout=timeout)
+    return smtplib.SMTP(host=smtp_host, port=smtp_port, timeout=timeout)
+
+
+def _send_smtp_message(
+    email_message: MIMEMultipart,
+    smtp_host: str,
+    smtp_port: int,
+    smtp_username: str | None,
+    smtp_password: str | None,
+    sender_email: str,
+    receiver_email: str,
+    timeout: float,
+) -> None:
+    using_ssl: Final = _should_use_smtp_ssl(smtp_port=smtp_port)
+    with _create_smtp_connection(
+        smtp_host=smtp_host,
+        smtp_port=smtp_port,
+        timeout=timeout,
+    ) as server:
+        if not using_ssl and os.getenv("SMTP_TLS", "True") != "False":
+            server.starttls(context=ssl.create_default_context())
+
+        if smtp_username and smtp_password:
+            server.login(
+                user=smtp_username,
+                password=smtp_password,
+            )
+
+        server.send_message(
+            msg=email_message,
+            from_addr=sender_email,
+            to_addrs=receiver_email,
+        )
 
 
 async def send_email(
@@ -6084,27 +6112,18 @@ async def send_email(
     email_message.attach(MIMEText(html, "html"))
 
     try:
-        using_ssl: Final = _should_use_smtp_ssl(smtp_port=smtp_port)
-        with _create_smtp_connection(
+        smtp_timeout: Final = float(os.getenv("SMTP_TIMEOUT", "30"))
+        await asyncio.to_thread(
+            _send_smtp_message,
+            email_message=email_message,
             smtp_host=smtp_host,
             smtp_port=smtp_port,
-        ) as server:
-            if not using_ssl and os.getenv("SMTP_TLS", "True") != "False":
-                server.starttls(context=ssl.create_default_context())
-
-            # Login to your email account only if smtp_username and smtp_password are provided
-            if smtp_username and smtp_password:
-                server.login(
-                    user=smtp_username,
-                    password=smtp_password,
-                )
-
-            # Send the email
-            server.send_message(
-                msg=email_message,
-                from_addr=sender_email,
-                to_addrs=receiver_email,
-            )
+            smtp_username=smtp_username,
+            smtp_password=smtp_password,
+            sender_email=sender_email,
+            receiver_email=receiver_email,
+            timeout=smtp_timeout,
+        )
 
     except Exception as e:
         verbose_proxy_logger.exception("An error occurred while sending the email:" + str(e))

litellm/repositories/prisma_protocols.py

@@ -144,4 +144,7 @@ class PrismaBatch(Protocol):
     @property
     def litellm_endusertable(self) -> BatchTable: ...
 
+    @property
+    def litellm_modelaccessgroupbudgettable(self) -> BatchTable: ...
+
     async def commit(self) -> None: ...

litellm/repositories/table_repositories.py

@@ -68,6 +68,10 @@ class SpendLogsRepository(PrismaTableRepository["prisma_models.LiteLLM_SpendLogs
     table_name = "litellm_spendlogs"
 
 
+class BudgetWindowSpendRepository(PrismaTableRepository["prisma_models.LiteLLM_BudgetWindowSpend"]):
+    table_name = "litellm_budgetwindowspend"
+
+
 class ClaudeCodePluginRepository(PrismaTableRepository["prisma_models.LiteLLM_ClaudeCodePluginTable"]):
     table_name = "litellm_claudecodeplugintable"
 
@@ -100,6 +104,10 @@ class TagRepository(PrismaTableRepository["prisma_models.LiteLLM_TagTable"]):
     table_name = "litellm_tagtable"
 
 
+class ModelAccessGroupBudgetRepository(PrismaTableRepository["prisma_models.LiteLLM_ModelAccessGroupBudgetTable"]):
+    table_name = "litellm_modelaccessgroupbudgettable"
+
+
 class InvitationLinkRepository(PrismaTableRepository["prisma_models.LiteLLM_InvitationLink"]):
     table_name = "litellm_invitationlink"
 

litellm/repositories/unit_of_work.py

@@ -118,6 +118,7 @@ class BudgetCascadeUnitOfWork:
     keys: LinkedSpendResetWrites
     organizations: LinkedSpendResetWrites
     tags: LinkedSpendResetWrites
+    model_access_groups: LinkedSpendResetWrites
     endusers: LinkedSpendResetWrites
     budgets: BudgetWindowWrites
 
@@ -143,6 +144,7 @@ async def budget_cascade_unit_of_work(
         keys=LinkedSpendResetWrites(table=batch.litellm_verificationtoken),
         organizations=LinkedSpendResetWrites(table=batch.litellm_organizationtable),
         tags=LinkedSpendResetWrites(table=batch.litellm_tagtable),
+        model_access_groups=LinkedSpendResetWrites(table=batch.litellm_modelaccessgroupbudgettable),
         endusers=LinkedSpendResetWrites(table=batch.litellm_endusertable),
         budgets=BudgetWindowWrites(table=batch.litellm_budgettable),
     )

litellm/types/integrations/custom_logger.py

@@ -5,8 +5,14 @@ from pydantic import BaseModel, Field
 CHAT_COMPLETION_AGENTIC_SURFACE: Final = "chat_completions"
 RESPONSES_AGENTIC_SURFACE: Final = "responses"
 CODE_INTERPRETER_INTERCEPTION_PREFIX: Final = "_code_interpreter_interception"
+HEADROOM_INTERCEPTION_PREFIX: Final = "_headroom_interception"
+HEADROOM_CONVERTED_STREAM_KEY: Final = f"{HEADROOM_INTERCEPTION_PREFIX}_converted_stream"
 NON_CODE_INTERPRETER_INTERCEPTION_INTERNAL_PREFIXES: Final = frozenset(
-    ("_websearch_interception", "_compression_interception")
+    (
+        "_websearch_interception",
+        "_compression_interception",
+        HEADROOM_INTERCEPTION_PREFIX,
+    )
 )
 INTERCEPTION_INTERNAL_PREFIXES: Final = frozenset(
     (

litellm/types/llms/vertex_ai_gemini_transcription.py

@@ -0,0 +1,72 @@
+from typing import Literal
+
+from pydantic import BaseModel, ConfigDict
+from typing_extensions import ReadOnly, TypedDict
+
+
+class VertexGeminiTranscriptionInlineData(TypedDict):
+    mimeType: ReadOnly[str]
+    data: ReadOnly[str]
+
+
+class VertexGeminiTranscriptionPart(TypedDict):
+    inlineData: ReadOnly[VertexGeminiTranscriptionInlineData]
+
+
+class VertexGeminiTranscriptionContent(TypedDict):
+    role: ReadOnly[Literal["user"]]
+    parts: ReadOnly[tuple[VertexGeminiTranscriptionPart, ...]]
+
+
+class VertexGeminiTranscriptionAudioConfig(TypedDict, total=False):
+    languageCodes: ReadOnly[tuple[str, ...]]
+
+
+class VertexGeminiTranscriptionGenerationConfig(TypedDict):
+    audioTranscriptionConfig: ReadOnly[VertexGeminiTranscriptionAudioConfig]
+
+
+class VertexGeminiTranscriptionRequest(TypedDict):
+    contents: ReadOnly[tuple[VertexGeminiTranscriptionContent, ...]]
+    generationConfig: ReadOnly[VertexGeminiTranscriptionGenerationConfig]
+
+
+class VertexGeminiTranscriptionResponsePart(BaseModel):
+    model_config = ConfigDict(extra="ignore")
+
+    text: str | None = None
+
+
+class VertexGeminiTranscriptionResponseContent(BaseModel):
+    model_config = ConfigDict(extra="ignore")
+
+    parts: tuple[VertexGeminiTranscriptionResponsePart, ...] = ()
+
+
+class VertexGeminiTranscriptionCandidate(BaseModel):
+    model_config = ConfigDict(extra="ignore")
+
+    content: VertexGeminiTranscriptionResponseContent | None = None
+
+
+class VertexGeminiTranscriptionModalityTokens(BaseModel):
+    model_config = ConfigDict(extra="ignore")
+
+    modality: str | None = None
+    tokenCount: int = 0
+
+
+class VertexGeminiTranscriptionUsageMetadata(BaseModel):
+    model_config = ConfigDict(extra="ignore")
+
+    promptTokenCount: int = 0
+    candidatesTokenCount: int = 0
+    totalTokenCount: int = 0
+    promptTokensDetails: tuple[VertexGeminiTranscriptionModalityTokens, ...] = ()
+
+
+class VertexGeminiTranscriptionResponse(BaseModel):
+    model_config = ConfigDict(extra="ignore")
+
+    candidates: tuple[VertexGeminiTranscriptionCandidate, ...] = ()
+    usageMetadata: VertexGeminiTranscriptionUsageMetadata | None = None

litellm/types/proxy/management_endpoints/config_overrides.py

@@ -52,6 +52,43 @@ class HashicorpVaultConfig(BaseModel):
     )
 
 
+class CyberArkConfig(BaseModel):
+    """Configuration for CyberArk Conjur secret manager integration."""
+
+    cyberark_api_base: str | None = Field(
+        default=None,
+        description="The address of the CyberArk Conjur server (e.g., https://conjur.example.com)",
+    )
+    cyberark_account: str | None = Field(
+        default=None,
+        description="The Conjur organization account name",
+    )
+    cyberark_username: str | None = Field(
+        default=None,
+        description="The Conjur username (login) to authenticate as",
+    )
+    cyberark_api_key: str | None = Field(
+        default=None,
+        description="API key for Conjur API-key authentication",
+    )
+    client_cert: str | None = Field(
+        default=None,
+        description="Path to the client TLS certificate for certificate-based authentication",
+    )
+    client_key: str | None = Field(
+        default=None,
+        description="Path to the client TLS private key for certificate-based authentication",
+    )
+    ssl_verify: str | None = Field(
+        default=None,
+        description="Set to false to disable SSL verification (e.g., for self-signed certificates)",
+    )
+    refresh_interval: str | None = Field(
+        default=None,
+        description="Auth token cache TTL in seconds (default: 300)",
+    )
+
+
 class ConfigOverrideSettingsResponse(BaseModel):
     """Response model for config override settings GET endpoints."""
 

litellm/types/proxy/management_endpoints/model_management_endpoints.py

@@ -1,6 +1,7 @@
+from datetime import datetime
 from typing import Any
 
-from pydantic import BaseModel, Field
+from pydantic import BaseModel, ConfigDict, Field
 
 from ...router import ModelGroupInfo
 
@@ -53,10 +54,42 @@ class DeleteModelGroupResponse(BaseModel):
     message: str
 
 
+class AccessGroupBudget(BaseModel):
+    budget_id: str
+    max_budget: float | None = None
+    soft_budget: float | None = None
+    budget_duration: str | None = None
+    budget_reset_at: datetime | None = None
+
+
+class AccessGroupBudgetRequest(BaseModel):
+    budget_id: str | None = None  # Link an existing budget instead of creating one
+    max_budget: float | None = Field(default=None, ge=0)
+    soft_budget: float | None = Field(default=None, ge=0)
+    budget_duration: str | None = None
+
+    # rejects tpm_limit/rpm_limit/max_parallel_requests: those are not enforced per access group
+    model_config = ConfigDict(extra="forbid")
+
+
+class AccessGroupBudgetResponse(BaseModel):
+    access_group: str
+    spend: float  # Shared spend accrued by every key that can reach this access group
+    budget: AccessGroupBudget | None = None
+
+
+class DeleteAccessGroupBudgetResponse(BaseModel):
+    access_group: str
+    budget_deleted: bool  # False when the access group had no budget to begin with
+    message: str
+
+
 class AccessGroupInfo(BaseModel):
     access_group: str
     model_names: list[str]  # List of model names in this access group
     deployment_count: int  # Total number of deployments with this access group
+    spend: float | None = None  # Only populated by /access_group/{access_group}/info
+    budget: AccessGroupBudget | None = None
 
 
 class ListAccessGroupsResponse(BaseModel):

litellm/types/proxy/model_access_group_budget.py

@@ -0,0 +1,19 @@
+"""The model access group budget state auth and the spend reservation path share."""
+
+from __future__ import annotations
+
+from pydantic import BaseModel
+
+
+class ModelAccessGroupBudget(BaseModel):
+    """One model access group's budget, flattened out of its joined ``LiteLLM_ModelAccessGroupBudgetTable`` row.
+
+    Both readers want only the recorded spend and the ceiling, and this sits on the per-request hot
+    path behind a cache, so the linked budget row is collapsed to ``max_budget`` rather than cached
+    whole. ``spend`` is the DB-recorded value, which lags the live counter and is only ever a
+    fallback for it.
+    """
+
+    access_group_name: str
+    spend: float = 0.0
+    max_budget: float | None = None

litellm/types/services.py

@@ -40,6 +40,8 @@ class ServiceTypes(str, enum.Enum):
     # spend update queue - current spend of key, user, team
     IN_MEMORY_SPEND_UPDATE_QUEUE = "in_memory_spend_update_queue"
     REDIS_SPEND_UPDATE_QUEUE = "redis_spend_update_queue"
+    # budget window spend queue - per-window spend of key, team
+    REDIS_WINDOW_SPEND_UPDATE_QUEUE = "redis_window_spend_update_queue"
 
 
 class ServiceConfig(TypedDict):

litellm/types/utils.py

@@ -40,7 +40,7 @@ from pydantic import (
     field_serializer,
     field_validator,
 )
-from typing_extensions import ReadOnly, Required, TypedDict
+from typing_extensions import NotRequired, ReadOnly, Required, TypedDict
 
 from litellm._logging import verbose_logger
 from litellm._uuid import uuid
@@ -2957,6 +2957,8 @@ class StandardLoggingHiddenParams(TypedDict):
     litellm_overhead_time_ms: float | None
     additional_headers: StandardLoggingAdditionalHeaders | None
     batch_models: list[str] | None
+    batch_successful_requests: ReadOnly[int | None]
+    batch_failed_requests: ReadOnly[int | None]
     litellm_model_name: str | None  # the model name sent to the provider by litellm
     usage_object: dict | None
 
@@ -3258,6 +3260,7 @@ class StandardLoggingPayload(TypedDict):
     cache_key: str | None
     saved_cache_cost: float
     request_tags: list
+    request_model_access_groups: NotRequired[ReadOnly[Sequence[str]]]
     end_user: str | None
     requester_ip_address: str | None
     user_agent: str | None
@@ -3503,6 +3506,7 @@ agentic_loop_internal_litellm_params: Final = [
     "_code_interpreter_interception_converted_stream",
     "_websearch_interception_emit_native_blocks",
     "_websearch_interception_converted_stream",
+    "_headroom_interception_converted_stream",
 ]
 
 # Proxy-owned callback credentials, stamped from admin-configured team/key callback

litellm/types/videos/main.py

@@ -2,7 +2,7 @@ from typing import Any, Literal
 
 from openai.types.audio.transcription_create_params import FileTypes
 from pydantic import BaseModel
-from typing_extensions import TypedDict
+from typing_extensions import ReadOnly, TypedDict
 
 
 class VideoObject(BaseModel):
@@ -76,6 +76,7 @@ class VideoCreateOptionalRequestParams(TypedDict, total=False):
     image: Any | None  # Image for image-to-video; dict with gcsUri/bytesBase64Encoded, or file-like object
     parameters: dict[str, Any] | None  # Provider-specific parameters block passed directly to the API
     model: str | None
+    resolution: ReadOnly[str | None]
     seconds: str | None
     size: str | None
     characters: list[dict[str, str]] | None

litellm/utils.py

@@ -69,6 +69,7 @@ from litellm.constants import (
     DEFAULT_REASONING_EFFORT_MEDIUM_THINKING_BUDGET,
     DEFAULT_TRIM_RATIO,
     FUNCTION_DEFINITION_TOKEN_COUNT,
+    HF_CONFIG_FETCH_TIMEOUT_SECONDS,
     INITIAL_RETRY_DELAY,
     JITTER,
     MAX_RETRY_DELAY,
@@ -3581,7 +3582,7 @@ def get_optional_params_embeddings(
             object = litellm.AmazonTitanMultimodalEmbeddingG1Config()
         elif "amazon.titan-embed-text-v2:0" in model:
             object = litellm.AmazonTitanV2Config()
-        elif "cohere.embed-multilingual-v3" in model or "cohere.embed-v4" in model:
+        elif "cohere.embed" in model:
             object = litellm.BedrockCohereEmbeddingConfig()
         elif "twelvelabs" in model or "marengo" in model:
             object = litellm.TwelveLabsMarengoEmbeddingConfig()
@@ -5168,7 +5169,7 @@ def get_max_tokens(model: str) -> int | None:
         config_url: Final = f"https://huggingface.co/{model_name}/raw/main/config.json"
         try:
             # Make the HTTP request to get the raw JSON file
-            response: Final = litellm.module_level_client.get(config_url)
+            response: Final = litellm.module_level_client.get(config_url, timeout=HF_CONFIG_FETCH_TIMEOUT_SECONDS)
             response.raise_for_status()  # Raise an exception for bad responses (4xx or 5xx)
 
             # Parse the JSON response
@@ -5522,7 +5523,7 @@ def _get_max_position_embeddings(model_name: str) -> int | None:
 
     try:
         # Make the HTTP request to get the raw JSON file
-        response: Final = litellm.module_level_client.get(config_url)
+        response: Final = litellm.module_level_client.get(config_url, timeout=HF_CONFIG_FETCH_TIMEOUT_SECONDS)
         response.raise_for_status()  # Raise an exception for bad responses (4xx or 5xx)
 
         # Parse the JSON response
@@ -8573,6 +8574,13 @@ class ProviderConfigManager:
 
             return SonioxAudioTranscriptionConfig()
         elif litellm.LlmProviders.VERTEX_AI == provider:
+            bare_vertex_model: Final = model.removeprefix("vertex_ai/")
+            if bare_vertex_model.startswith("gemini") and "transcribe" in bare_vertex_model:
+                from litellm.llms.vertex_ai.audio_transcription.gemini_transcribe_transformation import (
+                    VertexGeminiAudioTranscriptionConfig,
+                )
+
+                return VertexGeminiAudioTranscriptionConfig()
             from litellm.llms.vertex_ai.audio_transcription.transformation import (
                 VertexAIAudioTranscriptionConfig,
             )

pyproject.toml

@@ -67,8 +67,8 @@ proxy = [
     "azure-identity>=1.25.2,<2.0",
     "azure-storage-blob>=12.28.0,<13.0",
     "mcp>=1.28.1,<2.0",
-    "litellm-proxy-extras==0.4.90",
-    "litellm-enterprise==0.1.61",
+    "litellm-proxy-extras==0.4.91",
+    "litellm-enterprise==0.1.62",
     "RestrictedPython>=8.5,<9.0",
     "rich>=13.9.4,<14.0",
     "InquirerPy>=0.3.4,<1.0",

ruff-strict-budget.json

@@ -24,7 +24,7 @@
     "limit": 133
   },
   "ANN401": {
-    "limit": 655
+    "limit": 654
   },
   "ASYNC230": {
     "limit": 11
@@ -231,7 +231,7 @@
     "limit": 5
   },
   "TID251": {
-    "limit": 1117
+    "limit": 1116
   },
   "TRY002": {
     "limit": 524

ruff.toml

@@ -6,7 +6,8 @@ lint.extend-select = [
     "T20", "PGH004", "RUF008", "RUF009", "RUF100",
     "B033", "FURB136", "FURB168", "FURB188", "I001", "PERF402", "PIE790", "PIE800", "PLC0208",
     "PLR0402", "PLR1711", "PLR1730", "PLR2044", "PLW0133", "PYI030", "PYI041", "PYI064", "RET501",
-    "RUF010", "RUF022", "RUF023", "RUF051", "SIM114", "SIM118", "TC005", "UP006", "UP007", "UP008",
+    "RUF010", "RUF022", "RUF023", "RUF051", "S113", "SIM114", "SIM118", "TC005", "UP006", "UP007",
+    "UP008",
     "UP012", "UP018", "UP024", "UP032", "UP034", "UP035", "UP037", "UP045",
 ]
 # RUF100 (unused-noqa) only knows the rules enabled in THIS config, so it would strip

schema.prisma

@@ -29,6 +29,7 @@ model LiteLLM_BudgetTable {
   keys LiteLLM_VerificationToken[] // multiple keys can have the same budget
   end_users LiteLLM_EndUserTable[] // multiple end-users can have the same budget
   tags LiteLLM_TagTable[] // multiple tags can have the same budget
+  model_access_groups LiteLLM_ModelAccessGroupBudgetTable[] // multiple model access groups can have the same budget
   team_membership LiteLLM_TeamMembership[] // budgets of Users within a Team 
   organization_membership LiteLLM_OrganizationMembership[] // budgets of Users within a Organization
 }
@@ -585,6 +586,20 @@ model LiteLLM_EndUserTable {
   blocked Boolean @default(false)
 }
 
+// Budget and shared spend for a model access group. The groups themselves are not rows anywhere:
+// they are free-text strings in LiteLLM_ProxyModelTable.model_info.access_groups, so a row here
+// exists only once someone gives that group a budget.
+model LiteLLM_ModelAccessGroupBudgetTable {
+  access_group_name String @id
+  spend Float @default(0.0)
+  budget_id String?
+  litellm_budget_table LiteLLM_BudgetTable? @relation(fields: [budget_id], references: [budget_id])
+  created_at DateTime @default(now()) @map("created_at")
+  created_by String?
+  updated_at DateTime @default(now()) @updatedAt
+  updated_by String?
+}
+
 // Track tags with budgets and spend
 model LiteLLM_TagTable {
   tag_name String @id
@@ -649,6 +664,18 @@ model LiteLLM_SpendLogs {
   @@index([session_id])
 }
 
+model LiteLLM_BudgetWindowSpend {
+  entity_type     String
+  entity_id       String
+  window_duration String
+  window_start    DateTime
+  spend           Float    @default(0.0)
+  created_at      DateTime @default(now())
+  updated_at      DateTime @updatedAt
+
+  @@id([entity_type, entity_id, window_duration])
+}
+
 // View spend, model, api_key per request
 model LiteLLM_ErrorLogs {
   request_id          String   @id @default(uuid())

scripts/sync_together_ai_models.py

@@ -187,9 +187,22 @@ CAPABILITY_RULES: Final = (
     _rule("thinkingmachines/Inkling-Small", "reviewed for the LIT-5968 backfill; no tool or vision support documented"),
     _rule(
         "zai-org/GLM-5.2",
-        "reviewed for the LIT-5968 backfill against https://www.together.ai/models/glm-5-2",
+        "reviewed for the LIT-5968 backfill against https://www.together.ai/models/glm-5-2;"
+        " 128K output ceiling per https://docs.z.ai/guides/llm/glm-5.2",
         **_TOOLS,
         supports_reasoning=True,
+        max_output_tokens=128000,
+        max_tokens=128000,
+    ),
+    _rule(
+        "zai-org/GLM-5.3-Flash",
+        "reviewed for LIT-6489 against https://www.together.ai/models/glm-5-3-flash;"
+        " 128K output ceiling per https://docs.z.ai/guides/llm/glm-5.3",
+        **_TOOLS,
+        supports_reasoning=True,
+        supports_vision=True,
+        max_output_tokens=128000,
+        max_tokens=128000,
     ),
 )
 
@@ -288,15 +301,10 @@ def _api_fields(model: CatalogModel) -> RegistryEntry:
 
 def _new_entry(model: CatalogModel, mode: str) -> RegistryEntry:
     rule: Final = RULES_BY_ID.get(model.id)
-    length_fields: Final = (
-        {}
-        if model.context_length is None
-        else {"max_input_tokens": model.context_length, "max_tokens": model.context_length}
-        | ({"max_output_tokens": model.context_length} if mode == "chat" else {})
-    )
+    legacy_ceiling: Final = {} if model.context_length is None else {"max_tokens": model.context_length}
     merged: Final = {
         **_api_fields(model),
-        **length_fields,
+        **legacy_ceiling,
         "litellm_provider": PROVIDER,
         "mode": mode,
         "source": SOURCE_URL,