Merge remote-tracking branch 'origin/litellm_internal_staging' into litellm_realtime_audio_output_tokens

.github/workflows/check-ui-api-types.yml

@@ -83,6 +83,24 @@ jobs:
         if: steps.changes.outputs.relevant == 'true'
         run: uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma
 
+      - name: Regenerate the lazy OpenAPI snapshot
+        if: steps.changes.outputs.relevant == 'true'
+        run: uv run --no-sync python -m litellm.proxy._lazy_openapi_snapshot
+
+      - name: Fail if the lazy OpenAPI snapshot is stale
+        if: steps.changes.outputs.relevant == 'true'
+        run: |
+          if ! git diff --exit-code -- litellm/proxy/_lazy_openapi_snapshot.json; then
+            echo "::error file=litellm/proxy/_lazy_openapi_snapshot.json::The lazy OpenAPI snapshot is out of sync with the lazily loaded routes."
+            echo ""
+            echo "A lazily loaded route or model changed without regenerating the snapshot that /openapi.json serves for unloaded features."
+            echo "To fix, run from the repo root:"
+            echo "  uv run python -m litellm.proxy._lazy_openapi_snapshot"
+            echo "then run npm run gen:api from ui/litellm-dashboard and commit both files."
+            exit 1
+          fi
+          echo "_lazy_openapi_snapshot.json is in sync with the lazily loaded routes."
+
       - name: Set up Node.js
         if: steps.changes.outputs.relevant == 'true'
         uses: actions/setup-node@a0853c24544627f65ddf259abe73b1d18a591444 # v5.0.0

basedpyright-code-budget.json

@@ -84,7 +84,7 @@
     "limit": 56
   },
   "reportPrivateUsage": {
-    "limit": 1810
+    "limit": 1808
   },
   "reportRedeclaration": {
     "limit": 8
@@ -135,7 +135,7 @@
     "limit": 21
   },
   "reportUnusedFunction": {
-    "limit": 139
+    "limit": 138
   },
   "reportUnusedImport": {
     "limit": 544

db_scripts/partition_spend_logs.sql

@@ -10,6 +10,11 @@
 -- partitioned, so existing installs are unaffected until you run this.
 --
 -- IMPORTANT
+--   * After partitioning, `prisma db push` (including the proxy's
+--     --use_prisma_db_push startup mode) is NOT supported: it tries to rewrite
+--     the primary key back to ("request_id"), which Postgres rejects on a
+--     partitioned table. The proxy detects this and exits with guidance.
+--     Use the default startup path (`prisma migrate deploy`) instead.
 --   * Test on a staging copy first and take a backup.
 --   * Postgres cannot convert a populated table to partitioned in place, so this
 --     renames the old table aside and creates a fresh partitioned table.

enterprise/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "litellm-enterprise"
-version = "0.1.60"
+version = "0.1.61"
 description = "Package for LiteLLM Enterprise features"
 readme = "README.md"
 requires-python = ">=3.9"
@@ -26,7 +26,7 @@ required-version = ">=0.10.9"
 module-root = ""
 
 [tool.commitizen]
-version = "0.1.60"
+version = "0.1.61"
 version_files = [
     "pyproject.toml:^version",
     "../pyproject.toml:litellm-enterprise==",

litellm-proxy-extras/litellm_proxy_extras/utils.py

@@ -40,6 +40,65 @@ def _get_prisma_env() -> dict:
 
 _MIGRATION_TS_RE = re.compile(r"^(\d{14})_")
 
+_SPEND_LOGS_ALTER_RE = re.compile(r'^ALTER\s+TABLE\s+"LiteLLM_SpendLogs"\s', re.IGNORECASE)
+_SPEND_LOGS_ARTIFACT_DROP_RE = re.compile(
+    r'^DROP\s+TABLE\s+"LiteLLM_SpendLogs_[^"]*"', re.IGNORECASE
+)
+_SPEND_LOGS_PK_CLAUSE_RE = re.compile(
+    r'^(?:DROP\s+CONSTRAINT\s+"[^"]*_pkey"'
+    r'|ADD\s+(?:CONSTRAINT\s+"[^"]*"\s+)?PRIMARY\s+KEY\s*\([^)]*\))$',
+    re.IGNORECASE,
+)
+
+PARTITIONED_SPEND_LOGS_PUSH_ERROR = (
+    "LiteLLM_SpendLogs is a partitioned table (see db_scripts/partition_spend_logs.sql), "
+    "so its primary key must include the partition key (\"startTime\"). `prisma db push` "
+    "reconciles the database against schema.prisma, which declares the unpartitioned "
+    "primary key (\"request_id\"), and Postgres rejects that rewrite with: unique "
+    "constraint on partitioned table must include all partitioning columns. Start the "
+    "proxy without --use_prisma_db_push so it uses `prisma migrate deploy`, which only "
+    "applies shipped migrations and leaves the partitioned primary key alone."
+)
+
+
+def _without_sql_comments(statement: str) -> str:
+    return "\n".join(
+        line
+        for line in statement.splitlines()
+        if line.strip() and not line.strip().startswith("--")
+    ).strip()
+
+
+def _without_spend_logs_pk_clauses(statement: str) -> Optional[str]:
+    prefix_match = _SPEND_LOGS_ALTER_RE.match(statement)
+    if not prefix_match:
+        return statement
+    kept = tuple(
+        clause.strip()
+        for clause in statement[prefix_match.end():].split(",\n")
+        if not _SPEND_LOGS_PK_CLAUSE_RE.match(clause.strip())
+    )
+    if not kept:
+        return None
+    return statement[: prefix_match.end()] + ",\n".join(kept)
+
+
+def filter_partitioned_spend_logs_diff(diff_sql: str) -> str:
+    """Drop statements from a `prisma migrate diff` script that fight the
+    SpendLogs partitioning runbook (db_scripts/partition_spend_logs.sql): the
+    primary-key rewrite on "LiteLLM_SpendLogs", which Postgres rejects on a
+    partitioned table, and drops of runbook artifacts such as
+    "LiteLLM_SpendLogs_legacy"."""
+    kept = tuple(
+        filtered
+        for statement in diff_sql.split(";")
+        for bare in (_without_sql_comments(statement),)
+        if bare and not _SPEND_LOGS_ARTIFACT_DROP_RE.match(bare)
+        for filtered in (_without_spend_logs_pk_clauses(bare),)
+        if filtered is not None
+    )
+    return "".join(f"{statement};\n\n" for statement in kept)
+
 
 def _migration_timestamp(name: str) -> int:
     """Extract the leading `YYYYMMDDHHMMSS` timestamp from a migration name.
@@ -355,7 +414,24 @@ class ProxyExtrasDBManager:
             return
         logger.info(f"Migration diff created at {diff_sql_path}")
 
+        if ProxyExtrasDBManager.spend_logs_is_partitioned():
+            filtered_sql = filter_partitioned_spend_logs_diff(
+                diff_sql_path.read_text()
+            )
+            diff_sql_path.write_text(filtered_sql)
+            logger.info(
+                "LiteLLM_SpendLogs is partitioned; removed its primary-key "
+                "rewrite and partitioning artifacts from the drift script"
+            )
+            if not filtered_sql.strip():
+                logger.info("Drift script is empty after filtering; nothing to apply")
+                if not mark_all_applied:
+                    return
+                ProxyExtrasDBManager._mark_migrations_applied(migrations_dir)
+                return
+
         # 2. Run prisma db execute to apply the migration
+        applied_ok = False
         try:
             logger.info("Running prisma db execute to apply the migration diff...")
             result = subprocess.run(
@@ -376,6 +452,7 @@ class ProxyExtrasDBManager:
             )
             logger.info(f"prisma db execute stdout: {result.stdout}")
             logger.info("✅ Migration diff applied successfully")
+            applied_ok = True
         except subprocess.CalledProcessError as e:
             logger.warning(f"Failed to apply migration diff: {e.stderr}")
         except subprocess.TimeoutExpired:
@@ -384,6 +461,16 @@ class ProxyExtrasDBManager:
         # 3. Mark all migrations as applied
         if not mark_all_applied:
             return
+        if not applied_ok:
+            logger.warning(
+                "Drift script failed to apply; NOT marking migrations as "
+                "applied so a later migration run can retry them"
+            )
+            return
+        ProxyExtrasDBManager._mark_migrations_applied(migrations_dir)
+
+    @staticmethod
+    def _mark_migrations_applied(migrations_dir: str):
         migration_names = ProxyExtrasDBManager._get_migration_names(migrations_dir)
         logger.info(f"Resolving {len(migration_names)} migrations")
         for migration_name in migration_names:
@@ -410,6 +497,55 @@ class ProxyExtrasDBManager:
                         f"Failed to resolve migration {migration_name}: {e.stderr}"
                     )
 
+    @staticmethod
+    def spend_logs_is_partitioned() -> bool:
+        """True when the connected database's LiteLLM_SpendLogs is a
+        partitioned table in Prisma's target schema (the `schema` URL param,
+        falling back to Prisma's default target, public), i.e. the operator
+        ran db_scripts/partition_spend_logs.sql. Returns False when psycopg is
+        unavailable or the database cannot be reached, preserving the
+        pre-existing behavior in those cases."""
+        database_url = os.getenv("DATABASE_URL")
+        if not database_url:
+            return False
+
+        try:
+            import psycopg
+        except ImportError:
+            return False
+
+        cleaned_url = ProxyExtrasDBManager._strip_prisma_query_params(database_url)
+        try:
+            with psycopg.connect(
+                cleaned_url, connect_timeout=10, autocommit=True
+            ) as conn:
+                row = conn.execute(
+                    "SELECT 1 "
+                    "FROM pg_partitioned_table pt "
+                    "JOIN pg_class c ON c.oid = pt.partrelid "
+                    "JOIN pg_namespace n ON n.oid = c.relnamespace "
+                    "WHERE c.relname = 'LiteLLM_SpendLogs' "
+                    "  AND n.nspname = %s",
+                    (
+                        ProxyExtrasDBManager._prisma_schema_param(database_url)
+                        or "public",
+                    ),
+                ).fetchone()
+        except (psycopg.OperationalError, psycopg.DatabaseError):
+            return False
+        return row is not None
+
+    @staticmethod
+    def _prisma_schema_param(url: str) -> Optional[str]:
+        """The `schema` query param Prisma uses to pick its target schema,
+        or None when the URL does not set one."""
+        from urllib.parse import urlparse, parse_qsl
+
+        return next(
+            (v for k, v in parse_qsl(urlparse(url).query) if k == "schema"),
+            None,
+        )
+
     @staticmethod
     def _strip_prisma_query_params(url: str) -> str:
         """Remove Prisma-specific query params (connection_limit, pool_timeout,
@@ -528,7 +664,8 @@ class ProxyExtrasDBManager:
         migrations_dir = ProxyExtrasDBManager._get_prisma_dir()
 
         if not use_migrate:
-            # Preserve `prisma db push` path unchanged.
+            if ProxyExtrasDBManager.spend_logs_is_partitioned():
+                raise RuntimeError(PARTITIONED_SPEND_LOGS_PUSH_ERROR)
             original_dir = os.getcwd()
             os.chdir(migrations_dir)
             try:
@@ -972,6 +1109,8 @@ class ProxyExtrasDBManager:
                                 )
                                 raise
                 else:
+                    if ProxyExtrasDBManager.spend_logs_is_partitioned():
+                        raise RuntimeError(PARTITIONED_SPEND_LOGS_PUSH_ERROR)
                     # Use prisma db push with increased timeout
                     subprocess.run(
                         [_get_prisma_command(), "db", "push", "--accept-data-loss"],

litellm-proxy-extras/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "litellm-proxy-extras"
-version = "0.4.89"
+version = "0.4.90"
 description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package."
 readme = "README.md"
 requires-python = ">=3.9"
@@ -26,7 +26,7 @@ required-version = ">=0.10.9"
 module-root = ""
 
 [tool.commitizen]
-version = "0.4.89"
+version = "0.4.90"
 version_files = [
     "pyproject.toml:^version",
     "../pyproject.toml:litellm-proxy-extras==",

litellm/__init__.py

@@ -445,6 +445,7 @@ max_ui_session_budget: Optional[float] = (
     1.0  # USD budget for each dashboard login session (playground, test connection)
 )
 internal_user_budget_duration: Optional[str] = None
+budget_rollover: bool = False  # carry spend beyond max_budget into the next window instead of zeroing it
 tag_budget_config: Optional[Dict[str, "BudgetConfig"]] = None
 max_end_user_budget: Optional[float] = None
 max_end_user_budget_id: Optional[str] = None

litellm/constants.py

@@ -1473,6 +1473,12 @@ LITELLM_PROXY_MASTER_KEY_ALIAS: Final = "litellm_proxy_master_key"
 # ``ProxyLogging._handle_logging_proxy_only_error``.
 LITELLM_LOGGING_NO_UPSTREAM_LLM_CALL: Final = "litellm_no_upstream_llm_call"
 
+# Key/team metadata fields naming the OTel Resource ``service.name``, highest
+# precedence first. Shared between the OTel v2 tenant router (which reads them
+# out of ``user_api_key_auth_metadata``) and proxy request setup (which re-applies
+# the key's values after the team metadata merge so a key outranks its team).
+OTEL_SERVICE_NAME_METADATA_KEYS: Final = ("otel_service_name_override", "otel_service_name")
+
 # Key Rotation Constants
 LITELLM_KEY_ROTATION_ENABLED: Final = os.getenv("LITELLM_KEY_ROTATION_ENABLED", "false")
 LITELLM_KEY_ROTATION_CHECK_INTERVAL_SECONDS: Final = int(
@@ -1646,6 +1652,7 @@ LITELLM_SETTINGS_SAFE_DB_OVERRIDES: Final = [
     "enable_anthropic_prompt_caching",
     "anthropic_prompt_caching_ttl",
     "max_ui_session_budget",
+    "budget_rollover",
 ]
 SPECIAL_LITELLM_AUTH_TOKEN: Final = ["ui-token"]
 DEFAULT_MANAGEMENT_OBJECT_IN_MEMORY_CACHE_TTL = int(os.getenv("DEFAULT_MANAGEMENT_OBJECT_IN_MEMORY_CACHE_TTL", 60))

litellm/cost_calculator.py

@@ -557,9 +557,10 @@ def cost_per_token(
         )
     elif call_type == "atranscription" or call_type == "transcription":
         if _transcription_usage_has_token_details(usage_block):
-            return openai_cost_per_token(
+            return generic_cost_per_token(
                 model=model_without_prefix,
                 usage=usage_block,
+                custom_llm_provider=custom_llm_provider,
                 service_tier=service_tier,
                 data_residency=data_residency,
             )

litellm/integrations/custom_batch_logger.py

@@ -45,7 +45,7 @@ class CustomBatchLogger(CustomLogger):
 
         super().__init__(**kwargs)
 
-    async def periodic_flush(self):
+    async def periodic_flush(self) -> None:
         while True:
             await asyncio.sleep(self.flush_interval)
             verbose_logger.debug("CustomLogger periodic flush after %s seconds", self.flush_interval)

litellm/integrations/dotprompt/prompt_manager.py

@@ -149,7 +149,6 @@ class PromptManager:
                 )
                 self.prompts[template_id] = template
             except Exception:
-                # Optional: print(f"Error loading prompt from JSON: {template_id}")
                 pass
 
     def _load_prompt_file(self, file_path: str | Path, prompt_id: str) -> PromptTemplate:

litellm/integrations/newrelic/newrelic_metrics.py

@@ -0,0 +1,395 @@
+"""
+New Relic Metric API Integration - sends per-team cost/usage metrics to /metric/v1
+
+NR Reference API: https://docs.newrelic.com/docs/data-apis/ingest-apis/metric-api/introduction-metric-api/
+
+`async_log_success_event` / `async_log_failure_event` queue one record per request;
+at flush the queue is aggregated by (team, model group, model, provider, status)
+into count/summary metrics. `interval.ms` is the real window between flushes,
+computed at flush time.
+
+Team-scoped by construction: the ingest key is injected explicitly and there is
+deliberately no environment-variable fallback, so a team's metrics are never sent
+with the proxy operator's credentials (mirrors ``allow_env_credentials=False`` on
+the Datadog team logger).
+
+Error policy on flush: 4xx drops the batch (a retry would fail identically; 403
+is a permanent credential failure), 5xx/network re-queues capped at
+``max_queue_size`` records with the oldest dropped.
+
+For batching specific details see CustomBatchLogger class
+"""
+
+import asyncio
+import gzip
+import time
+import traceback
+from collections.abc import Mapping
+from math import ceil
+from types import MappingProxyType
+from typing import Final
+
+from httpx import HTTPStatusError, Response
+
+from litellm._logging import verbose_logger
+from litellm.integrations.custom_batch_logger import CustomBatchLogger
+from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
+from litellm.llms.custom_httpx.http_handler import (
+    get_async_httpx_client,
+    httpxSpecialProvider,
+)
+from litellm.types.integrations.newrelic import (
+    NEWRELIC_DEFAULT_REGION,
+    NEWRELIC_METRIC_ATTRIBUTE_MAX_LEN,
+    NEWRELIC_METRIC_COMPLETION_TOKENS,
+    NEWRELIC_METRIC_COST_USD,
+    NEWRELIC_METRIC_ENDPOINT_BY_REGION,
+    NEWRELIC_METRIC_PROMPT_TOKENS,
+    NEWRELIC_METRIC_REQUEST_DURATION_MS,
+    NEWRELIC_METRIC_REQUESTS,
+    NEWRELIC_METRIC_TOTAL_TOKENS,
+    NEWRELIC_METRICS_MAX_BATCH_SIZE,
+    NEWRELIC_METRICS_MAX_DRAIN_PASSES,
+    NEWRELIC_METRICS_MAX_RETRY_QUEUE_SIZE,
+    NewRelicCountMetric,
+    NewRelicMetric,
+    NewRelicMetricCommon,
+    NewRelicMetricEnvelope,
+    NewRelicMetricRecord,
+    NewRelicSummaryMetric,
+    NewRelicSummaryValue,
+)
+from litellm.types.utils import StandardLoggingPayload
+
+# 408 (request timeout) and 429 (rate limit) are transient client errors the
+# Metric API expects a retry on, unlike 400/403 which a retry would only repeat.
+_RETRYABLE_CLIENT_STATUSES: Final = frozenset({408, 429})
+
+
+def resolve_newrelic_metric_endpoint(newrelic_region: str | None) -> str:
+    if not newrelic_region:
+        return NEWRELIC_METRIC_ENDPOINT_BY_REGION[NEWRELIC_DEFAULT_REGION]
+    endpoint: Final = NEWRELIC_METRIC_ENDPOINT_BY_REGION.get(newrelic_region.lower())
+    if endpoint is None:
+        verbose_logger.warning(
+            "New Relic: unknown newrelic_region %r; supported regions: %s. Using the default (US) endpoint.",
+            newrelic_region,
+            ", ".join(sorted(NEWRELIC_METRIC_ENDPOINT_BY_REGION)),
+        )
+        return NEWRELIC_METRIC_ENDPOINT_BY_REGION[NEWRELIC_DEFAULT_REGION]
+    return endpoint
+
+
+def _metric_record_from_payload(standard_logging_object: StandardLoggingPayload) -> NewRelicMetricRecord:
+    metadata: Final = standard_logging_object.get("metadata")
+    team_id: Final = ((metadata.get("user_api_key_team_id") or metadata.get("team_id")) if metadata else None) or ""
+    team_alias: Final = (
+        (metadata.get("user_api_key_team_alias") or metadata.get("team_alias")) if metadata else None
+    ) or ""
+    return NewRelicMetricRecord(
+        team_id=team_id,
+        team_alias=team_alias,
+        model_group=standard_logging_object.get("model_group") or "",
+        model=standard_logging_object.get("model") or "",
+        custom_llm_provider=standard_logging_object.get("custom_llm_provider") or "",
+        status=str(standard_logging_object.get("status") or "success"),
+        response_cost=float(standard_logging_object.get("response_cost") or 0.0),
+        prompt_tokens=int(standard_logging_object.get("prompt_tokens") or 0),
+        completion_tokens=int(standard_logging_object.get("completion_tokens") or 0),
+        total_tokens=int(standard_logging_object.get("total_tokens") or 0),
+        duration_ms=float(standard_logging_object.get("response_time") or 0.0) * 1000.0,
+    )
+
+
+def _bucket_metrics(bucket_records: tuple[NewRelicMetricRecord, ...]) -> tuple[NewRelicMetric, ...]:
+    first: Final = bucket_records[0]
+    attributes: Final[Mapping[str, str]] = {  # mutable-ok: JSON leaf; safe_dumps stringifies MappingProxyType
+        key: value[:NEWRELIC_METRIC_ATTRIBUTE_MAX_LEN]
+        for key, value in (
+            ("team_id", first.team_id),
+            ("team_alias", first.team_alias),
+            ("model_group", first.model_group),
+            ("model", first.model),
+            ("custom_llm_provider", first.custom_llm_provider),
+            ("status", first.status),
+        )
+        if value
+    }
+    durations: Final = tuple(record.duration_ms for record in bucket_records)
+    counts: Final[tuple[tuple[str, float], ...]] = (
+        (NEWRELIC_METRIC_REQUESTS, float(len(bucket_records))),
+        (NEWRELIC_METRIC_COST_USD, sum(record.response_cost for record in bucket_records)),
+        (NEWRELIC_METRIC_PROMPT_TOKENS, float(sum(record.prompt_tokens for record in bucket_records))),
+        (NEWRELIC_METRIC_COMPLETION_TOKENS, float(sum(record.completion_tokens for record in bucket_records))),
+        (NEWRELIC_METRIC_TOTAL_TOKENS, float(sum(record.total_tokens for record in bucket_records))),
+    )
+    count_metrics: Final[tuple[NewRelicMetric, ...]] = tuple(
+        NewRelicCountMetric(name=name, type="count", value=value, attributes=attributes) for name, value in counts
+    )
+    summary_metric: Final = NewRelicSummaryMetric(
+        name=NEWRELIC_METRIC_REQUEST_DURATION_MS,
+        type="summary",
+        value=NewRelicSummaryValue(
+            count=len(durations),
+            sum=sum(durations),
+            min=min(durations),
+            max=max(durations),
+        ),
+        attributes=attributes,
+    )
+    return (*count_metrics, summary_metric)
+
+
+def build_metric_payload(
+    records: tuple[NewRelicMetricRecord, ...],
+    *,
+    window_start: float,
+    now: float,
+) -> tuple[NewRelicMetricEnvelope, ...]:
+    """Aggregates records into one Metric API envelope for the flush window."""
+    interval_ms: Final = max(1, int((now - window_start) * 1000))
+    bucket_keys: Final = tuple(dict.fromkeys(record.bucket_key for record in records))
+    metrics: Final = tuple(
+        metric
+        for key in bucket_keys
+        for metric in _bucket_metrics(tuple(record for record in records if record.bucket_key == key))
+    )
+    common: Final[NewRelicMetricCommon] = {
+        "timestamp": int(window_start * 1000),
+        "interval.ms": interval_ms,
+    }
+    return (NewRelicMetricEnvelope(common=common, metrics=metrics),)
+
+
+class NewRelicMetricsLogger(CustomBatchLogger):
+    def __init__(
+        self,
+        newrelic_api_key: str,
+        newrelic_region: str | None = None,
+    ) -> None:
+        if not newrelic_api_key:
+            raise ValueError(
+                "newrelic_api_key is required for NewRelicMetricsLogger; "
+                "team-scoped metrics never fall back to environment credentials"
+            )
+        self.newrelic_api_key: Final = newrelic_api_key
+        self.metric_api_url: Final = resolve_newrelic_metric_endpoint(newrelic_region)
+        self.async_client = get_async_httpx_client(llm_provider=httpxSpecialProvider.LoggingCallback)
+        self._stopped: bool = False
+        self._drain_lock = asyncio.Lock()
+        asyncio.create_task(self.periodic_flush())
+        self.flush_lock = asyncio.Lock()
+        super().__init__(
+            flush_lock=self.flush_lock,
+            batch_size=NEWRELIC_METRICS_MAX_BATCH_SIZE,
+            max_queue_size=NEWRELIC_METRICS_MAX_RETRY_QUEUE_SIZE,
+        )
+
+    def stop(self) -> None:
+        """Ends the periodic flush loop; called on DynamicLoggingCache eviction.
+
+        Schedules one final drain of anything still queued, so eviction never
+        silently discards records. Guarded so it can never raise into the
+        cache's eviction path.
+        """
+        self._stopped = True
+        try:
+            asyncio.get_running_loop().create_task(self._final_drain())
+        except Exception:  # noqa: BLE001  # no running loop / shutdown; the periodic loop's final drain still runs
+            verbose_logger.debug("New Relic Metrics: could not schedule final drain on stop()", exc_info=True)
+
+    async def _drain_with_retry(self) -> None:
+        """Deliver everything queued on a stopped logger, or drop it with a log.
+
+        A stopped logger has no periodic loop left, so every post-stop path
+        funnels through here. ``_drain_lock`` serializes drains: a callback that
+        appends and starts its own drain queues behind the running one instead
+        of racing it. Each pass attempts the whole current queue in
+        ``batch_size`` chunks, unlike the periodic path it does not stop at the
+        first failing chunk, so a persistently failing head never starves the
+        tail. Only after ``_MAX_DRAIN_PASSES`` against a permanently failing
+        destination is the remainder dropped, and then only the records that were
+        queued when this drain began, so every dropped record got the full retry
+        budget: a record a callback appended mid-drain is not in that snapshot,
+        so it is left for its own serialized drain rather than dropped after
+        fewer attempts, and is never stranded.
+        """
+        async with self._drain_lock:
+            attempted: Final = tuple(self.log_queue)
+            for _pass in range(NEWRELIC_METRICS_MAX_DRAIN_PASSES):
+                await self._drain_flush_once()
+                if not self.log_queue:
+                    return
+                if _pass < NEWRELIC_METRICS_MAX_DRAIN_PASSES - 1:
+                    await asyncio.sleep(2**_pass)
+            async with self.flush_lock:
+                tried_ids: Final = frozenset(id(record) for record in attempted)
+                survivors: Final = tuple(record for record in self.log_queue if id(record) not in tried_ids)
+                dropped: Final = len(self.log_queue) - len(survivors)
+                if dropped:
+                    verbose_logger.warning(
+                        "New Relic Metrics: dropping %s records after %s drain passes",
+                        dropped,
+                        NEWRELIC_METRICS_MAX_DRAIN_PASSES,
+                    )
+                self.log_queue[:] = list(survivors)  # mutable-ok: leave late arrivals for the next serialized drain
+
+    async def _drain_flush_once(self) -> None:
+        """Attempt every queued record once, in ``batch_size`` chunks, without
+        stopping at the first failing chunk so a persistently failing head does
+        not starve the tail (the periodic ``flush_queue`` deliberately stops
+        instead). Takes the queue under ``flush_lock`` and re-queues only the
+        chunks a 5xx/network error left undelivered, so records a concurrent
+        request appends during the sends survive for the next pass."""
+        async with self.flush_lock:
+            pending: Final = tuple(self.log_queue)
+            window_start: Final = self.last_flush_time
+            self.last_flush_time = time.time()
+            del self.log_queue[:]
+        if not pending:
+            return
+        chunks: Final = tuple(
+            pending[start : start + self.batch_size] for start in range(0, len(pending), self.batch_size)
+        )
+        delivered: Final = tuple([await self._classify_and_send(chunk, window_start) for chunk in chunks])
+        failed: Final = tuple(record for chunk, ok in zip(chunks, delivered) for record in (() if ok else chunk))
+        if failed:
+            self._requeue(failed)
+
+    async def _final_drain(self) -> None:
+        await self._drain_with_retry()
+
+    async def periodic_flush(self) -> None:
+        while not self._stopped:
+            await asyncio.sleep(self.flush_interval)
+            if self._stopped:
+                break
+            await self.flush_queue()
+        await self._final_drain()
+
+    async def async_log_success_event(self, kwargs, response_obj, start_time, end_time) -> None:
+        try:
+            await self._log_async_event(standard_logging_object=kwargs.get("standard_logging_object", None))
+        except Exception as e:  # noqa: BLE001  # logging must never break the request path
+            verbose_logger.exception("New Relic Metrics Layer Error - %s\n%s", e, traceback.format_exc())
+
+    async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time) -> None:
+        try:
+            await self._log_async_event(standard_logging_object=kwargs.get("standard_logging_object", None))
+        except Exception as e:  # noqa: BLE001  # logging must never break the request path
+            verbose_logger.exception("New Relic Metrics Layer Error - %s\n%s", e, traceback.format_exc())
+
+    async def _log_async_event(self, standard_logging_object: StandardLoggingPayload | None) -> None:
+        if standard_logging_object is None:
+            raise ValueError("standard_logging_object not found in kwargs")
+        self.log_queue.append(_metric_record_from_payload(standard_logging_object))
+        if self._stopped:
+            # A stopped logger has no periodic loop left; an in-flight callback
+            # that appends after the eviction drain delivers its own record.
+            await self._drain_with_retry()
+            return
+        if len(self.log_queue) >= self.batch_size:
+            await self.flush_queue()
+
+    async def flush_queue(self) -> None:
+        async with self.flush_lock:
+            window_start: Final = self.last_flush_time
+            self.last_flush_time = time.time()
+            queued: Final = len(self.log_queue)
+            if not queued:
+                return
+            verbose_logger.debug("New Relic Metrics: Flushing %s queued records", queued)
+            # Bounded by what is queued now: records appended mid-flush belong to
+            # the next window, and looping until empty would never end under load.
+            for _chunk in range(ceil(queued / self.batch_size)):
+                if not await self.async_send_batch(window_start=window_start):
+                    return
+
+    async def async_send_batch(self, window_start: float | None = None) -> bool:
+        """Sends the oldest ``batch_size`` records only, so a queue grown past that
+        by re-queues cannot breach the Metric API data point cap in one request.
+        Returns False once a chunk fails and is re-queued, so the caller stops."""
+        if not self.log_queue:
+            return False
+
+        batch_to_send: Final[tuple[NewRelicMetricRecord, ...]] = tuple(self.log_queue[: self.batch_size])
+        del self.log_queue[: len(batch_to_send)]
+
+        delivered: Final = await self._classify_and_send(
+            batch_to_send, window_start if window_start is not None else self.last_flush_time
+        )
+        if not delivered:
+            self._requeue(batch_to_send)
+        return delivered
+
+    async def _classify_and_send(self, batch: tuple[NewRelicMetricRecord, ...], window_start: float) -> bool:
+        """Send one chunk and classify the outcome, never touching the queue.
+        Returns True when the batch is done with (delivered on any 2xx, or a 4xx
+        a retry would only repeat, 403 being a permanent bad-key rejection), and
+        False when a 5xx or network error means the caller should re-queue it.
+
+        ``AsyncHTTPHandler.post`` raises ``HTTPStatusError`` on any non-2xx, so a
+        4xx never returns a response here; the status is read off the raised
+        error to keep the client-error path (drop) distinct from 5xx (retry)."""
+        payload: Final = build_metric_payload(records=batch, window_start=window_start, now=time.time())
+        try:
+            status = (
+                await self.async_send_compressed_data(payload)
+            ).status_code  # rebind-ok: reassigned from the raised HTTPStatusError below
+        except HTTPStatusError as e:
+            status = e.response.status_code
+        except Exception as e:  # noqa: BLE001  # transport/network failure re-queues the batch
+            verbose_logger.warning(
+                "New Relic Metrics: network error sending %s records, will retry - %s",
+                len(batch),
+                e,
+            )
+            return False
+
+        if 200 <= status < 300:
+            return True
+
+        if 400 <= status < 500 and status not in _RETRYABLE_CLIENT_STATUSES:
+            verbose_logger.warning(
+                "New Relic Metrics: %s from Metric API%s, dropping %s records.",
+                status,
+                " (permanent credential failure: invalid or revoked team ingest key)" if status == 403 else "",
+                len(batch),
+            )
+            return True
+
+        verbose_logger.warning(
+            "New Relic Metrics: %s from Metric API, will retry %s records",
+            status,
+            len(batch),
+        )
+        return False
+
+    def _requeue(self, batch: tuple[NewRelicMetricRecord, ...]) -> None:
+        """Prepends ``batch`` in place (never by assignment: records appended by
+        concurrent requests during the flush await must survive), keeping
+        chronological order so the cap drops the oldest records first."""
+        self.log_queue[:0] = batch
+        overflow: Final = len(self.log_queue) - self.max_queue_size
+        if overflow > 0:
+            del self.log_queue[:overflow]
+            verbose_logger.warning(
+                "New Relic Metrics: retry queue exceeded max_queue_size=%s; dropped %s oldest records.",
+                self.max_queue_size,
+                overflow,
+            )
+
+    async def async_send_compressed_data(self, payload: tuple[NewRelicMetricEnvelope, ...]) -> Response:
+        compressed_data: Final = gzip.compress(safe_dumps(payload).encode("utf-8"))
+        headers: Final[Mapping[str, str]] = MappingProxyType(
+            {
+                "Content-Type": "application/json",
+                "Content-Encoding": "gzip",
+                "Api-Key": self.newrelic_api_key,
+            }
+        )
+        return await self.async_client.post(
+            url=self.metric_api_url,
+            data=compressed_data,
+            headers=headers,
+        )

litellm/integrations/newrelic/newrelic_team_handler.py

@@ -0,0 +1,90 @@
+"""
+New Relic Team Handler
+
+Used to get the NewRelicMetricsLogger for a given request.
+Handles Key/Team Based New Relic metrics, following the same pattern as DataDogHandler.
+"""
+
+from typing import TYPE_CHECKING, Final
+
+from typing_extensions import ReadOnly, TypedDict
+
+from litellm._logging import verbose_logger
+from litellm.litellm_core_utils.litellm_logging import StandardCallbackDynamicParams
+
+from .newrelic_metrics import NewRelicMetricsLogger
+
+if TYPE_CHECKING:
+    from litellm.litellm_core_utils.litellm_logging import DynamicLoggingCache
+
+
+class NewRelicLoggingConfig(TypedDict):
+    newrelic_api_key: ReadOnly[str | None]
+    newrelic_region: ReadOnly[str | None]
+
+
+class NewRelicHandler:
+    @staticmethod
+    def get_newrelic_logger_for_request(
+        standard_callback_dynamic_params: StandardCallbackDynamicParams,
+        in_memory_dynamic_logger_cache: "DynamicLoggingCache",
+    ) -> NewRelicMetricsLogger:
+        """
+        Get a team-scoped NewRelicMetricsLogger for a given request.
+
+        Resolves and caches per-team NewRelicMetricsLogger instances using
+        DynamicLoggingCache, keyed by the team's New Relic credentials. Each unique
+        set of credentials gets its own logger instance with its own batch/flush loop.
+
+        Note: This handler is only called when a team-scoped newrelic_api_key is
+        present. The trace logger for the ``newrelic`` callback (OTel v2 / legacy
+        agent) is managed separately by _init_custom_logger_compatible_class via
+        _in_memory_loggers.
+        """
+        _credentials: Final = NewRelicHandler.get_dynamic_newrelic_logging_config(
+            standard_callback_dynamic_params=standard_callback_dynamic_params,
+        )
+
+        temp_newrelic_logger = in_memory_dynamic_logger_cache.get_cache(
+            credentials=_credentials, service_name="newrelic"
+        )
+
+        if temp_newrelic_logger is None:
+            temp_newrelic_logger = NewRelicHandler._create_newrelic_logger_from_credentials(
+                credentials=_credentials,
+                in_memory_dynamic_logger_cache=in_memory_dynamic_logger_cache,
+            )
+
+        return temp_newrelic_logger
+
+    @staticmethod
+    def _create_newrelic_logger_from_credentials(
+        credentials: NewRelicLoggingConfig,
+        in_memory_dynamic_logger_cache: "DynamicLoggingCache",
+    ) -> NewRelicMetricsLogger:
+        newrelic_logger: Final = NewRelicMetricsLogger(
+            newrelic_api_key=credentials.get("newrelic_api_key") or "",
+            newrelic_region=credentials.get("newrelic_region"),
+        )
+        in_memory_dynamic_logger_cache.set_cache(
+            credentials=credentials,
+            service_name="newrelic",
+            logging_obj=newrelic_logger,
+        )
+        verbose_logger.debug("New Relic: Created and cached new NewRelicMetricsLogger for team-scoped credentials")
+        return newrelic_logger
+
+    @staticmethod
+    def get_dynamic_newrelic_logging_config(
+        standard_callback_dynamic_params: StandardCallbackDynamicParams,
+    ) -> NewRelicLoggingConfig:
+        return NewRelicLoggingConfig(
+            newrelic_api_key=standard_callback_dynamic_params.get("newrelic_api_key"),
+            newrelic_region=standard_callback_dynamic_params.get("newrelic_region"),
+        )
+
+    @staticmethod
+    def _dynamic_newrelic_credentials_are_passed(
+        standard_callback_dynamic_params: StandardCallbackDynamicParams,
+    ) -> bool:
+        return standard_callback_dynamic_params.get("newrelic_api_key") is not None

litellm/integrations/otel/emitter.py

@@ -146,7 +146,7 @@ class SpanEmitter:
         For callers that own and manage their own span lifecycle. ``tracer``
         overrides the bound tracer for this span only, used for per-request
         multi-tenant credential routing. ``links`` records related-but-not-parent
-        spans (e.g. the transport span of an MCP message, per MCP semconv).
+        spans (e.g. the trace context an MCP client propagated in ``params._meta``).
         """
         return (tracer or self._tracer).start_span(
             name,
@@ -196,8 +196,8 @@ class SpanEmitter:
 
         Return the span, or ``None`` if it was deduplicated away. ``tracer``
         overrides the bound tracer for this span, used for per-request routing.
-        ``links`` records related-but-not-parent spans (the transport span of an
-        MCP message).
+        ``links`` records related-but-not-parent spans (e.g. the trace context an
+        MCP client propagated in ``params._meta``).
         """
         # LLM-call and MCP tool-call spans carry a dedup key (their request's
         # call id), so a sync+async double-firing coalesces. ``isinstance`` narrows

litellm/integrations/otel/logger.py

@@ -390,10 +390,10 @@ class OpenTelemetryV2(CustomLogger):
 
         MCP tool calls reach the success/failure callbacks like any other request
         (with ``call_type`` ``call_mcp_tool``), but they are not LLM calls and have
-        no ``pre_call`` carrier — so they get their own CLIENT span here. Per the MCP
-        semconv it parents to the trace context the client propagated in
-        ``params._meta`` (or starts a new root) and links the transport span, rather
-        than nesting under the HTTP/session span. Returns whether it handled the
+        no ``pre_call`` carrier — so they get their own CLIENT span here. It nests
+        under the transport span of the request carrying this message, and trace
+        context the client propagated in ``params._meta`` is recorded as a span
+        link (see ``resolve_mcp_span_context``). Returns whether it handled the
         event, so the caller skips the LLM-call path. The whole span is emitted at
         once (there is no boundary to open it at), deduped on the call id.
         """
@@ -436,9 +436,9 @@ class OpenTelemetryV2(CustomLogger):
 
         Like a tool call, listing reaches the success/failure callbacks (here with
         ``call_type`` ``list_mcp_tools``) with no ``pre_call`` carrier, so it gets its
-        own CLIENT span. Per the MCP semconv it parents to the ``params._meta`` trace
-        context (or starts a new root) and links the transport span, rather than
-        nesting under the HTTP/session span. Returns whether it handled the event so
+        own CLIENT span, nested under the transport span of the request carrying
+        this message with any ``params._meta`` trace context recorded as a span
+        link (see ``resolve_mcp_span_context``). Returns whether it handled the event so
         the caller skips the LLM-call path.
         """
         raw_payload: Final = kwargs.get("standard_logging_object")

litellm/integrations/otel/model/spans.py

@@ -10,6 +10,8 @@ Canonical hierarchy::
     │   └── DB_CALL (CLIENT)          #   its key/user/team lookups nest here
     ├── GUARDRAIL  (INTERNAL)         # request-lifecycle hook, sibling of LLM_CALL
     ├── LLM_CALL   (CLIENT)
+    ├── MCP_TOOL_CALL  (CLIENT)       # nests under the POST carrying the message
+    ├── MCP_LIST_TOOLS (CLIENT)       #   (client-propagated context is a span link)
     └── DB_CALL    (CLIENT)           # e.g. the spend-log write
 
 Guardrails parent to PROXY_REQUEST, not LLM_CALL: pre/during/post-call guardrail
@@ -18,14 +20,14 @@ before the LLM call even starts), so a guardrail is a sibling of the LLM call,
 not a child of it. The emitter parents every span to the ambient OTel context
 (the active server span), which matches this.
 
-MCP spans (``MCP_TOOL_CALL``, ``MCP_LIST_TOOLS``) have two shapes, chosen at emit
-time by :func:`resolve_mcp_span_context`. When the client propagates trace context
-in ``params._meta`` MCP and the HTTP transport are independent contexts per the
-OTel GenAI MCP semconv, so the span parents to that propagated context and records
-the ``PROXY_REQUEST`` transport span as a span *link*, never a parent — the shape
-this registry's ``parent=None, links=PROXY_REQUEST`` entry encodes. When nothing is
-propagated (the common case) the span nests under the transport span of the request
-carrying that message, so the tool call stays in one trace.
+MCP spans (``MCP_TOOL_CALL``, ``MCP_LIST_TOOLS``) are parented at emit time by
+:func:`resolve_mcp_span_context`: they nest under the ``PROXY_REQUEST`` transport
+span of the request carrying that message, so the tool call stays in one trace.
+Trace context the client propagated in ``params._meta`` (SEP-414) is recorded as
+a span *link*, never the parent — a remote parent would root the span in a trace
+whose root never reaches the gateway's tracing backend. Links always target that
+remote client context, never a registry role, so ``SpanSpec`` declares no link
+field; the concrete transport parent is resolved per message at emit time.
 
 Not every service call becomes a span — :func:`span_role_for_service` decides:
 
@@ -85,25 +87,19 @@ class SpanSpec:
     role: SpanRole
     kind: LiteLLMSpanKind
     parent: SpanRole | None
-    links: SpanRole | None = None
 
 
 SPAN_REGISTRY: Final[dict[SpanRole, SpanSpec]] = {
     SpanRole.PROXY_REQUEST: SpanSpec(SpanRole.PROXY_REQUEST, LiteLLMSpanKind.SERVER, parent=None),
     SpanRole.LLM_CALL: SpanSpec(SpanRole.LLM_CALL, LiteLLMSpanKind.CLIENT, parent=SpanRole.PROXY_REQUEST),
     # The proxy is an MCP client to the upstream server, so MCP spans are CLIENT
-    # spans. With trace context propagated in ``params._meta``, MCP and the HTTP
-    # transport are independent contexts (OTel GenAI MCP semconv): the span parents
-    # to the propagated context and records the PROXY_REQUEST transport span as a
-    # span *link*, never a parent — the shape ``parent=None, links=PROXY_REQUEST``
-    # encodes. With nothing propagated, ``resolve_mcp_span_context`` nests the span
-    # under that message's transport span instead, keeping the call in one trace.
-    SpanRole.MCP_TOOL_CALL: SpanSpec(
-        SpanRole.MCP_TOOL_CALL, LiteLLMSpanKind.CLIENT, parent=None, links=SpanRole.PROXY_REQUEST
-    ),
-    SpanRole.MCP_LIST_TOOLS: SpanSpec(
-        SpanRole.MCP_LIST_TOOLS, LiteLLMSpanKind.CLIENT, parent=None, links=SpanRole.PROXY_REQUEST
-    ),
+    # spans. ``resolve_mcp_span_context`` nests them under the PROXY_REQUEST
+    # transport span of the request carrying that message (resolved per message at
+    # emit time), keeping the call in one trace. Trace context the client
+    # propagated in ``params._meta`` becomes a span *link* to that remote context,
+    # which is not a registry role, so ``SpanSpec`` has no link field.
+    SpanRole.MCP_TOOL_CALL: SpanSpec(SpanRole.MCP_TOOL_CALL, LiteLLMSpanKind.CLIENT, parent=SpanRole.PROXY_REQUEST),
+    SpanRole.MCP_LIST_TOOLS: SpanSpec(SpanRole.MCP_LIST_TOOLS, LiteLLMSpanKind.CLIENT, parent=SpanRole.PROXY_REQUEST),
     SpanRole.GUARDRAIL: SpanSpec(SpanRole.GUARDRAIL, LiteLLMSpanKind.INTERNAL, parent=SpanRole.PROXY_REQUEST),
     SpanRole.DB_CALL: SpanSpec(SpanRole.DB_CALL, LiteLLMSpanKind.CLIENT, parent=SpanRole.PROXY_REQUEST),
     SpanRole.SERVICE: SpanSpec(SpanRole.SERVICE, LiteLLMSpanKind.INTERNAL, parent=SpanRole.PROXY_REQUEST),
@@ -209,8 +205,8 @@ def service_span_name(data: "ServiceSpanData") -> str:
 
 
 def root_roles() -> list[SpanRole]:
-    """Roles with no in-process parent. They start a new trace unless they adopt a
-    remote parent (e.g. an MCP span joining the client's propagated context)."""
+    """Roles with no in-process parent, i.e. they start a new trace (only the
+    instrumentor-owned ``PROXY_REQUEST`` server span today)."""
     return [role for role, spec in SPAN_REGISTRY.items() if spec.parent is None]
 
 
@@ -227,8 +223,6 @@ def validate_registry(
             raise ValueError(f"SPAN_REGISTRY[{role}] has mismatched role {spec.role}")
         if spec.parent is not None and spec.parent not in reg:
             raise ValueError(f"span role {role} declares unknown parent {spec.parent}")
-        if spec.links is not None and spec.links not in reg:
-            raise ValueError(f"span role {role} declares unknown link target {spec.links}")
     missing: Final = [role for role in SpanRole if role not in reg]
     if missing:
         raise ValueError(f"SPAN_REGISTRY is missing roles: {missing}")

litellm/integrations/otel/plumbing/context.py

@@ -57,8 +57,8 @@ def request_root_span() -> "Span | None":
 
 # The W3C trace-context carrier (``traceparent``/``tracestate``/``baggage``) the
 # MCP client propagated in the current request's ``params._meta``. The MCP gateway
-# sets it per message so the MCP span can parent to the client's span rather than
-# to the transport. A ``ContextVar`` because, like the root-span anchor, it must
+# sets it per message so the MCP span can record the client's span as a span
+# link. A ``ContextVar`` because, like the root-span anchor, it must
 # ride the request task and be readable by the inline success-logging callback.
 _mcp_message_trace_carrier: Final["ContextVar[Mapping[str, str] | None]"] = ContextVar(
     "litellm_otel_mcp_message_trace_carrier", default=None
@@ -148,10 +148,10 @@ def _mcp_transport_span_context() -> "SpanContext | None":
 
     Prefers the transport the gateway published for this specific message; falls
     back to the ambient request anchor for paths that emit an MCP span on the
-    request task itself (the REST MCP endpoints, the SDK). Parenting and linking
-    only need the immutable context, and unlike ``mcp_message_transport_span`` they
-    stay correct against a transport that has already finished, so this does not
-    require the span to still be recording.
+    request task itself (the REST MCP endpoints). Parenting needs only the
+    immutable context, and unlike ``mcp_message_transport_span`` it stays correct
+    against a transport that has already finished, so this does not require the
+    span to still be recording.
     """
     published: Final = _mcp_message_transport_span.get()
     if published is not None:
@@ -222,25 +222,31 @@ def resolve_mcp_span_context(
 ) -> "tuple[Context, tuple[Link, ...]]":
     """Parent context + links for an MCP message span.
 
+    The span always nests under the transport span of the request carrying this
+    message, so a tool call and the ``POST`` that carried it stay in one trace.
+    The transport comes from :func:`_mcp_transport_span_context`, which is the
+    *current message's* POST rather than whatever request happened to open the
+    session, so a long-lived session does not glue every message under its first
+    request.
+
     When the client propagates W3C trace context in the request's ``params._meta``
-    (SEP-414), MCP and the underlying transport are independent lifecycles — one
-    streamable-HTTP session multiplexes many messages, and the client's own span is
-    the truthful parent. So, per the OTel GenAI MCP semconv:
+    (SEP-414), that remote context is recorded as a span *link*, never the parent.
+    The OTel GenAI MCP semconv prefers the inverse (remote parent, transport link),
+    but the gateway's tracing backend only ever receives the gateway's half of such
+    a trace: parenting into the client's trace id roots the span in a trace whose
+    root span never reaches the backend, so the span is unreachable from the trace
+    view and the transport transaction shows a dangling link (observed with
+    clients that propagate synthetic trace ids). Anchoring to the gateway's own
+    request and linking the client's context keeps every trace renderable while
+    preserving the client-side correlation.
 
-    * parent to the trace context the client propagated (a *remote* parent), and
-    * record the transport span as a *link*, never the parent.
-
-    Almost no client implements SEP-414 yet, so in practice nothing is propagated.
-    Rooting the span there splits a single tool call into two disconnected traces
-    joined only by a link, which is how it surfaces in APM: the ``POST`` transaction
-    and the ``tools/call`` span share no trace. With no remote parent to honor,
-    parent to the transport span of the request carrying this message instead, so
-    the call stays in one trace; no link is added since the transport is now the
-    real parent. The transport comes from :func:`_mcp_transport_span_context`, which
-    is the *current message's* POST rather than whatever request happened to open
-    the session, so a long-lived session does not glue every message under its
-    first request. With neither a remote parent nor a transport the returned context
-    carries no span and the span legitimately starts its own root trace.
+    With no transport at all the span starts its own root trace, still carrying
+    the link — the client context is only ever a link, so this event keeps one
+    shape everywhere. Both returned contexts are built on an explicitly empty
+    base, so ambient (stale session) state can never leak in, and the span
+    inherits the transport's sampling decision exactly like every other
+    request-level span — a client's sampled flag neither forces nor suppresses
+    recording.
 
     Only trace context (``traceparent``/``tracestate``) is extracted, never the
     client's W3C Baggage: ``params._meta`` is caller-controlled, and the otel
@@ -251,13 +257,12 @@ def resolve_mcp_span_context(
     never fall through to the ambient (stale session) span.
     """
     source: Final = carrier if carrier is not None else _mcp_message_trace_carrier.get()
-    parent: Final = _PROPAGATOR.extract(dict(source or {}), context=Context())
+    propagated: Final = get_current_span(_PROPAGATOR.extract(dict(source or {}), context=Context()))
+    links: Final = (Link(propagated.get_span_context()),) if is_recordable_span(propagated) else ()
     transport: Final = _mcp_transport_span_context()
-    if is_recordable_span(get_current_span(parent)):
-        return parent, (Link(transport),) if transport is not None else ()
-    if transport is not None:
-        return context_from_span(NonRecordingSpan(transport)), ()
-    return parent, ()
+    if transport is None:
+        return Context(), links
+    return context_from_span(NonRecordingSpan(transport), context=Context()), links
 
 
 def is_recordable_span(obj: object) -> bool:

litellm/integrations/otel/plumbing/routing.py

@@ -2,12 +2,13 @@
 
 When a request carries team/key vendor credentials in
 ``standard_callback_dynamic_params``, or the key/team config resolved at auth
-names a destination project, its spans must export through a
-``TracerProvider`` whose OTLP headers carry those credentials / that project.
-``TenantTracerCache`` builds and caches one provider per distinct
-(credentials, project) pair, and otherwise hands back the logger's default
-tracer. This lets a single logger fan requests out to many tenants without
-needing a logger per tenant.
+names a destination project or a service name, its spans must export through a
+``TracerProvider`` whose OTLP headers carry those credentials / that project,
+or whose Resource carries that ``service.name``. ``TenantTracerCache`` builds
+and caches one provider per distinct (credentials, project, service name)
+tuple, and otherwise hands back the logger's default tracer. This lets a
+single logger fan requests out to many tenants without needing a logger per
+tenant.
 """
 
 import threading
@@ -22,6 +23,7 @@ from opentelemetry.sdk.trace import TracerProvider
 from opentelemetry.trace import Tracer
 
 from litellm._logging import verbose_logger
+from litellm.constants import OTEL_SERVICE_NAME_METADATA_KEYS
 from litellm.integrations.otel.model.config import ExporterSpec, OpenTelemetryV2Config
 from litellm.integrations.otel.plumbing.providers import (
     build_tracer_provider,
@@ -65,8 +67,30 @@ _MAX_RETIRED_PROVIDERS: Final = 64
 
 _HeaderItems: TypeAlias = tuple[tuple[str, str], ...]
 
+_RouteKey: TypeAlias = tuple[_HeaderItems, _HeaderItems, str | None, str | None]
+
 _NO_HEADERS: Final[Mapping[str, str]] = MappingProxyType({})
 
+#: Key/team config fields naming the Resource ``service.name``, highest
+#: precedence first. Read only from ``user_api_key_auth_metadata`` (the config
+#: the proxy resolved at auth), never from client-supplied request metadata:
+#: the service name picks the dataset/service traces land in (Honeycomb routes
+#: datasets by it), so a caller must not be able to choose one.
+_SERVICE_NAME_KEYS: Final = OTEL_SERVICE_NAME_METADATA_KEYS
+
+
+def tenant_service_name(auth_metadata: Mapping[str, str] | None) -> str | None:
+    """The per-request ``service.name`` override for this key/team, if any.
+
+    ``None`` keeps the env-configured default (``OTEL_SERVICE_NAME``).
+    """
+    if not auth_metadata:
+        return None
+    return next(
+        (stripped for key in _SERVICE_NAME_KEYS if (stripped := (auth_metadata.get(key) or "").strip())),
+        None,
+    )
+
 
 def _shutdown_provider(provider: TracerProvider) -> None:
     """Flush + stop an evicted provider's processors (reclaims their threads).
@@ -116,7 +140,7 @@ class TenantRoute:
 
 
 class TenantTracerCache:
-    """Credential/project-scoped ``TracerProvider`` cache keyed by the routing headers."""
+    """Tenant-scoped ``TracerProvider`` cache keyed by routing headers and service name."""
 
     def __init__(
         self,
@@ -131,7 +155,7 @@ class TenantTracerCache:
         # thread-pool workers concurrently with the event loop, so cache
         # updates, span counts, and retirement must be atomic.
         self._lock: Final = threading.Lock()
-        self._providers: OrderedDict[tuple[_HeaderItems, _HeaderItems, str | None], TracerProvider] = (
+        self._providers: OrderedDict[_RouteKey, TracerProvider] = (
             OrderedDict()  # mutable-ok: bounded LRU; eviction needs in-place ordered mutation
         )
         self._open_span_counts: dict[TracerProvider, int] = {}  # mutable-ok: live refcount state
@@ -172,10 +196,11 @@ class TenantTracerCache:
     ) -> TenantRoute:
         """Return the tracer (and trace-detachment flag) for this request.
 
-        Use ``default`` unless the request's dynamic credentials or its key/team
-        project require a scoped tracer, in which case build (or reuse) one. The
-        cache is a bounded LRU: the least-recently-used provider is flushed and
-        shut down on overflow so its exporter threads don't accumulate.
+        Use ``default`` unless the request's dynamic credentials, its key/team
+        project, or its key/team service name require a scoped tracer, in
+        which case build (or reuse) one. The cache is a bounded LRU: the
+        least-recently-used provider is flushed and shut down on overflow so
+        its exporter threads don't accumulate.
 
         A routed provider is returned already held — its open-span count is
         incremented in the same critical section as the cache update — so a
@@ -184,7 +209,8 @@ class TenantTracerCache:
         """
         credential_headers: Final = dynamic_otlp_headers(self._callback_name, dynamic_params) or _NO_HEADERS
         project_headers: Final = self._project_headers(auth_metadata)
-        if not credential_headers and not project_headers:
+        service_name: Final = tenant_service_name(auth_metadata)
+        if not credential_headers and not project_headers and service_name is None:
             return TenantRoute(tracer=default, detached=False)
         # A fixed per-integration region endpoint (New Relic us/eu), never a
         # caller-supplied host; ``None`` keeps the preset's own endpoint.
@@ -193,9 +219,12 @@ class TenantTracerCache:
             tuple(sorted(credential_headers.items())),
             tuple(sorted(project_headers.items())),
             endpoint,
+            service_name,
         )
         with self._lock:
-            provider: Final = self._cached_provider_locked(cache_key, credential_headers, project_headers, endpoint)
+            provider: Final = self._cached_provider_locked(
+                cache_key, credential_headers, project_headers, endpoint, service_name
+            )
             self._open_span_counts[provider] = self._open_span_counts.get(provider, 0) + 1
             evicted: Final = self._evicted_on_overflow_locked()
         if evicted is not None:
@@ -208,16 +237,19 @@ class TenantTracerCache:
 
     def _cached_provider_locked(
         self,
-        cache_key: tuple[_HeaderItems, _HeaderItems, str | None],
+        cache_key: _RouteKey,
         credential_headers: Mapping[str, str],
         project_headers: Mapping[str, str],
         endpoint: str | None,
+        service_name: str | None,
     ) -> TracerProvider:
         cached: Final = self._providers.get(cache_key)
         if cached is not None:
             self._providers.move_to_end(cache_key)
             return cached
-        built: Final = build_tracer_provider(self._routed_config(credential_headers, project_headers, endpoint))
+        built: Final = build_tracer_provider(
+            self._routed_config(credential_headers, project_headers, endpoint, service_name)
+        )
         self._providers[cache_key] = built
         return built
 
@@ -267,6 +299,7 @@ class TenantTracerCache:
         credential_headers: Mapping[str, str],
         project_headers: Mapping[str, str],
         endpoint: str | None = None,
+        service_name: str | None = None,
     ) -> OpenTelemetryV2Config:
         """Clone the config, rewriting headers on the callback's own exporter.
 
@@ -285,7 +318,10 @@ class TenantTracerCache:
             self._routed_exporter(spec, credential_headers, project_headers, endpoint)
             for spec in self._config.exporters
         ]
-        return self._config.model_copy(update={"exporters": exporters})
+        update: Final = (
+            {"exporters": exporters} if service_name is None else {"exporters": exporters, "service_name": service_name}
+        )
+        return self._config.model_copy(update=update)
 
     def _routed_exporter(
         self,

litellm/litellm_core_utils/exception_mapping_utils.py

@@ -2341,6 +2341,7 @@ def exception_type(
     litellm_response_headers: Final = _get_response_headers(original_exception=original_exception)
     try:
         error_str = redact_string(str(original_exception)) if _ENABLE_SECRET_REDACTION else str(original_exception)
+        extra_information = ""
         if model or custom_llm_provider:
             if hasattr(original_exception, "message"):
                 error_str = (
@@ -2357,7 +2358,6 @@ def exception_type(
             # Common Extra information needed for all providers
             # We pass num retries, api_base, vertex_deployment etc to the exception here
             ################################################################################
-            extra_information = ""
             try:
                 _api_base: Final = litellm.get_api_base(model=model, optional_params=extra_kwargs)
                 messages: Final = litellm.get_first_chars_messages(kwargs=completion_kwargs)

litellm/litellm_core_utils/litellm_logging.py

@@ -613,37 +613,60 @@ class Logging(LiteLLMLoggingBaseClass):
         processed_list: Final[list[str | Callable | CustomLogger]] = []
         for callback in callback_list:
             if isinstance(callback, str) and callback in litellm._known_custom_logger_compatible_callbacks:
-                # For callbacks that support team-scoped credentials (e.g. datadog),
-                # pass only the relevant dynamic params as custom_logger_init_args.
-                _custom_logger_init_args: dict | None = None
-                if callback == "datadog":
-                    # dd_* params are blocked from standard_callback_dynamic_params
-                    # (request-level security); only the proxy-stamped team/key
-                    # callback vars are admin-configured and trusted.
-                    _custom_logger_init_args = {k: v for k, v in self._trusted_callback_vars if k.startswith("dd_")}
-
-                callback_class = _init_custom_logger_compatible_class(
-                    callback,
-                    internal_usage_cache=None,
-                    llm_router=None,
-                    custom_logger_init_args=_custom_logger_init_args,
-                )
-                if callback_class is not None:
-                    processed_list.append(callback_class)
+                for callback_instance in self._resolve_dynamic_callback_string(callback):
+                    processed_list.append(callback_instance)
 
                     # If processing dynamic_success_callbacks, add to dynamic_async_success_callbacks
                     if dynamic_callbacks_type == "success":
                         if self.dynamic_async_success_callbacks is None:
                             self.dynamic_async_success_callbacks = []
-                        self.dynamic_async_success_callbacks.append(callback_class)
+                        self.dynamic_async_success_callbacks.append(callback_instance)
                     elif dynamic_callbacks_type == "failure":
                         if self.dynamic_async_failure_callbacks is None:
                             self.dynamic_async_failure_callbacks = []
-                        self.dynamic_async_failure_callbacks.append(callback_class)
+                        self.dynamic_async_failure_callbacks.append(callback_instance)
             else:
                 processed_list.append(callback)
         return processed_list
 
+    def _resolve_dynamic_callback_string(self, callback: str) -> "tuple[CustomLogger, ...]":
+        """
+        Resolve a known callback name to the logger instance(s) it dispatches to.
+
+        For callbacks that support team-scoped credentials (datadog, newrelic),
+        only the proxy-stamped team/key callback vars are passed as
+        custom_logger_init_args: dd_*/newrelic_* params are blocked from
+        standard_callback_dynamic_params (request-level security), so the
+        trusted-vars channel is the only way credentials reach a per-team logger.
+        """
+        _trusted_var_prefix: Final = "dd_" if callback == "datadog" else "newrelic_" if callback == "newrelic" else None
+        _custom_logger_init_args: Final[dict | None] = (
+            {k: v for k, v in self._trusted_callback_vars if k.startswith(_trusted_var_prefix)}
+            if _trusted_var_prefix is not None
+            else None
+        )
+
+        callback_class: Final = _init_custom_logger_compatible_class(
+            callback,
+            internal_usage_cache=None,
+            llm_router=None,
+            custom_logger_init_args=_custom_logger_init_args,
+        )
+        if callback_class is None:
+            return ()
+
+        # With team creds, "newrelic" resolves to the per-team METRICS logger;
+        # resolve the name again without creds so the trace logger (OTel v2 /
+        # legacy agent) keeps receiving this request.
+        _newrelic_trace_class: Final = (
+            _init_custom_logger_compatible_class(callback, internal_usage_cache=None, llm_router=None)
+            if callback == "newrelic" and _custom_logger_init_args and _custom_logger_init_args.get("newrelic_api_key")
+            else None
+        )
+        if _newrelic_trace_class is not None and _newrelic_trace_class is not callback_class:
+            return (callback_class, _newrelic_trace_class)
+        return (callback_class,)
+
     def initialize_standard_callback_dynamic_params(self, kwargs: dict | None = None) -> StandardCallbackDynamicParams:
         """
         Initialize the standard callback dynamic params from the kwargs
@@ -4642,6 +4665,19 @@ def _init_custom_logger_compatible_class(
             _in_memory_loggers.append(gitlab_logger)
             return gitlab_logger
         elif logging_integration == "newrelic":
+            if custom_logger_init_args.get("newrelic_api_key"):
+                # Team-scoped credentials: per-team METRICS logger, isolated per
+                # credential set via DynamicLoggingCache. The trace logger for
+                # this name stays on the global path below.
+                from litellm.integrations.newrelic.newrelic_team_handler import (
+                    NewRelicHandler,
+                )
+
+                return NewRelicHandler.get_newrelic_logger_for_request(
+                    standard_callback_dynamic_params=custom_logger_init_args,
+                    in_memory_dynamic_logger_cache=in_memory_dynamic_logger_cache,
+                )
+
             _v2 = _maybe_construct_otel_v2("newrelic", _in_memory_loggers)
             if _v2 is not None:
                 return _v2

litellm/litellm_core_utils/llm_cost_calc/tool_call_cost_tracking.py

@@ -7,7 +7,9 @@ from typing import Any, Final, Literal
 
 import litellm
 from litellm.constants import OPENAI_FILE_SEARCH_COST_PER_1K_CALLS
-from litellm.litellm_core_utils.llm_cost_calc.utils import _get_web_search_requests
+from litellm.litellm_core_utils.llm_cost_calc.utils import (
+    get_web_search_requests_from_usage,
+)
 from litellm.types.llms.openai import (
     FileSearchTool,
     ResponsesAPIResponse,
@@ -368,7 +370,7 @@ class StandardBuiltInToolCostTracking:
             get_anthropic_web_search_requests_from_response,
         )
 
-        if usage is not None and (_get_web_search_requests(getattr(usage, "server_tool_use", None)) is not None):
+        if usage is not None and (get_web_search_requests_from_usage(usage) is not None):
             return usage
         web_search_requests: Final = get_anthropic_web_search_requests_from_response(response_object)
         if web_search_requests is None:
@@ -416,7 +418,7 @@ class StandardBuiltInToolCostTracking:
                 # Anthropic Claude (direct API and Vertex AI) uses server_tool_use.web_search_requests.
                 # Without this check, Claude ModelResponse always falls through to return False
                 # and _handle_web_search_cost() is never called.
-                if hasattr(usage, "server_tool_use") and _get_web_search_requests(usage.server_tool_use) is not None:
+                if get_web_search_requests_from_usage(usage) is not None:
                     return True
                 # xAI reports usage.server_side_tool_usage_details.web_search_calls; a searched
                 # answer with no url_citation annotations has no other chat-path signal
@@ -429,16 +431,12 @@ class StandardBuiltInToolCostTracking:
                 response_object=response_object, output_type="web_search_call"
             )
         elif usage is not None:
-            if (
-                hasattr(usage, "server_tool_use")
-                and _get_web_search_requests(usage.server_tool_use) is not None
-                or (
-                    hasattr(usage, "prompt_tokens_details")
-                    and usage.prompt_tokens_details is not None
-                    and isinstance(usage.prompt_tokens_details, PromptTokensDetailsWrapper)
-                    and hasattr(usage.prompt_tokens_details, "web_search_requests")
-                    and usage.prompt_tokens_details.web_search_requests is not None
-                )
+            if get_web_search_requests_from_usage(usage) is not None or (
+                hasattr(usage, "prompt_tokens_details")
+                and usage.prompt_tokens_details is not None
+                and isinstance(usage.prompt_tokens_details, PromptTokensDetailsWrapper)
+                and hasattr(usage.prompt_tokens_details, "web_search_requests")
+                and usage.prompt_tokens_details.web_search_requests is not None
             ):
                 return True
             if _usage_reports_server_side_web_search_calls(usage):

litellm/litellm_core_utils/llm_cost_calc/utils.py

@@ -72,7 +72,7 @@ def _get_token_detail_value(details: object, key: str) -> int | None:
     return value if isinstance(value, int) else None
 
 
-def _get_web_search_requests(server_tool_use: Any) -> int | None:
+def get_web_search_requests(server_tool_use: Any) -> int | None:
     """
     Tolerantly read ``web_search_requests`` from a ``server_tool_use`` value
     that may be ``None``, a ``dict``, a ``ServerToolUse`` pydantic instance,
@@ -92,6 +92,16 @@ def _get_web_search_requests(server_tool_use: Any) -> int | None:
     return getattr(server_tool_use, "web_search_requests", None)
 
 
+def get_web_search_requests_from_usage(usage: Usage) -> int | None:
+    """Read ``web_search_requests`` from a ``Usage``'s ``server_tool_use``.
+
+    ``Usage`` deletes unset optional fields from ``__dict__`` (see
+    ``SafeAttributeModel``), so direct attribute access can raise
+    ``AttributeError``; ``getattr`` with a default is required here.
+    """
+    return get_web_search_requests(getattr(usage, "server_tool_use", None))
+
+
 def _is_above_128k(tokens: float) -> bool:
     if tokens > 128000:
         return True

litellm/litellm_core_utils/realtime_streaming.py

@@ -330,6 +330,24 @@ class RealTimeStreaming:
         except (AttributeError, TypeError):
             pass
 
+    def _flush_unbilled_transcription_usage(self) -> None:
+        if self.provider_config is None:
+            return
+        usage: Final = self.provider_config.unbilled_usage_on_session_close(self.model)
+        if usage is None:
+            return
+        flush_event: Final = (
+            cast(  # cast-ok: usage-only partial event, the same shape _capture_transcription_usage logs
+                OpenAIRealtimeEvents,
+                {
+                    "type": "conversation.item.input_audio_transcription.completed",
+                    "usage": usage,
+                },
+            )
+        )
+        self.store_message(flush_event)
+        self._capture_transcription_usage(flush_event)
+
     def _collect_tool_calls_from_response_done(self, event_obj: dict | OpenAIRealtimeEvents) -> None:
         """Extract function_call items from response.done events for spend logging."""
         try:
@@ -955,6 +973,7 @@ class RealTimeStreaming:
                 transcript = event.get("transcript", "")
                 self._collect_user_input_from_backend_event(cast(dict, event))
                 self.store_message(event_str)
+                self._capture_transcription_usage(event)
                 await self._send_event_to_client(event, event_str)
                 blocked = await self.run_realtime_guardrails(
                     cast(str, transcript),
@@ -1068,6 +1087,7 @@ class RealTimeStreaming:
         except Exception as e:
             verbose_logger.exception("Error in backend to client send messages: %s", e)
         finally:
+            self._flush_unbilled_transcription_usage()
             await self.log_messages()
 
     @staticmethod

litellm/litellm_core_utils/specialty_caches/dynamic_logging_cache.py

@@ -13,6 +13,7 @@ import json
 from typing import Any, Final
 
 import litellm
+from litellm._logging import verbose_logger
 from litellm.constants import _DEFAULT_TTL_FOR_HTTPX_CLIENTS
 
 from ...caching import InMemoryCache
@@ -46,6 +47,15 @@ class LangfuseInMemoryCache(InMemoryCache):
             _created_langfuse_logger.Langfuse.flush()
             _created_langfuse_logger.Langfuse.shutdown()
 
+        # Loggers with a periodic flush task (e.g. NewRelicMetricsLogger) expose
+        # stop() so eviction actually ends the task instead of leaking it.
+        _evicted_stop: Final = getattr(self.cache_dict[key], "stop", None)
+        if callable(_evicted_stop):
+            try:
+                _evicted_stop()
+            except Exception:  # noqa: BLE001  # a failing stop() must not block eviction
+                verbose_logger.debug("DynamicLoggingCache: stop() raised during eviction", exc_info=True)
+
         #########################################################
         # Call parent class to remove key from cache
         #########################################################

litellm/llms/anthropic/cost_calculation.py

@@ -8,9 +8,9 @@ from typing import TYPE_CHECKING, Final, Optional
 from pydantic import BaseModel, ValidationError
 
 from litellm.litellm_core_utils.llm_cost_calc.utils import (
-    _get_web_search_requests,
     generic_cost_per_token,
     get_provider_specific_geo_multiplier,
+    get_web_search_requests_from_usage,
 )
 
 if TYPE_CHECKING:
@@ -104,7 +104,7 @@ def get_cost_for_anthropic_web_search(
 
     if usage is None:
         return 0.0
-    web_search_requests: Final = _get_web_search_requests(getattr(usage, "server_tool_use", None))
+    web_search_requests: Final = get_web_search_requests_from_usage(usage)
     if web_search_requests is None:
         return 0.0
 

litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py

@@ -99,6 +99,7 @@ from litellm.types.llms.anthropic import (
     ContextManagementResponse,
     MessageBlockDelta,
     MessageDelta,
+    ServerToolUsage,
     StreamingContentBlockDeltaType,
     UsageDelta,
     UsageIteration,
@@ -1354,10 +1355,22 @@ class LiteLLMAnthropicMessagesAdapter:
             return explicit_value
         return cls._first_positive_prompt_tokens_detail_value(usage, ("cache_creation_tokens", "cache_write_tokens"))
 
+    @classmethod
+    def _get_web_search_request_count(cls, usage: Usage) -> int:
+        from litellm.litellm_core_utils.llm_cost_calc.utils import (
+            get_web_search_requests_from_usage,
+        )
+
+        from_server_tool_use: Final = cls._positive_int(get_web_search_requests_from_usage(usage))
+        if from_server_tool_use > 0:
+            return from_server_tool_use
+        return cls._first_positive_prompt_tokens_detail_value(usage, ("web_search_requests",))
+
     @classmethod
     def _translate_openai_usage_to_anthropic_usage_delta(cls, usage: Usage) -> UsageDelta:
         cache_read_input_tokens: Final = cls._get_cache_read_input_tokens(usage)
         cache_creation_input_tokens: Final = cls._get_cache_creation_input_tokens(usage)
+        web_search_requests: Final = cls._get_web_search_request_count(usage)
         input_tokens: Final = max(
             (usage.prompt_tokens or 0) - cache_read_input_tokens - cache_creation_input_tokens,
             0,
@@ -1371,6 +1384,11 @@ class LiteLLMAnthropicMessagesAdapter:
             usage_delta["cache_creation_input_tokens"] = cache_creation_input_tokens
         if cache_read_input_tokens > 0:
             usage_delta["cache_read_input_tokens"] = cache_read_input_tokens
+        if web_search_requests > 0:
+            return UsageDelta(
+                **usage_delta,
+                server_tool_use=ServerToolUsage(web_search_requests=web_search_requests),
+            )
         return usage_delta
 
     @classmethod

litellm/llms/base_llm/realtime/transformation.py

@@ -5,6 +5,7 @@ import httpx
 
 from litellm.types.llms.openai import OpenAIRealtimeStreamSessionEvents
 from litellm.types.realtime import (
+    RealtimeInputAudioTranscriptionUsage,
     RealtimeResponseTransformInput,
     RealtimeResponseTypedDict,
 )
@@ -70,6 +71,9 @@ class BaseRealtimeConfig(ABC):
     def session_configuration_request(self, model: str) -> str | None:  # message sent to setup the realtime session
         return None
 
+    def unbilled_usage_on_session_close(self, model: str) -> RealtimeInputAudioTranscriptionUsage | None:
+        return None
+
     def transform_session_created_event(
         self,
         model: str,

litellm/llms/bedrock/base_aws_llm.py

@@ -1434,9 +1434,12 @@ class BaseAWSLLM:
         data: str | bytes,
         headers: dict,
         api_key: str | None = None,
+        supports_bearer_token: bool = True,
     ) -> AWSPreparedRequest:
-        if api_key is not None:
-            aws_bearer_token: str | None = api_key
+        if not supports_bearer_token:
+            aws_bearer_token: str | None = None
+        elif api_key is not None:
+            aws_bearer_token = api_key
         else:
             aws_bearer_token = get_secret_str("AWS_BEARER_TOKEN_BEDROCK")
 

litellm/llms/bedrock/rerank/handler.py

@@ -138,11 +138,6 @@ class BedrockRerankHandler(BaseAWSLLM):
         data: dict,
         optional_params: dict,
     ) -> BedrockPreparedRequest:
-        try:
-            from botocore.auth import SigV4Auth
-            from botocore.awsrequest import AWSRequest
-        except ImportError:
-            raise ImportError("Missing boto3 to call bedrock. Run 'pip install boto3'.")
         boto3_credentials_info: Final = self._get_boto_credentials_from_optional_params(optional_params, model)
 
         ### SET RUNTIME ENDPOINT ###
@@ -153,24 +148,21 @@ class BedrockRerankHandler(BaseAWSLLM):
         )
         proxy_endpoint_url = proxy_endpoint_url.replace("bedrock-runtime", "bedrock-agent-runtime")
         proxy_endpoint_url = f"{proxy_endpoint_url}/rerank"
-        sigv4: Final = SigV4Auth(
-            boto3_credentials_info.credentials,
-            "bedrock",
-            boto3_credentials_info.aws_region_name,
-        )
-        # Make POST Request
-        body: Final = json.dumps(data).encode("utf-8")
 
+        body: Final = json.dumps(data).encode("utf-8")
         headers = {"Content-Type": "application/json"}
         if extra_headers is not None:
             headers = {"Content-Type": "application/json", **extra_headers}
-        request: Final = AWSRequest(method="POST", url=proxy_endpoint_url, data=body, headers=headers)
-        sigv4.add_auth(request)
-        if (
-            extra_headers is not None and "Authorization" in extra_headers
-        ):  # prevent sigv4 from overwriting the auth header
-            request.headers["Authorization"] = extra_headers["Authorization"]
-        prepped: Final = request.prepare()
+
+        prepped: Final = self.get_request_headers(
+            credentials=boto3_credentials_info.credentials,
+            aws_region_name=boto3_credentials_info.aws_region_name,
+            extra_headers=extra_headers,
+            endpoint_url=proxy_endpoint_url,
+            data=body,
+            headers=headers,
+            supports_bearer_token=False,
+        )
 
         return BedrockPreparedRequest(
             endpoint_url=proxy_endpoint_url,

litellm/llms/gemini/audio_transcription/transformation.py

@@ -0,0 +1,250 @@
+import base64
+from collections.abc import Mapping, Sequence
+from typing import Final
+
+from httpx import Headers, Response
+
+from litellm.litellm_core_utils.audio_utils.utils import (
+    normalize_transcription_language_to_bcp47,
+    process_audio_file,
+)
+from litellm.llms.base_llm.audio_transcription.transformation import (
+    AudioTranscriptionRequestData,
+    BaseAudioTranscriptionConfig,
+)
+from litellm.llms.base_llm.chat.transformation import BaseLLMException
+from litellm.llms.gemini.common_utils import GeminiError, GeminiModelInfo
+from litellm.types.llms.gemini_audio_transcription import (
+    GeminiTranscriptionAudioInput,
+    GeminiTranscriptionConfig,
+    GeminiTranscriptionInteractionRequest,
+    GeminiTranscriptionInteractionResponse,
+    GeminiTranscriptionWordAnnotation,
+)
+from litellm.types.llms.openai import (
+    AllMessageValues,
+    OpenAIAudioTranscriptionOptionalParams,
+)
+from litellm.types.utils import (
+    FileTypes,
+    TranscriptionResponse,
+    TranscriptionUsageInputTokenDetailsObject,
+    TranscriptionUsageTokensObject,
+)
+
+INTERACTIONS_API_REVISION: Final = "2026-05-20"
+WORD_INFO_ANNOTATION_TYPE: Final = "word_info"
+
+
+class GeminiAudioTranscriptionConfig(BaseAudioTranscriptionConfig):
+    """
+    Maps OpenAI /v1/audio/transcriptions onto the Gemini Interactions API
+    (POST /v1beta/interactions) for transcription models like
+    gemini-3.5-transcribe. https://ai.google.dev/gemini-api/docs/transcribe
+    """
+
+    def get_supported_openai_params(
+        self, model: str
+    ) -> list[OpenAIAudioTranscriptionOptionalParams]:  # mutable-ok: BaseAudioTranscriptionConfig signature
+        return ["language", "response_format", "timestamp_granularities"]  # mutable-ok: base contract returns a list
+
+    def map_openai_params(
+        self,
+        non_default_params: Mapping[str, object],
+        optional_params: Mapping[str, object],
+        model: str,
+        drop_params: bool,
+    ) -> dict:  # mutable-ok: BaseAudioTranscriptionConfig signature
+        supported_params: Final = frozenset(self.get_supported_openai_params(model))
+        accepted: Final = tuple((k, v) for k, v in non_default_params.items() if k in supported_params)
+        return dict((*optional_params.items(), *accepted))  # mutable-ok: base contract returns a plain dict
+
+    def get_error_class(
+        self,
+        error_message: str,
+        status_code: int,
+        headers: dict | Headers,  # mutable-ok: base signature and BaseLLMException take dict | Headers
+    ) -> BaseLLMException:
+        return GeminiError(status_code=status_code, message=error_message, headers=headers)
+
+    def validate_environment(
+        self,
+        headers: Mapping[str, str],
+        model: str,
+        messages: Sequence[AllMessageValues],
+        optional_params: Mapping[str, object],
+        litellm_params: Mapping[str, object],
+        api_key: str | None = None,
+        api_base: str | None = None,
+    ) -> dict:  # mutable-ok: BaseAudioTranscriptionConfig signature
+        resolved_api_key: Final = GeminiModelInfo.get_api_key(api_key)
+        if not resolved_api_key:
+            raise GeminiError(
+                status_code=401,
+                message="Google API key is required. Set GOOGLE_API_KEY or GEMINI_API_KEY environment variable.",
+            )
+        return {  # mutable-ok: the http handler passes these headers straight to httpx
+            **headers,
+            "Content-Type": "application/json",
+            "x-goog-api-key": resolved_api_key,
+            "Api-Revision": INTERACTIONS_API_REVISION,
+        }
+
+    def get_complete_url(
+        self,
+        api_base: str | None,
+        api_key: str | None,
+        model: str,
+        optional_params: Mapping[str, object],
+        litellm_params: Mapping[str, object],
+        stream: bool | None = None,
+    ) -> str:
+        resolved_api_base: Final = GeminiModelInfo.get_api_base(api_base)
+        return f"{resolved_api_base}/v1beta/interactions"
+
+    def transform_audio_transcription_request(
+        self,
+        model: str,
+        audio_file: FileTypes,
+        optional_params: Mapping[str, object],
+        litellm_params: Mapping[str, object],
+    ) -> AudioTranscriptionRequestData:
+        processed_audio: Final = process_audio_file(audio_file)
+        audio_input: Final = GeminiTranscriptionAudioInput(
+            type="audio",
+            data=base64.b64encode(processed_audio.file_content).decode("utf-8"),
+            mime_type=processed_audio.content_type,
+        )
+        request: Final = _build_interaction_request(
+            model=model,
+            audio_input=audio_input,
+            transcription_config=_build_transcription_config(optional_params),
+        )
+        return AudioTranscriptionRequestData(data=dict(request))  # mutable-ok: AudioTranscriptionRequestData wants dict
+
+    def transform_audio_transcription_response(
+        self,
+        raw_response: Response,
+    ) -> TranscriptionResponse:
+        try:
+            response_json: Final = raw_response.json()
+        except ValueError:
+            raise GeminiError(
+                status_code=raw_response.status_code,
+                message=f"Received non-JSON response from Gemini Interactions API: {raw_response.text}",
+            )
+        parsed: Final = GeminiTranscriptionInteractionResponse.model_validate(response_json)
+        if parsed.status != "completed":
+            raise GeminiError(
+                status_code=raw_response.status_code,
+                message=f"Gemini transcription interaction did not complete (status={parsed.status}): {raw_response.text}",
+            )
+        text_contents: Final = tuple(
+            content
+            for step in parsed.steps
+            for content in step.content
+            if content.type == "text" and content.text is not None
+        )
+        response: Final = TranscriptionResponse(text=" ".join(content.text or "" for content in text_contents))
+        response["task"] = "transcribe"
+        words: Final = tuple(
+            word
+            for content in text_contents
+            for annotation in content.annotations
+            if (word := _annotation_to_word(annotation)) is not None
+        )
+        if words:
+            response["words"] = list(words)  # mutable-ok: verbose_json words is a JSON array
+            last_word_end: Final = words[-1].get("end")
+            if last_word_end is not None:
+                response["duration"] = last_word_end
+        if parsed.usage is not None:
+            audio_tokens: Final = sum(
+                by_modality.tokens
+                for by_modality in parsed.usage.input_tokens_by_modality
+                if by_modality.modality == "audio"
+            )
+            response.usage = TranscriptionUsageTokensObject(
+                type="tokens",
+                input_tokens=parsed.usage.total_input_tokens,
+                output_tokens=parsed.usage.total_output_tokens,
+                total_tokens=parsed.usage.total_tokens,
+                input_token_details=TranscriptionUsageInputTokenDetailsObject(
+                    audio_tokens=audio_tokens,
+                    text_tokens=parsed.usage.total_input_tokens - audio_tokens,
+                ),
+            )
+        return response
+
+
+_EMPTY_TRANSCRIPTION_CONFIG: Final[GeminiTranscriptionConfig] = {}
+_WORD_TIMESTAMP_CONFIG: Final[GeminiTranscriptionConfig] = {
+    "mode": {
+        "type": "verbatim",
+        "timestamp_granularities": ("word",),
+        "diarization_mode": "speaker",
+    },
+}
+
+
+def _build_interaction_request(
+    model: str,
+    audio_input: GeminiTranscriptionAudioInput,
+    transcription_config: GeminiTranscriptionConfig,
+) -> GeminiTranscriptionInteractionRequest:
+    if not transcription_config:
+        bare_request: Final[GeminiTranscriptionInteractionRequest] = {
+            "model": model.removeprefix("gemini/"),
+            "input": (audio_input,),
+        }
+        return bare_request
+    configured_request: Final[GeminiTranscriptionInteractionRequest] = {
+        "model": model.removeprefix("gemini/"),
+        "input": (audio_input,),
+        "generation_config": {"transcription_config": transcription_config},
+    }
+    return configured_request
+
+
+def _language_config(language: object) -> GeminiTranscriptionConfig:
+    if not isinstance(language, str) or not language:
+        return _EMPTY_TRANSCRIPTION_CONFIG
+    language_config: Final[GeminiTranscriptionConfig] = {
+        "language_codes": (normalize_transcription_language_to_bcp47(language),),
+    }
+    return language_config
+
+
+def _timestamp_config(timestamp_granularities: object) -> GeminiTranscriptionConfig:
+    if isinstance(timestamp_granularities, list) and "word" in timestamp_granularities:
+        return _WORD_TIMESTAMP_CONFIG
+    return _EMPTY_TRANSCRIPTION_CONFIG
+
+
+def _build_transcription_config(optional_params: Mapping[str, object]) -> GeminiTranscriptionConfig:
+    transcription_config: Final[GeminiTranscriptionConfig] = {
+        **_language_config(optional_params.get("language")),
+        **_timestamp_config(optional_params.get("timestamp_granularities")),
+    }
+    return transcription_config
+
+
+def _annotation_to_word(annotation: GeminiTranscriptionWordAnnotation) -> Mapping[str, str | float] | None:
+    if annotation.type != WORD_INFO_ANNOTATION_TYPE or annotation.text is None:
+        return None
+    entries: Final = (
+        ("word", annotation.text),
+        ("start", _parse_offset_seconds(annotation.start_offset)),
+        ("end", _parse_offset_seconds(annotation.end_offset)),
+        ("speaker", annotation.speaker),
+    )
+    return {key: value for key, value in entries if value is not None}  # mutable-ok: word entries serialize to JSON
+
+
+def _parse_offset_seconds(offset: str | None) -> float | None:
+    if offset is None or not offset.endswith("s"):
+        return None
+    try:
+        return float(offset[:-1])
+    except ValueError:
+        return None

litellm/llms/gemini/cost_calculator.py

@@ -39,28 +39,35 @@ def cost_per_web_search_request(usage: "Usage", model_info: "ModelInfo") -> floa
     ``model_info`` when available, falling back to $0.035 for models not
     yet updated in the pricing JSON.
     """
+    from litellm.litellm_core_utils.llm_cost_calc.utils import (
+        get_web_search_requests_from_usage,
+    )
     from litellm.types.utils import PromptTokensDetailsWrapper
 
     _DEFAULT_COST: Final = 35e-3
     search_costs: Final = model_info.get("search_context_cost_per_query") or {}
     _cost: Final = search_costs.get("search_context_size_medium", _DEFAULT_COST)
 
-    number_of_web_search_requests = 0
-    if (
-        usage is not None
-        and usage.prompt_tokens_details is not None
-        and isinstance(usage.prompt_tokens_details, PromptTokensDetailsWrapper)
-        and hasattr(usage.prompt_tokens_details, "web_search_requests")
-        and usage.prompt_tokens_details.web_search_requests is not None
-    ):
-        number_of_web_search_requests = usage.prompt_tokens_details.web_search_requests
+    requests_from_prompt_details: Final = (
+        usage.prompt_tokens_details.web_search_requests
+        if (
+            usage is not None
+            and usage.prompt_tokens_details is not None
+            and isinstance(usage.prompt_tokens_details, PromptTokensDetailsWrapper)
+            and hasattr(usage.prompt_tokens_details, "web_search_requests")
+            and usage.prompt_tokens_details.web_search_requests is not None
+        )
+        else None
+    )
+    requests_from_server_tool_use: Final = get_web_search_requests_from_usage(usage)
+    number_of_web_search_requests: Final = requests_from_prompt_details or requests_from_server_tool_use or 0
 
-    # per_prompt billing: clamp to 1 (flat fee per grounded API call)
     billing_mode: Final = model_info.get("web_search_billing_unit") or "per_prompt"
-    if number_of_web_search_requests > 0 and billing_mode == "per_prompt":
-        number_of_web_search_requests = 1
+    billable_requests: Final = (
+        1 if (number_of_web_search_requests > 0 and billing_mode == "per_prompt") else number_of_web_search_requests
+    )
 
-    return _cost * number_of_web_search_requests
+    return _cost * billable_requests
 
 
 GOOGLE_MAPS_GROUNDING_DEFAULT_COST_PER_QUERY: Final = 14e-3

litellm/llms/gemini/realtime/transformation.py

@@ -4,7 +4,7 @@ This file contains the transformation logic for the Gemini realtime API.
 
 import json
 from collections import OrderedDict
-from collections.abc import Mapping
+from collections.abc import Mapping, Sequence
 from typing import Any, Final, cast
 
 import litellm
@@ -53,6 +53,7 @@ from litellm.types.llms.vertex_ai import (
 )
 from litellm.types.realtime import (
     ALL_DELTA_TYPES,
+    RealtimeInputAudioTranscriptionUsage,
     RealtimeModalityResponseTransformOutput,
     RealtimeResponseTransformInput,
     RealtimeResponseTypedDict,
@@ -95,6 +96,18 @@ def _gemini_live_speech_config(voice: object) -> Mapping[str, object] | None:
     return VertexGeminiConfig()._map_audio_params({"voice": voice})
 
 
+# Google bills Live transcription at an estimated 25 audio tokens/sec of input and
+# 175 text tokens/min of output (ai.google.dev/gemini-api/docs/pricing).
+GEMINI_LIVE_TRANSCRIBE_AUDIO_TOKENS_PER_SECOND: Final = 25
+GEMINI_LIVE_TRANSCRIBE_OUTPUT_TEXT_TOKENS_PER_MINUTE: Final = 175
+PCM16_INPUT_AUDIO_BYTES_PER_SECOND: Final = 48000
+
+
+def _base64_decoded_byte_count(data: str) -> int:
+    padding: Final = 2 if data.endswith("==") else 1 if data.endswith("=") else 0
+    return max(len(data) * 3 // 4 - padding, 0)
+
+
 class GeminiRealtimeConfig(BaseRealtimeConfig):
     _TOOL_CALL_ID_TO_NAME_MAX = 256  # LRU cap for call_id→name mapping
 
@@ -104,6 +117,7 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
         # Gemini Live sometimes emits usageMetadata in a standalone frame between
         # turns; buffer it here so the next response.done carries the token counts.
         self._pending_usage_metadata: dict | None = None
+        self._unbilled_input_audio_bytes: int = 0
 
     def is_setup_message(self, msg_obj: dict) -> bool:
         return "setup" in msg_obj
@@ -384,17 +398,25 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
         return bool(entry.get("gemini_native_audio") or entry.get("gemini_audio_only_live"))
 
     @staticmethod
-    def _coerce_response_modalities(model: str, modalities: list[Any]) -> list[str]:
-        """Map unsupported TEXT responseModalities to AUDIO for audio-only Live models."""
-        normalized: Final = [
+    def _is_text_only_live_model(model: str) -> bool:
+        return GeminiRealtimeConfig._model_cost_entry(model).get("mode") == "audio_transcription"
+
+    @staticmethod
+    def _default_response_modality(model: str) -> GeminiResponseModalities:
+        return "TEXT" if GeminiRealtimeConfig._is_text_only_live_model(model) else "AUDIO"
+
+    @staticmethod
+    def _coerce_response_modalities(model: str, modalities: Sequence[Any]) -> tuple[str, ...]:
+        """Swap responseModalities a Live model cannot produce: TEXT to AUDIO for
+        audio-only models, AUDIO to TEXT for text-only ones (e.g. transcribe-live)."""
+        normalized: Final = tuple(
             modality.upper() if isinstance(modality, str) else str(modality).upper() for modality in modalities
-        ]
-        if not GeminiRealtimeConfig._is_audio_only_live_model(model):
-            return normalized
-        if "TEXT" not in normalized:
-            return normalized
-        without_text: Final = [modality for modality in normalized if modality != "TEXT"]
-        return without_text if without_text else ["AUDIO"]
+        )
+        if GeminiRealtimeConfig._is_audio_only_live_model(model) and "TEXT" in normalized:
+            return tuple(modality for modality in normalized if modality != "TEXT") or ("AUDIO",)
+        if GeminiRealtimeConfig._is_text_only_live_model(model) and "AUDIO" in normalized:
+            return tuple(modality for modality in normalized if modality != "AUDIO") or ("TEXT",)
+        return normalized
 
     @staticmethod
     def _finalize_gemini_live_setup(model: str, setup: dict[str, Any]) -> dict[str, Any]:
@@ -436,7 +458,7 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
 
         if session_configuration_request is None:
             generation_config: Final = new_overrides.setdefault("generationConfig", {})
-            generation_config.setdefault("responseModalities", ["AUDIO"])
+            generation_config.setdefault("responseModalities", [GeminiRealtimeConfig._default_response_modality(model)])
             new_overrides.setdefault("inputAudioTranscription", {})
             new_overrides["model"] = f"models/{model}"
             verbose_logger.debug("Gemini Realtime: Sending initial setup with tools to backend")
@@ -558,9 +580,10 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
             return self._handle_conversation_item(json_message)
 
         if msg_type == "input_audio_buffer.append":
-            realtime_input_dict["audio"] = HttpxBlobType(
-                mimeType=self.get_audio_mime_type(), data=json_message["audio"]
-            )
+            audio_b64: Final = json_message["audio"]
+            if isinstance(audio_b64, str):
+                self._unbilled_input_audio_bytes += _base64_decoded_byte_count(audio_b64)
+            realtime_input_dict["audio"] = HttpxBlobType(mimeType=self.get_audio_mime_type(), data=audio_b64)
 
             realtime_input_dict = cast(
                 BidiGenerateContentRealtimeInput,
@@ -1151,6 +1174,26 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
             raise ValueError(f"Unknown openai event: {key}, value: {value}")
         return openai_event
 
+    def _consume_input_transcription_usage_estimate(self, model: str) -> RealtimeInputAudioTranscriptionUsage | None:
+        """Gemini Live sends no usageMetadata for transcribe sessions; estimate billing from streamed audio duration."""
+        if self._unbilled_input_audio_bytes <= 0 or not self._is_text_only_live_model(model):
+            return None
+        audio_seconds: Final = self._unbilled_input_audio_bytes / PCM16_INPUT_AUDIO_BYTES_PER_SECOND
+        self._unbilled_input_audio_bytes = 0
+        audio_tokens: Final = round(audio_seconds * GEMINI_LIVE_TRANSCRIBE_AUDIO_TOKENS_PER_SECOND)
+        output_tokens: Final = round(audio_seconds * GEMINI_LIVE_TRANSCRIBE_OUTPUT_TEXT_TOKENS_PER_MINUTE / 60)
+        usage: Final[RealtimeInputAudioTranscriptionUsage] = {
+            "type": "tokens",
+            "input_tokens": audio_tokens,
+            "output_tokens": output_tokens,
+            "total_tokens": audio_tokens + output_tokens,
+            "input_token_details": {"text_tokens": 0, "audio_tokens": audio_tokens},
+        }
+        return usage
+
+    def unbilled_usage_on_session_close(self, model: str) -> RealtimeInputAudioTranscriptionUsage | None:
+        return self._consume_input_transcription_usage_estimate(model)
+
     def transform_realtime_response(
         self,
         message: str | bytes,
@@ -1190,6 +1233,7 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
         if isinstance(server_content, dict):
             input_tx: Final = server_content.get("inputTranscription")
             if isinstance(input_tx, dict) and input_tx.get("text"):
+                transcription_usage: Final = self._consume_input_transcription_usage_estimate(model)
                 returned_message.append(
                     cast(
                         OpenAIRealtimeEvents,
@@ -1199,6 +1243,7 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
                             "transcript": input_tx["text"],
                             "item_id": f"item_{uuid.uuid4()}",
                             "content_index": 0,
+                            **({} if transcription_usage is None else {"usage": transcription_usage}),
                         },
                     )
                 )
@@ -1235,6 +1280,12 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
                     )
                 )
 
+            # Transcription-only models emit generationComplete with no prior
+            # modelTurn delta; there is no started OpenAI response to close, so
+            # drop it and let siblings (turnComplete, usageMetadata) process.
+            if current_delta_type is None and "modelTurn" not in server_content:
+                server_content.pop("generationComplete", None)
+
             # Mark transcription-only serverContent as handled so the main loop
             # skips it; sibling keys like toolCall are still processed below.
             _model_content_keys: Final = {
@@ -1583,7 +1634,9 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
         ```
         """
 
-        response_modalities: Final[list[GeminiResponseModalities]] = ["AUDIO"]
+        response_modalities: Final[list[GeminiResponseModalities]] = [
+            GeminiRealtimeConfig._default_response_modality(model)
+        ]
         output_audio_transcription: Final = False
         # if "audio" in model: ## UNCOMMENT THIS WHEN AUDIO IS SUPPORTED
         #     output_audio_transcription = True

litellm/main.py

@@ -1219,6 +1219,7 @@ def _register_custom_pricing_for_request(
             shared_key: CustomPricingLiteLLMParams.strip_custom_pricing_fields(entry),
         },
         persist_across_reloads=False,
+        warning_display_name=shared_key,
     )
 
 

litellm/proxy/_experimental/mcp_server/mcp_server_manager.py

@@ -5256,7 +5256,9 @@ class MCPServerManager:
             proxy_logging_obj: Optional ProxyLogging object for hook integration
             host_progress_callback: Optional callback for progress updates
             hook_extra_headers: Optional headers injected by pre_mcp_call guardrail
-                hooks. Merged last (highest priority) into outbound request headers.
+                hooks. Merged last into outbound request headers, except a hook
+                Authorization header is dropped when an upstream credential already
+                occupies the Authorization slot.
 
         Returns:
             CallToolResult from the MCP server
@@ -5347,27 +5349,26 @@ class MCPServerManager:
         if hook_extra_headers:
             if extra_headers is None:
                 extra_headers = {}
-            if "Authorization" in hook_extra_headers:
-                if "Authorization" in extra_headers:
-                    verbose_logger.warning(
-                        "MCPServerManager: hook_extra_headers 'Authorization' will overwrite "
-                        "the existing Authorization header from static_headers. "
-                        "The hook JWT will take precedence."
-                    )
-                elif server_auth_header is not None:
-                    # server_auth_header is passed separately to _create_mcp_client as
-                    # auth_value.  Both will reach the upstream server — warn so admins
-                    # know two Authorization credentials are being sent.
-                    verbose_logger.warning(
-                        "MCPServerManager: hook_extra_headers injects 'Authorization' while "
-                        "server '%s' already has a configured authentication_token. "
-                        "Both credentials will be sent; the hook header is in extra_headers "
-                        "and the server token is in auth_value — the upstream server decides "
-                        "which one wins.  Consider unsetting authentication_token if you want "
-                        "the hook JWT to be the sole credential.",
-                        mcp_server.server_name or mcp_server.name,
-                    )
-            extra_headers.update(hook_extra_headers)
+            hook_has_authorization: Final = any(k.lower() == "authorization" for k in hook_extra_headers)
+            existing_has_authorization: Final = any(k.lower() == "authorization" for k in extra_headers)
+            server_auth_occupies_authorization: Final = (
+                any(k.lower() == "authorization" for k in server_auth_header)
+                if isinstance(server_auth_header, dict)
+                else server_auth_header is not None and mcp_server.auth_type != MCPAuth.api_key
+            )
+            if hook_has_authorization and (existing_has_authorization or server_auth_occupies_authorization):
+                # Mirror the tools/list signer guard: an upstream credential (user OAuth,
+                # static header, or configured authentication_token) already occupies the
+                # Authorization slot, so the hook must not replace it.
+                verbose_logger.warning(
+                    "MCPServerManager: dropping hook-injected 'Authorization' header for "
+                    "server '%s' because an upstream credential already occupies the "
+                    "Authorization slot; the existing credential is kept.",
+                    mcp_server.server_name or mcp_server.name,
+                )
+                extra_headers.update({k: v for k, v in hook_extra_headers.items() if k.lower() != "authorization"})
+            else:
+                extra_headers.update(hook_extra_headers)
 
         # Reset to None if no headers were actually added
         if extra_headers is not None and len(extra_headers) == 0:

litellm/proxy/_experimental/mcp_server/outbound_credentials/bridge_credentials.py

@@ -239,5 +239,6 @@ def resolve_bridge_envelope(
     if opened.identity.server_id != expected_server_id:
         return BridgeEnvelopeInvalid()
     grant: Final = opened.grant
-    upstream_authorization: Final = f"{grant.token_type} {grant.access_token.get_secret_value()}"
+    authorization_scheme: Final = "Bearer" if grant.token_type.lower() == "bearer" else grant.token_type
+    upstream_authorization: Final = f"{authorization_scheme} {grant.access_token.get_secret_value()}"
     return BridgeEnvelopeAdmitted(identity=opened.identity, upstream_authorization=SecretStr(upstream_authorization))

litellm/proxy/_experimental/mcp_server/server.py

@@ -246,11 +246,12 @@ def _mcp_meta_trace_carrier(req_ctx: object) -> dict[str, str] | None:
     """The W3C trace context (``traceparent``/``tracestate``) the MCP client
     propagated in the request's ``params._meta`` (SEP-414), or ``None``.
 
-    When present, per the OTel MCP semconv the MCP span parents to this propagated
-    context rather than to the HTTP transport (which is recorded as a link instead).
-    When absent, the span nests under the transport span of the request carrying
-    this specific message, so a streamable-HTTP session that multiplexes many
-    messages still does not glue every message under the session's first request;
+    When present, the MCP span records this propagated context as a span *link*,
+    never the parent — a remote parent would root the span in a trace whose root
+    never reaches the gateway's tracing backend. The span itself nests under the
+    transport span of the request carrying this specific message, so a
+    streamable-HTTP session that multiplexes many messages still does not glue
+    every message under the session's first request;
     see ``resolve_mcp_span_context``. The client's W3C Baggage is
     deliberately excluded: it is caller-controlled, and the otel baggage processor
     stamps allowlisted baggage keys (``litellm.team.id``, ``litellm.metadata.*``,

litellm/proxy/_lazy_openapi_snapshot.py

@@ -3,18 +3,27 @@ Per-feature OpenAPI snapshot for lazy-loaded routers.
 
 The committed JSON is generated by `python -m litellm.proxy._lazy_openapi_snapshot`
 and consumed at runtime so /openapi.json can show full route info for unloaded
-features without importing them. No CI job regenerates this file; drift surfaces
-only indirectly through check-ui-api-types.yml, which rebuilds schema.d.ts from
-app.openapi() with the committed snapshot injected. After changing any lazily
-loaded route or this generator, rerun the module and commit the JSON, then run
-`npm run gen:api` in ui/litellm-dashboard and commit schema.d.ts.
+features without importing them. check-ui-api-types.yml (mirrored locally by
+`make check`) regenerates this file and fails when the committed copy differs,
+then rebuilds schema.d.ts from app.openapi() with the snapshot injected. After
+changing any lazily loaded route or this generator, rerun the module and commit
+the JSON, then run `npm run gen:api` in ui/litellm-dashboard and commit schema.d.ts.
 """
 
 import json
 import re
 import sys
+from collections.abc import Callable, Mapping
+from dataclasses import dataclass
 from pathlib import Path
-from typing import Final
+from typing import TYPE_CHECKING, Final
+
+from typing_extensions import ReadOnly, TypedDict
+
+if TYPE_CHECKING:
+    from fastapi import FastAPI
+
+    from litellm.proxy._lazy_features import LazyFeature
 
 SNAPSHOT_FILE: Final = Path(__file__).parent / "_lazy_openapi_snapshot.json"
 HTTP_METHOD_SUFFIXES: Final = {
@@ -83,51 +92,84 @@ def _normalize_operation_ids(paths: dict[str, dict]) -> None:
                     break
 
 
-def generate_snapshot() -> dict[str, dict]:
+class SnapshotFragment(TypedDict):
+    paths: ReadOnly[Mapping[str, Mapping[str, object]]]
+    components: ReadOnly[Mapping[str, Mapping[str, object]]]
+
+
+@dataclass(frozen=True, slots=True)
+class SnapshotResult:
+    fragments: Mapping[str, SnapshotFragment]
+    skipped: tuple[str, ...]
+
+
+def _register_feature(app: "FastAPI", feat: "LazyFeature") -> str | None:
     import importlib
 
+    try:
+        feat.register_fn(app, importlib.import_module(feat.module_path))
+    except Exception as exc:
+        sys.stderr.write(f"warning: skip {feat.name}: {exc}\n")
+        return feat.name
+    return None
+
+
+def _feature_fragment(app: "FastAPI", feat: "LazyFeature", used_operation_ids: set[str]) -> SnapshotFragment | None:
     from fastapi.openapi.utils import get_openapi
 
+    from litellm.proxy.proxy_server import ensure_unique_openapi_operation_ids
+
+    feat_routes: Final = [r for r in app.routes if feat.matches(getattr(r, "path", ""))]
+    if not feat_routes:
+        return None
+    _stabilize_multi_method_route_ids(feat_routes)
+    full: Final = get_openapi(title=app.title, version=app.version, routes=feat_routes)
+    paths: Final = full.get("paths", {})
+    _normalize_operation_ids(paths)
+    # Group all of a feature's routes under one tag.
+    for path_ops in paths.values():
+        for method, op in path_ops.items():
+            if isinstance(op, dict):
+                operation_id = op.get("operationId")
+                if isinstance(operation_id, str):
+                    for suffix in HTTP_METHOD_SUFFIXES:
+                        if operation_id.endswith(f"_{suffix}"):
+                            op["operationId"] = operation_id[: -len(suffix)] + method
+                            break
+                op["tags"] = [feat.name]
+    unique: Final = ensure_unique_openapi_operation_ids(full, used_operation_ids)
+    return {
+        "paths": paths,
+        "components": {"schemas": unique.get("components", {}).get("schemas", {})},
+    }
+
+
+def generate_snapshot() -> SnapshotResult:
     from litellm.proxy._lazy_features import LAZY_FEATURES
-    from litellm.proxy.proxy_server import app, ensure_unique_openapi_operation_ids
+    from litellm.proxy.proxy_server import app
 
-    for feat in LAZY_FEATURES:
-        try:
-            module = importlib.import_module(feat.module_path)
-            feat.register_fn(app, module)
-        except Exception as exc:
-            sys.stderr.write(f"warning: skip {feat.name}: {exc}\n")
-
-    fragments: Final[dict[str, dict]] = {}
+    skipped: Final = tuple(name for feat in LAZY_FEATURES if (name := _register_feature(app, feat)) is not None)
     used_operation_ids: Final[set[str]] = set()
-    for feat in LAZY_FEATURES:
-        feat_routes = [r for r in app.routes if feat.matches(getattr(r, "path", ""))]
-        if not feat_routes:
-            continue
-        _stabilize_multi_method_route_ids(feat_routes)
-        full = get_openapi(title=app.title, version=app.version, routes=feat_routes)
-        paths = full.get("paths", {})
-        _normalize_operation_ids(paths)
-        # Group all of a feature's routes under one tag.
-        for path_ops in full.get("paths", {}).values():
-            for method, op in path_ops.items():
-                if isinstance(op, dict):
-                    operation_id = op.get("operationId")
-                    if isinstance(operation_id, str):
-                        for suffix in HTTP_METHOD_SUFFIXES:
-                            if operation_id.endswith(f"_{suffix}"):
-                                op["operationId"] = operation_id[: -len(suffix)] + method
-                                break
-                    op["tags"] = [feat.name]
-        full = ensure_unique_openapi_operation_ids(full, used_operation_ids)
-        fragments[feat.name] = {
-            "paths": paths,
-            "components": {"schemas": full.get("components", {}).get("schemas", {})},
-        }
-    return fragments
+    fragments: Final = {
+        feat.name: fragment
+        for feat in LAZY_FEATURES
+        if (fragment := _feature_fragment(app, feat, used_operation_ids)) is not None
+    }
+    return SnapshotResult(fragments=fragments, skipped=skipped)
+
+
+def main(snapshot_file: Path = SNAPSHOT_FILE, generate: Callable[[], SnapshotResult] = generate_snapshot) -> int:
+    result: Final = generate()
+    if result.skipped:
+        sys.stderr.write(
+            f"error: {len(result.skipped)} feature(s) failed to import, so their fragments would vanish from the "
+            f"snapshot: {', '.join(result.skipped)}\n"
+        )
+        return 1
+    snapshot_file.write_text(json.dumps(result.fragments, indent=2, sort_keys=True) + "\n")
+    sys.stdout.write(f"wrote {len(result.fragments)} feature fragments to {snapshot_file}\n")
+    return 0
 
 
 if __name__ == "__main__":
-    fragments: Final = generate_snapshot()
-    SNAPSHOT_FILE.write_text(json.dumps(fragments, indent=2, sort_keys=True) + "\n")
-    sys.stdout.write(f"wrote {len(fragments)} feature fragments to {SNAPSHOT_FILE}\n")
+    sys.exit(main())

litellm/proxy/_types.py

@@ -2510,6 +2510,18 @@ class ConfigGeneralSettings(LiteLLMPydanticObjectBase):
             "are skipped for on-demand GET /health as well as the background health loop."
         ),
     )
+    background_health_check_model_groups: tuple[str, ...] | None = Field(
+        None,
+        description=(
+            "Opt-in allowlist of model group names for background health checks and "
+            "health-check routing. When set, the background loop probes only deployments "
+            "whose model_name is listed, and enable_health_check_routing filters unhealthy "
+            "deployments only within the listed groups; every other group, including newly "
+            "added deployments, is skipped and keeps its configured routing strategy. "
+            "When unset, all deployments participate (opt out per deployment via "
+            "model_info.disable_background_health_check)."
+        ),
+    )
     model_list_healthy_only: bool | None = Field(
         None,
         description=(

litellm/proxy/auth/user_api_key_auth.py

@@ -1769,7 +1769,12 @@ async def _user_api_key_auth_builder(
 
             return valid_token
 
-        if valid_token is not None and isinstance(valid_token, UserAPIKeyAuth) and valid_token.team_id is not None:
+        if (
+            valid_token is not None
+            and isinstance(valid_token, UserAPIKeyAuth)
+            and valid_token.team_id is not None
+            and valid_token.team_id != UI_TEAM_ID
+        ):
             ## UPDATE TEAM VALUES BASED ON CACHED TEAM OBJECT - allows `/team/update` values to work for cached token
             try:
                 team_obj: Final[LiteLLM_TeamTableCachedObj] = await get_team_object(
@@ -2149,6 +2154,8 @@ async def _user_api_key_auth_builder(
             # Check 6: Additional Common Checks across jwt + key auth
             if valid_token.team_id is not None:
                 try:
+                    if valid_token.team_id == UI_TEAM_ID:
+                        raise TeamNotFoundError(team_id=UI_TEAM_ID)
                     with tracer.trace("litellm.proxy.auth.get_team_object"):
                         _team_obj = await get_team_object(
                             team_id=valid_token.team_id,
@@ -2443,7 +2450,7 @@ async def _run_centralized_common_checks(
         )
 
     fetch_coros: Final = []
-    if user_api_key_auth_obj.team_id is not None:
+    if user_api_key_auth_obj.team_id is not None and user_api_key_auth_obj.team_id != UI_TEAM_ID:
         fetch_coros.append(
             _safe_fetch(
                 "team",
@@ -2567,7 +2574,9 @@ async def _run_centralized_common_checks(
         else:
             raise team_result
     else:
-        team_object = team_result
+        team_object = (
+            _team_obj_from_token(user_api_key_auth_obj) if user_api_key_auth_obj.team_id == UI_TEAM_ID else team_result
+        )
 
     user_object: LiteLLM_UserTable | None = None if isinstance(user_result, BaseException) else user_result
     project_object: Final[LiteLLM_ProjectTableCachedObj | None] = (

litellm/proxy/common_utils/reset_budget_job.py

@@ -1,5 +1,6 @@
 import asyncio
 import json
+import math
 import time
 from collections.abc import Awaitable, Callable, Iterable, Mapping, Sequence
 from dataclasses import dataclass
@@ -45,6 +46,7 @@ from litellm.repositories.table_repositories import (
 )
 from litellm.repositories.team_repository import TeamRepository
 from litellm.repositories.unit_of_work import (
+    LinkedSpendResetWrites,
     budget_cascade_unit_of_work,
     spend_reset_unit_of_work,
 )
@@ -59,7 +61,15 @@ _LINKED_KEYS_WHERE: Final[Mapping[str, object]] = MappingProxyType({"budget_dura
 _SPENT_ROWS_WHERE: Final[Mapping[str, object]] = MappingProxyType({"spend": {"gt": 0}})
 
 
-class _TeamMembershipRow(Protocol):
+class _BudgetLinkedRow(Protocol):
+    @property
+    def spend(self) -> float | None: ...
+
+    @property
+    def budget_id(self) -> str | None: ...
+
+
+class _TeamMembershipRow(_BudgetLinkedRow, Protocol):
     @property
     def user_id(self) -> str: ...
 
@@ -67,26 +77,48 @@ class _TeamMembershipRow(Protocol):
     def team_id(self) -> str: ...
 
 
-class _KeyRow(Protocol):
+class _KeyRow(_BudgetLinkedRow, Protocol):
     @property
     def token(self) -> str: ...
 
 
-class _OrgRow(Protocol):
+class _OrgRow(_BudgetLinkedRow, Protocol):
     @property
     def organization_id(self) -> str: ...
 
 
-class _TagRow(Protocol):
+class _TagRow(_BudgetLinkedRow, Protocol):
     @property
     def tag_name(self) -> str: ...
 
 
-class _EndUserRow(Protocol):
+class _EndUserRow(_BudgetLinkedRow, Protocol):
     @property
     def user_id(self) -> str: ...
 
 
+def _rollover_enabled() -> bool:
+    return litellm.budget_rollover is True
+
+
+def _rollover_cap(max_budget: float | None) -> float | None:
+    if max_budget is None or not math.isfinite(max_budget):
+        return None
+    return max_budget
+
+
+def _carried_spend(spend: float | None, cap: float | None) -> float:
+    if cap is None:
+        return 0.0
+    return max(0.0, (spend or 0.0) - cap)
+
+
+def _row_carried_spend(row: _BudgetLinkedRow, caps: Mapping[str, float]) -> float:
+    if not caps:
+        return 0.0
+    return _carried_spend(row.spend, caps.get(row.budget_id) if row.budget_id is not None else None)
+
+
 def _team_membership_counter_key(row: _TeamMembershipRow) -> str:
     return f"spend:team_member:{row.user_id}:{row.team_id}"
 
@@ -129,6 +161,59 @@ def _budget_link_where(
     return {"budget_id": {"in": list(budget_ids)}, **extra}
 
 
+def _queue_budget_linked_resets(
+    writes: LinkedSpendResetWrites,
+    cascade: "_BudgetCascade",
+    extra: Mapping[str, object] = MappingProxyType({}),
+) -> None:
+    """Reset one linked table's spend for every expiring tier: tiers with a
+    rollover cap keep spend beyond the cap (decrement preserves writes racing
+    the reset), everything else is zeroed as before. Zero the under-cap rows
+    BEFORE decrementing the over-cap ones: the statements run sequentially in
+    one transaction, so the reverse order lets the zero re-match a row the
+    decrement just moved into the (0, cap] range and erase its carried spend."""
+    for budget_id, cap in cascade.rollover_caps.items():
+        writes.queue_spend_zero(
+            where={"budget_id": budget_id, **extra, "spend": {"gt": 0, "lte": cap}}
+        )  # mutable-ok: prisma where filter must be a dict
+        writes.queue_spend_decrement(
+            where={"budget_id": budget_id, **extra, "spend": {"gt": cap}}, amount=cap
+        )  # mutable-ok: prisma where filter must be a dict
+    plain_ids: Final = tuple(bid for bid in cascade.budget_ids if bid not in cascade.rollover_caps)
+    if plain_ids:
+        writes.queue_spend_zero(where=_budget_link_where(plain_ids, extra))
+
+
+def _queue_enduser_resets(writes: LinkedSpendResetWrites, cascade: "_BudgetCascade") -> None:
+    """End users are matched by id rather than budget link: rows with no
+    budget_id ride the default budget tier (litellm.max_end_user_budget_id).
+    Zero-before-decrement ordering matters here too (see
+    _queue_budget_linked_resets)."""
+    if not cascade.rollover_caps:
+        if cascade.endusers:
+            writes.queue_spend_zero(
+                where={"user_id": {"in": [row.user_id for row in cascade.endusers]}}
+            )  # mutable-ok: prisma where filter must be a dict
+        return
+    tiered: Final = tuple((row.budget_id or litellm.max_end_user_budget_id, row.user_id) for row in cascade.endusers)
+    for budget_id, cap in cascade.rollover_caps.items():
+        if not (
+            user_ids := [uid for bid, uid in tiered if bid == budget_id]
+        ):  # mutable-ok: prisma "in" filter takes a list
+            continue
+        writes.queue_spend_zero(
+            where={"user_id": {"in": user_ids}, "spend": {"lte": cap}}
+        )  # mutable-ok: prisma where filter must be a dict
+        writes.queue_spend_decrement(
+            where={"user_id": {"in": user_ids}, "spend": {"gt": cap}}, amount=cap
+        )  # mutable-ok: prisma where filter must be a dict
+    plain: Final = [
+        uid for bid, uid in tiered if bid is None or bid not in cascade.rollover_caps
+    ]  # mutable-ok: prisma "in" filter takes a list
+    if plain:
+        writes.queue_spend_zero(where={"user_id": {"in": plain}})  # mutable-ok: prisma where filter must be a dict
+
+
 @dataclass(frozen=True, slots=True)
 class _BudgetCascade:
     """Everything one budget-tier reset touches, resolved before any write."""
@@ -137,8 +222,9 @@ class _BudgetCascade:
     budget_ids: tuple[str, ...] = ()
     budget_resets: tuple[tuple[str, datetime], ...] = ()
     endusers: tuple[_EndUserRow, ...] = ()
-    counter_keys: tuple[str, ...] = ()
+    counter_resets: tuple[tuple[str, float], ...] = ()
     cache_keys: tuple[str, ...] = ()
+    rollover_caps: Mapping[str, float] = MappingProxyType({})
 
 
 @dataclass(frozen=True, slots=True)
@@ -404,8 +490,10 @@ class ResetBudgetJob:
         )
 
     @staticmethod
-    async def _invalidate_spend_counter(counter_key: str) -> None:
-        """Zero a spend counter so a DB-row reset takes effect immediately.
+    async def _invalidate_spend_counter(counter_key: str, new_spend: float = 0.0) -> None:
+        """Overwrite a spend counter with the post-reset value (0, or the carried
+        overage when budget rollover is enabled) so a DB-row reset takes effect
+        immediately.
 
         Call AFTER the DB write commits. Clearing Redis before the DB
         commit opens a window where get_current_spend reads 0 from Redis
@@ -414,10 +502,10 @@ class ResetBudgetJob:
         try:
             from litellm.proxy.proxy_server import spend_counter_cache
 
-            spend_counter_cache.in_memory_cache.set_cache(key=counter_key, value=0.0, ttl=60)
+            spend_counter_cache.in_memory_cache.set_cache(key=counter_key, value=new_spend, ttl=60)
             if spend_counter_cache.redis_cache is not None:
                 try:
-                    await spend_counter_cache.redis_cache.async_set_cache(key=counter_key, value=0.0, ttl=60)
+                    await spend_counter_cache.redis_cache.async_set_cache(key=counter_key, value=new_spend, ttl=60)
                 except Exception as redis_err:
                     verbose_proxy_logger.warning(
                         "Failed to reset spend counter %s in Redis: %s. "
@@ -522,6 +610,15 @@ class ResetBudgetJob:
             where=_budget_link_where(budget_ids, _SPENT_ROWS_WHERE),
             log_subject="tags",
         )
+        rollover_caps: Final[Mapping[str, float]] = MappingProxyType(
+            {  # mutable-ok: MappingProxyType wraps a one-shot dict comprehension
+                b.budget_id: cap
+                for b in budgets_to_reset
+                if b.budget_id is not None and (cap := _rollover_cap(b.max_budget)) is not None
+            }
+            if _rollover_enabled()
+            else {}  # mutable-ok: empty sentinel immediately frozen by MappingProxyType
+        )
         return _BudgetCascade(
             budgets=tuple(budgets_to_reset),
             budget_ids=budget_ids,
@@ -534,12 +631,16 @@ class ResetBudgetJob:
                 if b.budget_id is not None and b.budget_duration is not None
             ),
             endusers=await self._collect_endusers_to_reset(budget_ids),
-            counter_keys=(
-                *(_team_membership_counter_key(row) for row in team_memberships),
-                *(_key_counter_key(row) for row in keys),
-                *(_org_counter_key(row) for row in orgs),
-                *(_tag_counter_key(row) for row in tags),
+            counter_resets=(
+                *(
+                    (_team_membership_counter_key(row), _row_carried_spend(row, rollover_caps))
+                    for row in team_memberships
+                ),
+                *((_key_counter_key(row), _row_carried_spend(row, rollover_caps)) for row in keys),
+                *((_org_counter_key(row), _row_carried_spend(row, rollover_caps)) for row in orgs),
+                *((_tag_counter_key(row), _row_carried_spend(row, rollover_caps)) for row in tags),
             ),
+            rollover_caps=rollover_caps,
             cache_keys=(
                 *(key for row in team_memberships for key in _team_membership_cache_keys(row)),
                 *(key for row in keys for key in _key_cache_keys(row)),
@@ -565,20 +666,18 @@ class ResetBudgetJob:
         )
 
     async def _commit_budget_cascade_once(self, cascade: _BudgetCascade) -> None:
-        enduser_ids: Final = tuple(row.user_id for row in cascade.endusers)
         async with budget_cascade_unit_of_work(self.prisma_client.db.batch_) as uow:
-            uow.team_memberships.queue_spend_zero(where=_budget_link_where(cascade.budget_ids))
-            uow.keys.queue_spend_zero(where=_budget_link_where(cascade.budget_ids, _LINKED_KEYS_WHERE))
-            uow.organizations.queue_spend_zero(where=_budget_link_where(cascade.budget_ids, _SPENT_ROWS_WHERE))
-            uow.tags.queue_spend_zero(where=_budget_link_where(cascade.budget_ids, _SPENT_ROWS_WHERE))
-            if enduser_ids:
-                uow.endusers.queue_spend_zero(where={"user_id": {"in": list(enduser_ids)}})
+            _queue_budget_linked_resets(uow.team_memberships, cascade)
+            _queue_budget_linked_resets(uow.keys, cascade, extra=_LINKED_KEYS_WHERE)
+            _queue_budget_linked_resets(uow.organizations, cascade, extra=_SPENT_ROWS_WHERE)
+            _queue_budget_linked_resets(uow.tags, cascade, extra=_SPENT_ROWS_WHERE)
+            _queue_enduser_resets(uow.endusers, cascade)
             for budget_id, budget_reset_at in cascade.budget_resets:
                 uow.budgets.queue_window_advance(budget_id=budget_id, budget_reset_at=budget_reset_at)
 
     async def _invalidate_budget_cascade_caches(self, cascade: _BudgetCascade) -> None:
-        for counter_key in cascade.counter_keys:
-            await self._invalidate_spend_counter(counter_key)
+        for counter_key, new_spend in cascade.counter_resets:
+            await self._invalidate_spend_counter(counter_key, new_spend=new_spend)
         for cache_key in cascade.cache_keys:
             await self._invalidate_user_api_key_cache_entry(cache_key)
 
@@ -708,7 +807,11 @@ class ResetBudgetJob:
             for k in updated_keys:
                 if k.token is None:
                     continue
-                uow.keys.queue_spend_reset(token=k.token, budget_reset_at=k.budget_reset_at)
+                uow.keys.queue_spend_reset(
+                    token=k.token,
+                    budget_reset_at=k.budget_reset_at,
+                    spend_decrement=k.max_budget if (k.spend or 0.0) > 0.0 else None,
+                )
 
     async def _write_user_reset_updates(self, updated_users: list[LiteLLM_UserTable]) -> None:
         """
@@ -726,7 +829,11 @@ class ResetBudgetJob:
     async def _write_user_reset_updates_once(self, updated_users: list[LiteLLM_UserTable]) -> None:
         async with spend_reset_unit_of_work(self.prisma_client.db.batch_) as uow:
             for u in updated_users:
-                uow.users.queue_spend_reset(user_id=u.user_id, budget_reset_at=u.budget_reset_at)
+                uow.users.queue_spend_reset(
+                    user_id=u.user_id,
+                    budget_reset_at=u.budget_reset_at,
+                    spend_decrement=u.max_budget if (u.spend or 0.0) > 0.0 else None,
+                )
 
     async def _write_team_reset_updates(self, updated_teams: list[LiteLLM_TeamTable]) -> None:
         """
@@ -744,7 +851,11 @@ class ResetBudgetJob:
     async def _write_team_reset_updates_once(self, updated_teams: list[LiteLLM_TeamTable]) -> None:
         async with spend_reset_unit_of_work(self.prisma_client.db.batch_) as uow:
             for t in updated_teams:
-                uow.teams.queue_spend_reset(team_id=t.team_id, budget_reset_at=t.budget_reset_at)
+                uow.teams.queue_spend_reset(
+                    team_id=t.team_id,
+                    budget_reset_at=t.budget_reset_at,
+                    spend_decrement=t.max_budget if (t.spend or 0.0) > 0.0 else None,
+                )
 
     def _emit_phase_failure(
         self,
@@ -820,7 +931,7 @@ class ResetBudgetJob:
                     for k in updated_keys:
                         token = getattr(k, "token", None)
                         if token:
-                            await self._invalidate_spend_counter(f"spend:key:{token}")
+                            await self._invalidate_spend_counter(f"spend:key:{token}", new_spend=k.spend or 0.0)
 
             end_time = time.time()
             outcome: Final = _ChunkOutcome(
@@ -925,7 +1036,7 @@ class ResetBudgetJob:
                     for u in updated_users:
                         user_id = getattr(u, "user_id", None)
                         if user_id:
-                            await self._invalidate_spend_counter(f"spend:user:{user_id}")
+                            await self._invalidate_spend_counter(f"spend:user:{user_id}", new_spend=u.spend or 0.0)
                         if user_id == LITELLM_PROXY_BUDGET_NAME:
                             await self._invalidate_global_proxy_spend_cache()
 
@@ -1034,7 +1145,7 @@ class ResetBudgetJob:
                     for t in updated_teams:
                         team_id = getattr(t, "team_id", None)
                         if team_id:
-                            await self._invalidate_spend_counter(f"spend:team:{team_id}")
+                            await self._invalidate_spend_counter(f"spend:team:{team_id}", new_spend=t.spend or 0.0)
 
             end_time = time.time()
             outcome: Final = _ChunkOutcome(
@@ -1107,10 +1218,11 @@ class ResetBudgetJob:
         reset_at: Final = datetime.fromisoformat(reset_at_str.replace("Z", "+00:00")).replace(tzinfo=None)
         if reset_at > now:
             return False
-        spend_counter_cache.in_memory_cache.set_cache(key=counter_key, value=0.0)
+        new_value: Final = await ResetBudgetJob._window_carried_spend(window, counter_key, spend_counter_cache)
+        spend_counter_cache.in_memory_cache.set_cache(key=counter_key, value=new_value)
         if spend_counter_cache.redis_cache is not None:
             try:
-                await spend_counter_cache.redis_cache.async_set_cache(key=counter_key, value=0.0)
+                await spend_counter_cache.redis_cache.async_set_cache(key=counter_key, value=new_value)
             except Exception as redis_err:
                 verbose_proxy_logger.warning("Failed to reset Redis counter %s: %s", counter_key, redis_err)
         window["reset_at"] = compute_budget_reset_at(
@@ -1118,6 +1230,27 @@ class ResetBudgetJob:
         ).isoformat()
         return True
 
+    @staticmethod
+    async def _window_carried_spend(
+        window: Mapping[str, object], counter_key: str, spend_counter_cache: DualCache
+    ) -> float:
+        """Per-window spend lives only in the counter, so the carried overage is
+        read from it before the reset overwrites it."""
+        if not _rollover_enabled():
+            return 0.0
+        window_max: Final = window.get("max_budget")
+        cap: Final = _rollover_cap(window_max) if isinstance(window_max, (int, float)) else None
+        if cap is None:
+            return 0.0
+        try:
+            current: Final = await spend_counter_cache.async_get_cache(key=counter_key)
+        except Exception as e:  # noqa: BLE001  # an unreadable counter falls back to a plain zero reset
+            verbose_proxy_logger.warning("Failed to read spend counter %s for rollover: %s", counter_key, e)
+            return 0.0
+        if not isinstance(current, (int, float)):
+            return 0.0
+        return _carried_spend(float(current), cap)
+
     async def reset_budget_windows(self) -> None:
         """
         For keys and teams with budget_limits, reset any individual windows where
@@ -1222,7 +1355,7 @@ class ResetBudgetJob:
         still holds the pre-reset value, admitting requests past the cap.
         """
         try:
-            item.spend = 0.0
+            item.spend = _carried_spend(item.spend, _rollover_cap(item.max_budget)) if _rollover_enabled() else 0.0
             if hasattr(item, "budget_duration") and item.budget_duration is not None:
                 item.budget_reset_at = compute_budget_reset_at(
                     budget_duration=item.budget_duration, settings=reset_settings

litellm/proxy/db/prisma_client.py

@@ -887,6 +887,22 @@ class PrismaManager:
             return
         ProxyExtrasDBManager.apply_replica_identity_full_if_requested()
 
+    @staticmethod
+    def _raise_if_partitioned_spend_logs() -> None:
+        """`prisma db push` rewrites a doc-partitioned LiteLLM_SpendLogs
+        primary key back to ("request_id"), which Postgres rejects. Fail fast
+        with guidance instead of retrying into that raw error. No-op when
+        litellm-proxy-extras is absent."""
+        try:
+            from litellm_proxy_extras.utils import (
+                PARTITIONED_SPEND_LOGS_PUSH_ERROR,
+                ProxyExtrasDBManager,
+            )
+        except ImportError:
+            return
+        if ProxyExtrasDBManager.spend_logs_is_partitioned():
+            raise RuntimeError(PARTITIONED_SPEND_LOGS_PUSH_ERROR)
+
     @staticmethod
     def setup_database(use_migrate: bool = False, use_v2_resolver: bool = False) -> bool:
         """
@@ -921,6 +937,7 @@ class PrismaManager:
                         use_v2_resolver=use_v2_resolver,
                     )
                 else:
+                    PrismaManager._raise_if_partitioned_spend_logs()
                     # Use prisma db push with increased timeout
                     subprocess.run(
                         [

litellm/proxy/health_check.py

@@ -7,8 +7,11 @@ import sys
 import threading
 import time
 from collections.abc import Mapping, Sequence
+from collections.abc import Set as AbstractSet
 from types import MappingProxyType
-from typing import TYPE_CHECKING, Final
+from typing import TYPE_CHECKING, Final, TypeVar
+
+from pydantic import TypeAdapter, ValidationError
 
 import litellm
 
@@ -16,6 +19,7 @@ if TYPE_CHECKING:
     from litellm.router import Router
 
 logger: Final = logging.getLogger(__name__)
+_DeploymentT: Final = TypeVar("_DeploymentT", bound=Mapping[str, object])
 from litellm.constants import (
     BACKGROUND_HEALTH_CHECK_MAX_TOKENS,
     BACKGROUND_HEALTH_CHECK_MAX_TOKENS_REASONING,
@@ -167,6 +171,38 @@ def health_check_filter_kwargs_from_general_settings(
     }
 
 
+def parse_background_health_check_model_groups(
+    general_settings: Mapping[str, object] | None,
+) -> frozenset[str] | None:
+    """
+    Read ``general_settings.background_health_check_model_groups``.
+
+    ``None`` means the allowlist is unset and every deployment participates
+    (legacy behavior). A list scopes background health checks and health-check
+    routing to deployments whose ``model_name`` is listed. A malformed value
+    raises so the proxy fails at startup instead of silently probing everything.
+    """
+    raw: Final = (general_settings or {}).get("background_health_check_model_groups")
+    if raw is None:
+        return None
+    try:
+        return frozenset(TypeAdapter(list[str]).validate_python(raw))
+    except ValidationError as e:
+        raise ValueError(
+            "general_settings.background_health_check_model_groups must be a list of model group names"
+        ) from e
+
+
+def filter_deployments_to_model_groups(
+    model_list: Sequence[_DeploymentT],
+    model_groups: AbstractSet[str] | None,
+) -> tuple[_DeploymentT, ...]:
+    """Deployments whose ``model_name`` is in ``model_groups``; all of them when unset."""
+    if model_groups is None:
+        return tuple(model_list)
+    return tuple(x for x in model_list if x.get("model_name") in model_groups)
+
+
 def filter_deployments_by_id(
     model_list: Sequence[Mapping[str, object]],
 ) -> list:

litellm/proxy/litellm_pre_call_utils.py

@@ -19,6 +19,7 @@ from litellm.constants import (
     CONSUMED_REQUEST_TAGS_METADATA_KEY,
     INTERNAL_CALL_ORIGIN_METADATA_KEY,
     LITELLM_PROXY_MASTER_KEY_ALIAS,
+    OTEL_SERVICE_NAME_METADATA_KEYS,
     PRE_CALL_EXECUTED_GUARDRAILS_KEY,
     SESSION_DEPLOYMENT_AFFINITY_TTL_METADATA_KEY,
 )
@@ -2003,6 +2004,19 @@ async def add_litellm_data_to_request(
         _metadata_variable_name=_metadata_variable_name,
     )
 
+    # A key's OTel service name outranks its team's, so the key's values are
+    # re-applied after the last-writer-wins team metadata merge above
+    _key_otel_service_names: Final = {
+        field: value
+        for field, value in (key_metadata or {}).items()
+        if field in OTEL_SERVICE_NAME_METADATA_KEYS and isinstance(value, str) and value.strip()
+    }
+    data = LiteLLMProxyRequestSetup.add_management_endpoint_metadata_to_request_metadata(
+        data=data,
+        management_endpoint_metadata=_key_otel_service_names,
+        _metadata_variable_name=_metadata_variable_name,
+    )
+
     # Team spend, budget - used by prometheus.py
     data[_metadata_variable_name]["user_api_key_team_max_budget"] = user_api_key_dict.team_max_budget
     data[_metadata_variable_name]["user_api_key_team_spend"] = user_api_key_dict.team_spend

litellm/proxy/management_endpoints/key_management_endpoints.py

@@ -2190,7 +2190,7 @@ async def _get_and_validate_existing_key(
 
         existing_key_row: Final[LiteLLM_VerificationToken | None] = await _prisma_table(
             VerificationTokenRepository(prisma_client)
-        ).find_unique(where={"token": hashed_token})
+        ).find_unique(where={"token": hashed_token}, include={"object_permission": True})
 
         if existing_key_row is None:
             raise ProxyException(
@@ -2442,11 +2442,13 @@ async def _validate_mcp_servers_for_key_update(
             check_db_only=True,
         )
     object_permission_dict: Final = _object_permission_to_dict(data.object_permission)
+    team_unchanged: Final = data.team_id is None or data.team_id == existing_key_row.team_id
     normalized_object_permission: Final = await validate_key_mcp_servers_against_team(
         object_permission=object_permission_dict,
         team_obj=effective_team_obj,
         prisma_client=prisma_client,
         is_proxy_admin=is_proxy_admin,
+        existing_key_object_permission=existing_key_row.object_permission if team_unchanged else None,
     )
     await validate_key_search_tools_against_team(
         object_permission=object_permission_dict,

litellm/proxy/management_endpoints/scim/scim_v2.py

@@ -6,6 +6,7 @@ This is an enterprise feature and requires a premium license.
 
 import re
 from collections.abc import Awaitable, Callable, Iterable, Mapping, Sequence
+from copy import deepcopy
 from dataclasses import dataclass
 from functools import partial
 from itertools import chain
@@ -2375,6 +2376,37 @@ async def get_group(
         raise handle_exception_on_proxy(e)
 
 
+def _new_team_request_with_defaults(
+    team_id: str,
+    team_alias: str | None,
+    members_with_roles: Sequence[Member],
+) -> NewTeamRequest:
+    """Build the SCIM group's team request, applying litellm.default_team_params
+    (including models) the same way SSO auto-created teams do."""
+    default_params: Final = litellm.default_team_params
+    defaults: Final[Mapping[str, object]] = (
+        deepcopy(default_params)
+        if isinstance(default_params, dict)
+        else default_params.model_dump(exclude_none=True)
+        if default_params is not None
+        else {}
+    )
+    default_metadata: Final = defaults.get("metadata")
+    metadata: Final = {
+        **(default_metadata if isinstance(default_metadata, dict) else {}),
+        SCIM_MANAGED_TEAM_METADATA_KEY: True,
+    }
+    return NewTeamRequest.model_validate(
+        {
+            **defaults,
+            "team_id": team_id,
+            "team_alias": team_alias,
+            "members_with_roles": members_with_roles,
+            "metadata": metadata,
+        }
+    )
+
+
 @scim_router.post(
     "/Groups",
     response_model=SCIMGroup,
@@ -2412,11 +2444,10 @@ async def create_group(
 
         # Create team in database
         created_team: Final = await new_team(
-            data=NewTeamRequest(
+            data=_new_team_request_with_defaults(
                 team_id=team_id,
                 team_alias=group.displayName,
                 members_with_roles=members_with_roles,
-                metadata={SCIM_MANAGED_TEAM_METADATA_KEY: True},
             ),
             http_request=Request(scope={"type": "http", "path": "/scim/v2/Groups"}),
             user_api_key_dict=UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN),

litellm/proxy/management_endpoints/ui_sso.py

@@ -808,6 +808,15 @@ def normalize_email(email: str | None) -> str | None:
     return email.lower() if isinstance(email, str) else email
 
 
+# Ordered highest to lowest privilege
+LITELLM_USER_ROLE_HIERARCHY: Final = (
+    LitellmUserRoles.PROXY_ADMIN,
+    LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY,
+    LitellmUserRoles.INTERNAL_USER,
+    LitellmUserRoles.INTERNAL_USER_VIEW_ONLY,
+)
+
+
 def determine_role_from_groups(
     user_groups: list[str],
     role_mappings: "RoleMappings",
@@ -832,19 +841,11 @@ def determine_role_from_groups(
         # No role mappings configured, return default_role
         return role_mappings.default_role
 
-    # Role hierarchy (highest to lowest)
-    role_hierarchy: Final = [
-        LitellmUserRoles.PROXY_ADMIN,
-        LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY,
-        LitellmUserRoles.INTERNAL_USER,
-        LitellmUserRoles.INTERNAL_USER_VIEW_ONLY,
-    ]
-
     # Convert user_groups to a set for efficient lookup
     user_groups_set: Final = set(user_groups) if isinstance(user_groups, list) else set()
 
     # Find the highest privilege role the user belongs to
-    for role in role_hierarchy:
+    for role in LITELLM_USER_ROLE_HIERARCHY:
         if role in role_mappings.roles:
             role_groups = role_mappings.roles[role]
             if isinstance(role_groups, list) and user_groups_set.intersection(set(role_groups)):
@@ -4236,15 +4237,7 @@ class MicrosoftSSOHandler:
         verbose_proxy_logger.debug("Extracted app roles from id_token: %s", app_roles)
 
         # Combine groups and app roles
-        user_role: LitellmUserRoles | None = None
-        if app_roles:
-            # Check if any app role is a valid LitellmUserRoles
-            for role_str in app_roles:
-                role = get_litellm_user_role(role_str)
-                if role is not None:
-                    user_role = role
-                    verbose_proxy_logger.debug("Found valid LitellmUserRoles '%s' in app_roles", role.value)
-                    break
+        user_role: Final = MicrosoftSSOHandler.get_user_role_from_app_roles(app_roles)
 
         verbose_proxy_logger.debug("Combined team_ids (groups + app roles): %s", user_team_ids)
 
@@ -4282,6 +4275,27 @@ class MicrosoftSSOHandler:
         verbose_proxy_logger.debug("Microsoft SSO OpenID Response: %s", openid_response)
         return openid_response
 
+    @staticmethod
+    def get_user_role_from_app_roles(
+        app_roles: Sequence[str] | None,
+    ) -> LitellmUserRoles | None:
+        """
+        Resolve the one role LiteLLM stores for a user from their Entra app roles.
+
+        Entra does not guarantee `roles` claim ordering, so a user holding several app
+        roles resolves to the highest privilege one rather than whichever the claim
+        listed first. Roles the hierarchy does not rank (org_admin, team, customer)
+        resolve by name to stay deterministic
+        """
+        resolved: Final = frozenset(
+            role for role in (get_litellm_user_role(role_str) for role_str in app_roles or ()) if role is not None
+        )
+        if not resolved:
+            return None
+
+        ranked: Final = next((role for role in LITELLM_USER_ROLE_HIERARCHY if role in resolved), None)
+        return ranked if ranked is not None else min(resolved, key=lambda role: role.value)
+
     @staticmethod
     def get_app_roles_from_id_token(id_token: str | None) -> list[str]:
         """

litellm/proxy/management_helpers/object_permission_utils.py

@@ -447,6 +447,36 @@ async def enforce_all_proxy_mcp_servers_grant_is_admin_only(
     )
 
 
+async def _get_grandfathered_key_mcp_server_ids(
+    existing_object_permission: Optional["LiteLLM_ObjectPermissionTable"],
+    prisma_client: PrismaClient | None,
+) -> frozenset[str]:
+    """
+    Resolve the canonical MCP server IDs a key's stored object_permission already
+    grants. Updates that keep or shrink those grants stay valid even when the
+    team allowlist has since changed; sentinels are excluded so they cannot
+    grandfather anything.
+    """
+    if existing_object_permission is None or prisma_client is None:
+        return frozenset()
+    raw_tool_perms: Final = existing_object_permission.mcp_tool_permissions or {}
+    tool_perm_keys: Final[frozenset[str]] = frozenset(
+        json.loads(raw_tool_perms).keys() if isinstance(raw_tool_perms, str) else raw_tool_perms.keys()
+    )
+    identifiers: Final = (frozenset(existing_object_permission.mcp_servers or []) | tool_perm_keys) - {
+        SpecialMCPServerNames.no_mcp_servers.value,
+        SpecialMCPServerName.all_proxy_servers.value,
+    }
+    return frozenset(
+        _flatten_resolved_mcp_server_ids(
+            await _resolve_mcp_server_identifiers_to_ids(
+                identifiers=set(identifiers),
+                prisma_client=prisma_client,
+            )
+        )
+    )
+
+
 async def _get_team_allowed_mcp_servers(
     team_obj: Optional["LiteLLM_TeamTableCachedObj"],
     prisma_client: PrismaClient | None = None,
@@ -527,10 +557,16 @@ async def validate_key_mcp_servers_against_team(
     team_obj: Optional["LiteLLM_TeamTableCachedObj"],
     prisma_client: PrismaClient | None = None,
     is_proxy_admin: bool = False,
+    existing_key_object_permission: Optional["LiteLLM_ObjectPermissionTable"] = None,
 ) -> ObjectPermissionDict | None:
     """
     Validate that MCP servers requested on a key are within the allowed scope.
 
+    When ``existing_key_object_permission`` is provided (key updates), servers
+    the key already holds are grandfathered: keeping or removing them stays valid
+    even if the team allowlist has since shrunk, while adding new servers outside
+    the allowlist is still rejected.
+
     Rules:
     - If key is in a team: key's mcp_servers must be a subset of
       (team's allowed servers + allow_all_keys servers)
@@ -589,7 +625,11 @@ async def validate_key_mcp_servers_against_team(
         if teamless_admin_assignment:
             allowed_servers = all_allowed_servers | active_requested_servers
 
-        disallowed_servers: Final = active_requested_servers - allowed_servers
+        grandfathered_servers: Final = await _get_grandfathered_key_mcp_server_ids(
+            existing_object_permission=existing_key_object_permission,
+            prisma_client=prisma_client,
+        )
+        disallowed_servers: Final = active_requested_servers - allowed_servers - grandfathered_servers
         if disallowed_servers:
             if team_obj is not None:
                 team_id = team_obj.team_id

litellm/proxy/prompts/prompt_endpoints.py

@@ -1021,19 +1021,7 @@ async def delete_prompt(
         # Delete versions from the database (scoped to environment if provided)
         await _prompt_table(prisma_client).delete_many(where=delete_where)
 
-        # Remove matching prompts from memory — scope to environment if provided
-        if environment:
-            prompts_to_delete: Final = [
-                pid
-                for pid, prompt in IN_MEMORY_PROMPT_REGISTRY.IN_MEMORY_PROMPTS.items()
-                if get_base_prompt_id(prompt_id=pid) == base_prompt_id and prompt.environment == environment
-            ]
-            for pid in prompts_to_delete:
-                del IN_MEMORY_PROMPT_REGISTRY.IN_MEMORY_PROMPTS[pid]
-                if pid in IN_MEMORY_PROMPT_REGISTRY.prompt_id_to_custom_prompt:
-                    del IN_MEMORY_PROMPT_REGISTRY.prompt_id_to_custom_prompt[pid]
-        else:
-            IN_MEMORY_PROMPT_REGISTRY.delete_prompts_by_base_id(base_prompt_id)
+        IN_MEMORY_PROMPT_REGISTRY.delete_prompts_by_base_id(base_prompt_id, environment=environment or None)
 
         env_msg: Final = f" from {environment}" if environment else ""
         return {"message": f"Prompt {base_prompt_id} deleted successfully{env_msg}"}

litellm/proxy/prompts/prompt_registry.py

@@ -195,12 +195,22 @@ class InMemoryPromptRegistry:
         """
         return self.prompt_id_to_custom_prompt.get(prompt_id)
 
-    def delete_prompts_by_base_id(self, base_prompt_id: str) -> list[str]:
+    def remove_prompt(self, prompt_id: str) -> None:
+        import litellm
+
+        self.IN_MEMORY_PROMPTS.pop(prompt_id, None)
+        stale_callback: Final = self.prompt_id_to_custom_prompt.pop(prompt_id, None)
+        if stale_callback is not None:
+            litellm.logging_callback_manager.remove_callback_from_all_lists(stale_callback)
+
+    def delete_prompts_by_base_id(self, base_prompt_id: str, environment: str | None = None) -> list[str]:
         """
-        Delete all prompts matching the given base prompt ID from memory.
+        Delete all prompts matching the given base prompt ID from memory, along with their
+        registered callbacks; scoped to one environment when given.
 
         Args:
             base_prompt_id: The base prompt ID (without version suffix)
+            environment: When set, only delete prompts deployed to this environment
 
         Returns:
             List of prompt IDs that were deleted
@@ -208,13 +218,14 @@ class InMemoryPromptRegistry:
         from litellm.proxy.prompts.prompt_endpoints import get_base_prompt_id
 
         prompts_to_delete: Final = [
-            pid for pid in self.IN_MEMORY_PROMPTS if get_base_prompt_id(prompt_id=pid) == base_prompt_id
+            pid
+            for pid, prompt in self.IN_MEMORY_PROMPTS.items()
+            if get_base_prompt_id(prompt_id=pid) == base_prompt_id
+            and (environment is None or prompt.environment == environment)
         ]
 
         for pid in prompts_to_delete:
-            del self.IN_MEMORY_PROMPTS[pid]
-            if pid in self.prompt_id_to_custom_prompt:
-                del self.prompt_id_to_custom_prompt[pid]
+            self.remove_prompt(prompt_id=pid)
 
         return prompts_to_delete
 

litellm/proxy/proxy_cli.py

@@ -1321,10 +1321,10 @@ def run_server(
                             use_v2_resolver=use_v2_migration_resolver,
                         )
                     except RuntimeError as e:
-                        # v2 resolver raises on unrecoverable migration errors
-                        # (e.g. non-idempotent failures, permission issues).
-                        # v1 never raises here, so this only fires when the
-                        # operator opted into v2.
+                        # Raised on unrecoverable migration errors: the v2
+                        # resolver's non-idempotent failures and permission
+                        # issues, and any `prisma db push` against a
+                        # partitioned LiteLLM_SpendLogs.
                         print(
                             f"\033[1;31mLiteLLM Proxy: Database migration cannot proceed. {e}\033[0m",
                             file=sys.stderr,

litellm/proxy/proxy_server.py

@@ -411,7 +411,9 @@ from litellm.proxy.guardrails.init_guardrails import (
     initialize_guardrails,
 )
 from litellm.proxy.health_check import (
+    filter_deployments_to_model_groups,
     health_check_filter_kwargs_from_general_settings,
+    parse_background_health_check_model_groups,
     perform_health_check,
 )
 from litellm.proxy.health_endpoints._health_endpoints import router as health_router
@@ -3660,6 +3662,13 @@ async def _run_background_health_check():
         _llm_model_list = [
             m for m in _llm_model_list if not m.get("model_info", {}).get("disable_background_health_check", False)
         ]
+        scoped_model_groups = llm_router.background_health_check_model_groups if llm_router is not None else None
+        _llm_model_list = list(filter_deployments_to_model_groups(_llm_model_list, scoped_model_groups))
+        if scoped_model_groups is not None and not _llm_model_list:
+            verbose_proxy_logger.warning(
+                "background_health_check_model_groups matched no deployments; groups=%s",
+                sorted(scoped_model_groups),
+            )
         model_count_enabled = len(_llm_model_list)
         expected_peak_in_flight = model_count_enabled
         if isinstance(health_check_concurrency, int) and health_check_concurrency > 0 and model_count_enabled > 0:
@@ -5239,6 +5248,7 @@ class ProxyConfig:
         general_settings = config.get("general_settings", {})
         if general_settings is None:
             general_settings = {}
+        _bg_hc_model_groups: Final = parse_background_health_check_model_groups(general_settings)
         _enable_hc_routing = False
         _hc_staleness = None
         _hc_ignore_transient = False
@@ -5434,13 +5444,14 @@ class ProxyConfig:
             _hc_staleness = general_settings.get("health_check_staleness_threshold", None)
             _hc_ignore_transient = general_settings.get("health_check_ignore_transient_errors", False)
             verbose_proxy_logger.info(
-                "background_health_check_config enabled=%s shared=%s interval_seconds=%s max_concurrency=%s details=%s health_check_routing=%s",
+                "background_health_check_config enabled=%s shared=%s interval_seconds=%s max_concurrency=%s details=%s health_check_routing=%s model_groups=%s",
                 use_background_health_checks,
                 use_shared_health_check,
                 health_check_interval,
                 health_check_concurrency,
                 health_check_details,
                 _enable_hc_routing,
+                sorted(_bg_hc_model_groups) if _bg_hc_model_groups is not None else None,
             )
 
             ### RBAC ###
@@ -5472,6 +5483,8 @@ class ProxyConfig:
             router_params["health_check_staleness_threshold"] = _hc_staleness
         if _hc_ignore_transient:
             router_params["health_check_ignore_transient_errors"] = True
+        if _bg_hc_model_groups is not None:
+            router_params["background_health_check_model_groups"] = sorted(_bg_hc_model_groups)
         ## MODEL LIST
         model_list: Final = config.get("model_list", None)
         if model_list:
@@ -7268,6 +7281,7 @@ class ProxyConfig:
                 return None
 
         try:
+            prompt_ids_loaded_before_db_read: Final = frozenset(IN_MEMORY_PROMPT_REGISTRY.IN_MEMORY_PROMPTS)
             prompts_in_db: Final[Sequence[object]] = await PromptRepository(prisma_client).table.find_many()
             parsed_specs: Final[tuple[PromptSpec, ...]] = tuple(
                 spec for row in prompts_in_db if (spec := parse_row(row)) is not None
@@ -7290,6 +7304,18 @@ class ProxyConfig:
                         prompt_spec.prompt_id,
                         prompt_sync_error,
                     )
+            # An unparsable row still exists in the DB, so skip the sweep rather than unload its in-memory copy
+            every_row_parsed: Final = len(parsed_specs) == len(prompts_in_db)
+            if every_row_parsed:
+                deleted_db_prompt_ids: Final = tuple(
+                    prompt_id
+                    for prompt_id in prompt_ids_loaded_before_db_read
+                    if (loaded_spec := IN_MEMORY_PROMPT_REGISTRY.IN_MEMORY_PROMPTS.get(prompt_id)) is not None
+                    and loaded_spec.prompt_info.prompt_type == "db"
+                    and prompt_id not in newest_spec_per_id
+                )
+                for deleted_prompt_id in deleted_db_prompt_ids:
+                    IN_MEMORY_PROMPT_REGISTRY.remove_prompt(prompt_id=deleted_prompt_id)
         except Exception as e:
             verbose_proxy_logger.debug("litellm.proxy.proxy_server.py::ProxyConfig:_init_prompts_in_db - %s", e)
 
@@ -9227,6 +9253,7 @@ class ProxyStartupEvent:
                     prisma_client,
                     pod_lock_manager=proxy_logging_obj.db_spend_update_writer.pod_lock_manager,
                     alert=_alert_ptu_rollup_failure,
+                    router=llm_router,
                 )
 
             scheduler.add_job(
@@ -16389,6 +16416,13 @@ _GENERAL_SETTINGS_UI_LITELLM_FIELDS: Final[dict[str, GeneralSettingsUILiteLLMFie
         "tab": "prompt_caching",
         "description": "Empty uses Anthropic's 5m default. 1h suits long sessions but doubles the cache write cost.",
     },
+    "budget_rollover": {  # mutable-ok: registry literal, frozen with its siblings below
+        "type": "Boolean",
+        "description": (
+            "Carry spend beyond max_budget into the next window when budgets reset, instead of "
+            "forgiving it. Applies to key, user, team, team member, org, tag and end-user budgets."
+        ),
+    },
     "max_ui_session_budget": {
         "type": "Dollar",
         "default": 1.0,

litellm/proxy/spend_tracking/ptu_flat_cost_rollup.py

@@ -14,7 +14,6 @@ and share the existing unique constraint.
 
 import asyncio
 import json
-import sys
 from collections.abc import Awaitable, Callable, Mapping
 from dataclasses import dataclass
 from datetime import date, datetime, time, timedelta, timezone
@@ -326,16 +325,6 @@ class _LoadedDeployments:
     scanned_ids: frozenset[str]
 
 
-def _running_router() -> object | None:
-    """The proxy's router, or None outside a running proxy.
-
-    Read out of ``sys.modules`` rather than imported, so a rollup driven from a test or a
-    script does not pull the whole proxy server in behind it.
-    """
-    proxy_server: Final = sys.modules.get("litellm.proxy.proxy_server")
-    return getattr(proxy_server, "llm_router", None) if proxy_server is not None else None
-
-
 def _config_deployments(router: object | None, *, owned_by_db: frozenset[str]) -> tuple[_PTUDeployment, ...]:
     """Deployments the router holds that no ``LiteLLM_ProxyModelTable`` row owns.
 
@@ -356,15 +345,17 @@ def _config_deployments(router: object | None, *, owned_by_db: frozenset[str]) -
     )
 
 
-async def _load_ptu_models(prisma_client: "PrismaClient") -> _LoadedDeployments:
+async def _load_ptu_models(prisma_client: "PrismaClient", *, router: object | None) -> _LoadedDeployments:
     """Every deployment carrying valid manual PTU config, and every id the scan saw.
 
     Reserved capacity is billed by the provider whichever file declared it, so a
     deployment the proxy only knows from config.yaml accrues alongside the stored ones.
+    The router is handed in rather than read off the proxy module, so a run prices exactly
+    the deployments its caller declares and nothing a co-resident process left behind.
     """
     rows: Final = await prisma_client.db.litellm_proxymodeltable.find_many()
     db_ids: Final = frozenset(model_id for row in rows if (model_id := str(getattr(row, "model_id", "") or "")))
-    config_records: Final = _config_deployments(_running_router(), owned_by_db=db_ids)
+    config_records: Final = _config_deployments(router, owned_by_db=db_ids)
     models: Final = tuple(
         parsed for parsed in (_parse_ptu_model(row) for row in (*rows, *config_records)) if parsed is not None
     )
@@ -380,6 +371,7 @@ async def run_ptu_flat_cost_rollup(
     prisma_client: "PrismaClient",
     target_date: date | None = None,
     may_prune: bool = True,
+    router: object | None = None,
 ) -> RollupResult:
     """Rollup one UTC day of flat PTU cost across all PTU-configured model deployments.
 
@@ -406,7 +398,7 @@ async def run_ptu_flat_cost_rollup(
     date_str: Final = day.isoformat()
     run_started: Final = datetime.now(timezone.utc)
 
-    loaded: Final = await _load_ptu_models(prisma_client)
+    loaded: Final = await _load_ptu_models(prisma_client, router=router)
     ptu_models: Final = loaded.models
     charges: Final = _aggregate_charges(ptu_models, day)
 
@@ -527,6 +519,7 @@ async def _existing_sentinel_keys(
 async def run_ptu_flat_cost_backfill(
     prisma_client: "PrismaClient",
     today: date | None = None,
+    router: object | None = None,
 ) -> BackfillResult:
     """Price the elapsed days of every PTU window that carry no sentinel row yet.
 
@@ -546,7 +539,7 @@ async def run_ptu_flat_cost_backfill(
         verbose_proxy_logger.warning("PTU backfill: prisma_client is None, skipping")
         return BackfillResult(start=end, end=end, days_scanned=0, rows_written=0)
 
-    ptu_models: Final = (await _load_ptu_models(prisma_client)).models
+    ptu_models: Final = (await _load_ptu_models(prisma_client, router=router)).models
     days: Final = _backfill_window(ptu_models, end)
 
     if not days:
@@ -591,6 +584,7 @@ async def run_scheduled_ptu_rollup(
     pod_lock_manager: "PodLockManager | None" = None,
     target_date: date | None = None,
     alert: Callable[[str], Awaitable[None]] | None = None,
+    router: object | None = None,
 ) -> RollupResult | None:
     """Run the daily rollup under a cross-pod lock so only one proxy reconciles a day.
 
@@ -615,7 +609,7 @@ async def run_scheduled_ptu_rollup(
         return None
 
     if pod_lock_manager is None or pod_lock_manager.redis_cache is None:
-        return await _run_and_alert(prisma_client, target_date=target_date, alert=alert, may_prune=False)
+        return await _run_and_alert(prisma_client, target_date=target_date, alert=alert, may_prune=False, router=router)
 
     if not await pod_lock_manager.acquire_lock(cronjob_id=PTU_ROLLUP_JOB_ID, ttl=PTU_ROLLUP_LOCK_TTL_SECONDS):
         if await _lock_is_held(pod_lock_manager):
@@ -629,10 +623,10 @@ async def run_scheduled_ptu_rollup(
             "PTU rollup: could not take the rollup lock and no other pod holds it, "
             "running unguarded rather than skipping the day"
         )
-        return await _run_and_alert(prisma_client, target_date=target_date, alert=alert, may_prune=False)
+        return await _run_and_alert(prisma_client, target_date=target_date, alert=alert, may_prune=False, router=router)
 
     try:
-        return await _run_and_alert(prisma_client, target_date=target_date, alert=alert, may_prune=True)
+        return await _run_and_alert(prisma_client, target_date=target_date, alert=alert, may_prune=True, router=router)
     finally:
         await pod_lock_manager.release_lock(cronjob_id=PTU_ROLLUP_JOB_ID)
 
@@ -657,6 +651,7 @@ async def _run_and_alert(
     target_date: date | None,
     alert: "Callable[[str], Awaitable[None]] | None",
     may_prune: bool = True,
+    router: object | None = None,
 ) -> RollupResult:
     """Reconcile the day, catch up any days left unpriced, and alert on charges that did not land.
 
@@ -669,7 +664,9 @@ async def _run_and_alert(
     explicit date means reconcile exactly that day, so it stays a single-day operation.
     Its failure is contained: the day's own result is returned either way.
     """
-    result: Final = await run_ptu_flat_cost_rollup(prisma_client, target_date=target_date, may_prune=may_prune)
+    result: Final = await run_ptu_flat_cost_rollup(
+        prisma_client, target_date=target_date, may_prune=may_prune, router=router
+    )
     if result.rows_failed:
         await _deliver_alert(
             alert,
@@ -686,7 +683,7 @@ async def _run_and_alert(
             "by the provider with nothing attributing it here. Extend the window, or retire the deployment.",
         )
     if target_date is None:
-        await _backfill_and_alert(prisma_client, alert=alert)
+        await _backfill_and_alert(prisma_client, alert=alert, router=router)
     return result
 
 
@@ -694,6 +691,7 @@ async def _backfill_and_alert(
     prisma_client: "PrismaClient",
     *,
     alert: "Callable[[str], Awaitable[None]] | None",
+    router: object | None = None,
 ) -> None:
     """Catch up unpriced PTU days, alerting on charges that did not land.
 
@@ -701,7 +699,7 @@ async def _backfill_and_alert(
     caller whatever the catch-up pass does.
     """
     try:
-        backfill: Final = await run_ptu_flat_cost_backfill(prisma_client)
+        backfill: Final = await run_ptu_flat_cost_backfill(prisma_client, router=router)
     except Exception as exc:  # noqa: BLE001  # the catch-up pass must not fail the day's rollup
         verbose_proxy_logger.error("PTU backfill: catch-up pass failed, the day's rollup still stands: %s", exc)
         return

litellm/proxy/spend_tracking/spend_management_endpoints.py

@@ -23,6 +23,7 @@ from typing_extensions import ReadOnly
 
 import litellm
 from litellm._logging import verbose_proxy_logger
+from litellm.constants import LITTELM_INTERNAL_HEALTH_SERVICE_ACCOUNT_NAME
 from litellm.proxy._types import *
 from litellm.proxy._types import ProviderBudgetResponse, ProviderBudgetResponseObject
 from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
@@ -54,6 +55,11 @@ router: Final = APIRouter()
 
 SPEND_LOGS_PAGINATION_COUNT_CAP: Final = 10000
 
+_INTERNAL_HEALTH_CHECK_API_KEYS: Final = (
+    LITTELM_INTERNAL_HEALTH_SERVICE_ACCOUNT_NAME,
+    hash_token(token=LITTELM_INTERNAL_HEALTH_SERVICE_ACCOUNT_NAME),
+)
+
 _RowT = TypeVar("_RowT")
 
 
@@ -2248,6 +2254,10 @@ async def ui_view_spend_logs(
     status_filter: str | None = fastapi.Query(
         default=None, description="Filter logs by status (e.g., success, failure)"
     ),
+    cache_hit_filter: str | None = fastapi.Query(
+        default=None,
+        description="Filter logs by cache state: 'hit' or 'miss'. Miss includes legacy rows with a null/unknown cache state",
+    ),
     model: str | None = fastapi.Query(default=None, description="Filter logs by model"),
     model_id: str | None = fastapi.Query(
         default=None,
@@ -2268,6 +2278,10 @@ async def ui_view_spend_logs(
         default="desc",
         description="Sort order: asc or desc",
     ),
+    exclude_internal_health_checks: bool = fastapi.Query(
+        default=False,
+        description="Exclude LiteLLM internal health check requests from results",
+    ),
 ):
     """
     View spend logs with pagination support.
@@ -2320,6 +2334,13 @@ async def ui_view_spend_logs(
             param="sort_order",
             code=status.HTTP_400_BAD_REQUEST,
         )
+    if isinstance(cache_hit_filter, str) and cache_hit_filter not in {"hit", "miss"}:
+        raise ProxyException(
+            message=f"Invalid cache_hit_filter: {cache_hit_filter}. Must be one of: hit, miss",
+            type="bad_request",
+            param="cache_hit_filter",
+            code=status.HTTP_400_BAD_REQUEST,
+        )
 
     try:
         is_admin_view: Final = _is_admin_view_safe(user_api_key_dict=user_api_key_dict)
@@ -2560,6 +2581,16 @@ async def ui_view_spend_logs(
                 sql_params.append(status_filter)
                 p += 1
 
+        if cache_hit_filter == "hit":
+            sql_conditions.append("LOWER(cache_hit) = 'true'")
+        elif cache_hit_filter == "miss":
+            sql_conditions.append("(cache_hit IS NULL OR LOWER(cache_hit) != 'true')")
+
+        if exclude_internal_health_checks:
+            sql_conditions.append(f"api_key NOT IN (${p}, ${p + 1})")
+            sql_params.extend(_INTERNAL_HEALTH_CHECK_API_KEYS)
+            p += 2  # rebind-ok: advances the file's shared $N placeholder counter
+
         # Spend range
         if min_spend is not None:
             sql_conditions.append(f"spend >= ${p}")

litellm/repositories/unit_of_work.py

@@ -19,32 +19,57 @@ from collections.abc import AsyncGenerator, Callable, Mapping
 from contextlib import asynccontextmanager
 from dataclasses import dataclass
 from datetime import datetime
+from typing import Final
 
 from litellm.repositories.prisma_protocols import BatchTable, PrismaBatch
 
 
+def _spend_reset_data(budget_reset_at: datetime | None, spend_decrement: float | None) -> Mapping[str, object]:
+    spend: Final[object] = (
+        {"decrement": spend_decrement}  # mutable-ok: prisma update payload must be a dict
+        if spend_decrement is not None
+        else 0
+    )
+    return {"spend": spend, "budget_reset_at": budget_reset_at}  # mutable-ok: prisma update payload must be a dict
+
+
 @dataclass(frozen=True, slots=True)
 class KeySpendResetWrites:
     table: BatchTable
 
-    def queue_spend_reset(self, token: str, budget_reset_at: datetime | None) -> None:
-        self.table.update(where={"token": token}, data={"spend": 0, "budget_reset_at": budget_reset_at})
+    def queue_spend_reset(
+        self, token: str, budget_reset_at: datetime | None, spend_decrement: float | None = None
+    ) -> None:
+        self.table.update(
+            where={"token": token},  # mutable-ok: prisma where filter must be a dict
+            data=_spend_reset_data(budget_reset_at, spend_decrement),
+        )
 
 
 @dataclass(frozen=True, slots=True)
 class UserSpendResetWrites:
     table: BatchTable
 
-    def queue_spend_reset(self, user_id: str, budget_reset_at: datetime | None) -> None:
-        self.table.update(where={"user_id": user_id}, data={"spend": 0, "budget_reset_at": budget_reset_at})
+    def queue_spend_reset(
+        self, user_id: str, budget_reset_at: datetime | None, spend_decrement: float | None = None
+    ) -> None:
+        self.table.update(
+            where={"user_id": user_id},  # mutable-ok: prisma where filter must be a dict
+            data=_spend_reset_data(budget_reset_at, spend_decrement),
+        )
 
 
 @dataclass(frozen=True, slots=True)
 class TeamSpendResetWrites:
     table: BatchTable
 
-    def queue_spend_reset(self, team_id: str, budget_reset_at: datetime | None) -> None:
-        self.table.update(where={"team_id": team_id}, data={"spend": 0, "budget_reset_at": budget_reset_at})
+    def queue_spend_reset(
+        self, team_id: str, budget_reset_at: datetime | None, spend_decrement: float | None = None
+    ) -> None:
+        self.table.update(
+            where={"team_id": team_id},  # mutable-ok: prisma where filter must be a dict
+            data=_spend_reset_data(budget_reset_at, spend_decrement),
+        )
 
 
 @dataclass(frozen=True, slots=True)
@@ -54,6 +79,14 @@ class LinkedSpendResetWrites:
     def queue_spend_zero(self, where: Mapping[str, object]) -> None:
         self.table.update_many(where=where, data={"spend": 0})
 
+    def queue_spend_decrement(self, where: Mapping[str, object], amount: float) -> None:
+        """``decrement`` rather than a read-then-set, so spend written between the
+        cascade's read and its commit survives the reset instead of being erased."""
+        self.table.update_many(
+            where=where,
+            data={"spend": {"decrement": amount}},  # mutable-ok: prisma update payload must be a dict
+        )
+
 
 @dataclass(frozen=True, slots=True)
 class BudgetWindowWrites:

litellm/router.py

@@ -602,6 +602,7 @@ class Router:
         enable_health_check_routing: bool = False,
         health_check_staleness_threshold: int | None = None,
         health_check_ignore_transient_errors: bool = False,
+        background_health_check_model_groups: Sequence[str] | None = None,
         enable_weighted_failover: bool = False,
     ) -> None:
         """
@@ -811,6 +812,11 @@ class Router:
         self.enable_health_check_routing = enable_health_check_routing
         self.enable_weighted_failover = enable_weighted_failover
         self.health_check_ignore_transient_errors = health_check_ignore_transient_errors
+        self.background_health_check_model_groups: frozenset[str] | None = (
+            frozenset(background_health_check_model_groups)
+            if background_health_check_model_groups is not None
+            else None
+        )
         _staleness: Final = health_check_staleness_threshold or (
             DEFAULT_HEALTH_CHECK_INTERVAL * DEFAULT_HEALTH_CHECK_STALENESS_MULTIPLIER
         )
@@ -9210,7 +9216,11 @@ class Router:
             }
 
         if model_id is not None:
-            litellm.register_model(model_cost={model_id: model_info}, persist_across_reloads=False)
+            litellm.register_model(
+                model_cost={model_id: model_info},
+                persist_across_reloads=False,
+                warning_display_name=model,
+            )
 
         ## OLD MODEL REGISTRATION ## Kept to prevent breaking changes
         backend_keys: Final = Router._backend_cost_map_keys(model=model, custom_llm_provider=custom_llm_provider)
@@ -12719,6 +12729,10 @@ class Router:
         """
         Filter out deployments marked unhealthy by background health checks.
         No-op when enable_health_check_routing is False.
+        When background_health_check_model_groups is set, only deployments in the
+        listed model groups are filtered; every other group keeps its configured
+        routing strategy untouched, and a router-level allowed_fails_policy no
+        longer disables the filter for the listed groups.
         Returns all deployments if health state is unavailable, stale, or would
         exclude every candidate (safety net).
         """
@@ -12727,8 +12741,10 @@ class Router:
 
         # When allowed_fails_policy is set, cooldown is the sole routing exclusion
         # mechanism -- skip the binary health check filter so the policy threshold
-        # is respected before any deployment is excluded.
-        if self.allowed_fails_policy is not None:
+        # is respected before any deployment is excluded. With a model-group
+        # allowlist the filter is already scoped, so listed groups keep it.
+        scoped_groups: Final = self.background_health_check_model_groups
+        if self.allowed_fails_policy is not None and scoped_groups is None:
             return healthy_deployments
 
         unhealthy_ids: Final = await self.health_state_cache.async_get_unhealthy_deployment_ids(
@@ -12737,7 +12753,12 @@ class Router:
         if not unhealthy_ids:
             return healthy_deployments
 
-        filtered: Final = [d for d in healthy_deployments if d["model_info"]["id"] not in unhealthy_ids]
+        filtered: Final = [
+            d
+            for d in healthy_deployments
+            if d["model_info"]["id"] not in unhealthy_ids
+            or (scoped_groups is not None and d["model_name"] not in scoped_groups)
+        ]
 
         if not filtered:
             verbose_router_logger.warning("All deployments marked unhealthy by health checks, bypassing health filter")
@@ -12754,14 +12775,20 @@ class Router:
         if not self.enable_health_check_routing:
             return healthy_deployments
 
-        if self.allowed_fails_policy is not None:
+        scoped_groups: Final = self.background_health_check_model_groups
+        if self.allowed_fails_policy is not None and scoped_groups is None:
             return healthy_deployments
 
         unhealthy_ids: Final = self.health_state_cache.get_unhealthy_deployment_ids(parent_otel_span=parent_otel_span)
         if not unhealthy_ids:
             return healthy_deployments
 
-        filtered: Final = [d for d in healthy_deployments if d["model_info"]["id"] not in unhealthy_ids]
+        filtered: Final = [
+            d
+            for d in healthy_deployments
+            if d["model_info"]["id"] not in unhealthy_ids
+            or (scoped_groups is not None and d["model_name"] not in scoped_groups)
+        ]
 
         if not filtered:
             verbose_router_logger.warning("All deployments marked unhealthy by health checks, bypassing health filter")

litellm/router_utils/health_state_cache.py

@@ -43,12 +43,33 @@ class DeploymentHealthCache:
         self.staleness_threshold = staleness_threshold
 
     def set_deployment_health_states(self, states: dict[str, DeploymentHealthStateValue]) -> None:
-        """Bulk-write all deployment health states as a single cache entry."""
+        """Merge the given states into the shared cache entry, pruning expired ones.
+
+        Merging instead of replacing lets writers probing different deployment
+        scopes (e.g. pods with different background health check allowlists)
+        coexist on the one shared entry without erasing each other's results.
+        The snapshot is read from Redis when available, since a pod-local read
+        would only ever see this writer's own previous merge. When the Redis
+        read comes back empty (a miss, or a swallowed connection error), the
+        pod-local copy of the last merge is used so peers are not erased.
+        """
         try:
+            redis_raw: Final = (
+                self.cache.redis_cache.get_cache(self.CACHE_KEY) if self.cache.redis_cache is not None else None
+            )
+            raw: Final = redis_raw if isinstance(redis_raw, dict) else self.cache.get_cache(key=self.CACHE_KEY)
+            existing: Final = raw if isinstance(raw, dict) else {}
+            expiry_seconds: Final = self.staleness_threshold * 1.5
+            now: Final = time.time()
+            merged: Final = {
+                model_id: state
+                for model_id, state in {**existing, **states}.items()
+                if isinstance(state, dict) and (now - state.get("timestamp", 0)) < expiry_seconds
+            }
             self.cache.set_cache(
                 key=self.CACHE_KEY,
-                value=states,
-                ttl=int(self.staleness_threshold * 1.5),
+                value=merged,
+                ttl=int(expiry_seconds),
             )
         except Exception as e:
             verbose_logger.error(

litellm/types/integrations/newrelic.py

@@ -1,3 +1,10 @@
+from collections.abc import Mapping, Sequence
+from dataclasses import dataclass
+from types import MappingProxyType
+from typing import Final, Literal
+
+from typing_extensions import ReadOnly, TypedDict
+
 from litellm.types.integrations.custom_logger import StandardCustomLoggerInitParams
 
 
@@ -5,3 +12,110 @@ class NewRelicInitParams(StandardCustomLoggerInitParams):
     """
     Params for initializing a New Relic logger on litellm
     """
+
+
+#: Region -> Metric API endpoint. A fixed table by design: team config picks a
+#: region enum rather than a free-form endpoint, so callback vars can never
+#: redirect metrics to an arbitrary host.
+NEWRELIC_METRIC_ENDPOINT_BY_REGION: Final[Mapping[str, str]] = MappingProxyType(
+    {
+        "us": "https://metric-api.newrelic.com/metric/v1",
+        "eu": "https://metric-api.eu.newrelic.com/metric/v1",
+    }
+)
+
+NEWRELIC_DEFAULT_REGION: Final = "us"
+
+#: Metric API caps a payload at 2000 data points / 1MB compressed; each queued
+#: record expands to at most 6 metrics, so cap the per-flush record count well
+#: below that.
+NEWRELIC_METRICS_MAX_BATCH_SIZE: Final = 250
+
+#: Hard cap on records retained across failed flushes (5xx/network requeue).
+#: Beyond this the oldest records are dropped.
+NEWRELIC_METRICS_MAX_RETRY_QUEUE_SIZE: Final = 10_000
+# Outer passes over a stopped logger's queue: each pass retries the whole
+# queue, so records that arrive mid-drain still get attempts before the bounded
+# terminal drop. Serialized by a per-logger drain lock, so this bounds work.
+NEWRELIC_METRICS_MAX_DRAIN_PASSES: Final = 3
+# Metric API caps attribute values; 255 keeps caller-controlled model strings
+# from inflating the shared batch payload into a 413
+NEWRELIC_METRIC_ATTRIBUTE_MAX_LEN: Final = 255
+
+NEWRELIC_METRIC_REQUESTS: Final = "litellm.requests"
+NEWRELIC_METRIC_COST_USD: Final = "litellm.cost.usd"
+NEWRELIC_METRIC_PROMPT_TOKENS: Final = "litellm.tokens.prompt"
+NEWRELIC_METRIC_COMPLETION_TOKENS: Final = "litellm.tokens.completion"
+NEWRELIC_METRIC_TOTAL_TOKENS: Final = "litellm.tokens.total"
+NEWRELIC_METRIC_REQUEST_DURATION_MS: Final = "litellm.request.duration_ms"
+
+
+class NewRelicSummaryValue(TypedDict):
+    """Value shape of a Metric API ``summary`` data point."""
+
+    count: ReadOnly[int]
+    sum: ReadOnly[float]
+    min: ReadOnly[float]
+    max: ReadOnly[float]
+
+
+class NewRelicCountMetric(TypedDict):
+    name: ReadOnly[str]
+    type: ReadOnly[Literal["count"]]
+    value: ReadOnly[float]
+    attributes: ReadOnly[Mapping[str, str]]
+
+
+class NewRelicSummaryMetric(TypedDict):
+    name: ReadOnly[str]
+    type: ReadOnly[Literal["summary"]]
+    value: ReadOnly[NewRelicSummaryValue]
+    attributes: ReadOnly[Mapping[str, str]]
+
+
+NewRelicMetric = NewRelicCountMetric | NewRelicSummaryMetric
+
+
+#: ``interval.ms`` has a dot in it, so the functional TypedDict form is required.
+NewRelicMetricCommon = TypedDict(
+    "NewRelicMetricCommon",
+    {  # mutable-ok: functional TypedDict requires a dict-literal fields argument ("interval.ms" key)
+        "timestamp": ReadOnly[int],
+        "interval.ms": ReadOnly[int],
+    },
+)
+
+
+class NewRelicMetricEnvelope(TypedDict):
+    """One element of the Metric API request body (``[{common, metrics}]``)."""
+
+    common: ReadOnly[NewRelicMetricCommon]
+    metrics: ReadOnly[Sequence[NewRelicMetric]]
+
+
+@dataclass(frozen=True, slots=True)
+class NewRelicMetricRecord:
+    """One request's contribution to the per-flush aggregation."""
+
+    team_id: str
+    team_alias: str
+    model_group: str
+    model: str
+    custom_llm_provider: str
+    status: str
+    response_cost: float
+    prompt_tokens: int
+    completion_tokens: int
+    total_tokens: int
+    duration_ms: float
+
+    @property
+    def bucket_key(self) -> tuple[str, str, str, str, str, str]:
+        return (
+            self.team_id,
+            self.team_alias,
+            self.model_group,
+            self.model,
+            self.custom_llm_provider,
+            self.status,
+        )

litellm/types/llms/anthropic.py

@@ -502,11 +502,16 @@ class MessageDelta(TypedDict, total=False):
     stop_reason: str | None
 
 
+class ServerToolUsage(TypedDict, total=False):
+    web_search_requests: ReadOnly[int]
+
+
 class UsageDelta(TypedDict, total=False):
     input_tokens: int
     output_tokens: int
     cache_creation_input_tokens: int
     cache_read_input_tokens: int
+    server_tool_use: ReadOnly[ServerToolUsage]
 
 
 class AppliedEdit(TypedDict, total=False):

litellm/types/llms/anthropic_messages/anthropic_response.py

@@ -1,11 +1,12 @@
 from typing import Any, Literal, TypeAlias
 
-from typing_extensions import NotRequired, TypedDict
+from typing_extensions import NotRequired, ReadOnly, TypedDict
 
 from litellm.types.llms.anthropic import (
     AnthropicResponseContentBlockText,
     AnthropicResponseContentBlockToolUse,
     ContextManagementResponse,
+    ServerToolUsage,
 )
 
 
@@ -71,6 +72,11 @@ class AnthropicUsage(TypedDict, total=False):
     cache_creation_input_tokens: int
     cache_read_input_tokens: int
 
+    """
+    Server-side tool usage (e.g. web search request counts)
+    """
+    server_tool_use: NotRequired[ReadOnly[ServerToolUsage]]
+
 
 class AnthropicMessagesResponse(TypedDict, total=False):
     """

litellm/types/llms/gemini_audio_transcription.py

@@ -0,0 +1,81 @@
+from typing import Literal, Required
+
+from pydantic import BaseModel, ConfigDict
+from typing_extensions import ReadOnly, TypedDict
+
+
+class GeminiTranscriptionAudioInput(TypedDict):
+    type: ReadOnly[Literal["audio"]]
+    data: ReadOnly[str]
+    mime_type: ReadOnly[str]
+
+
+class GeminiTranscriptionVerbatimMode(TypedDict, total=False):
+    type: ReadOnly[Required[Literal["verbatim"]]]
+    timestamp_granularities: ReadOnly[tuple[Literal["word"], ...]]
+    diarization_mode: ReadOnly[Literal["speaker"]]
+
+
+class GeminiTranscriptionConfig(TypedDict, total=False):
+    language_codes: ReadOnly[tuple[str, ...]]
+    mode: ReadOnly[GeminiTranscriptionVerbatimMode]
+
+
+class GeminiTranscriptionGenerationConfig(TypedDict):
+    transcription_config: ReadOnly[GeminiTranscriptionConfig]
+
+
+class GeminiTranscriptionInteractionRequest(TypedDict, total=False):
+    model: ReadOnly[Required[str]]
+    input: ReadOnly[Required[tuple[GeminiTranscriptionAudioInput, ...]]]
+    generation_config: ReadOnly[GeminiTranscriptionGenerationConfig]
+
+
+class GeminiTranscriptionWordAnnotation(BaseModel):
+    model_config = ConfigDict(extra="ignore")
+
+    type: str | None = None
+    text: str | None = None
+    speaker: str | None = None
+    start_offset: str | None = None
+    end_offset: str | None = None
+
+
+class GeminiTranscriptionContent(BaseModel):
+    model_config = ConfigDict(extra="ignore")
+
+    type: str | None = None
+    text: str | None = None
+    annotations: tuple[GeminiTranscriptionWordAnnotation, ...] = ()
+
+
+class GeminiTranscriptionStep(BaseModel):
+    model_config = ConfigDict(extra="ignore")
+
+    type: str | None = None
+    content: tuple[GeminiTranscriptionContent, ...] = ()
+
+
+class GeminiTranscriptionModalityTokens(BaseModel):
+    model_config = ConfigDict(extra="ignore")
+
+    modality: str | None = None
+    tokens: int = 0
+
+
+class GeminiTranscriptionUsage(BaseModel):
+    model_config = ConfigDict(extra="ignore")
+
+    total_tokens: int = 0
+    total_input_tokens: int = 0
+    total_output_tokens: int = 0
+    input_tokens_by_modality: tuple[GeminiTranscriptionModalityTokens, ...] = ()
+
+
+class GeminiTranscriptionInteractionResponse(BaseModel):
+    model_config = ConfigDict(extra="ignore")
+
+    id: str | None = None
+    status: str | None = None
+    usage: GeminiTranscriptionUsage | None = None
+    steps: tuple[GeminiTranscriptionStep, ...] = ()

litellm/types/realtime.py

@@ -162,3 +162,16 @@ class RealtimeErrorDetail(TypedDict):
 class RealtimeErrorEvent(TypedDict):
     type: ReadOnly[Literal["error"]]
     error: ReadOnly[RealtimeErrorDetail]
+
+
+class RealtimeInputAudioTranscriptionUsageInputTokenDetails(TypedDict):
+    text_tokens: ReadOnly[int]
+    audio_tokens: ReadOnly[int]
+
+
+class RealtimeInputAudioTranscriptionUsage(TypedDict):
+    type: ReadOnly[Literal["tokens"]]
+    input_tokens: ReadOnly[int]
+    output_tokens: ReadOnly[int]
+    total_tokens: ReadOnly[int]
+    input_token_details: ReadOnly[RealtimeInputAudioTranscriptionUsageInputTokenDetails]

litellm/utils.py

@@ -2948,7 +2948,12 @@ def reapply_runtime_model_cost_registrations() -> None:
         register_model(model_cost=dict(_runtime_registered_model_cost))  # mutable-ok: snapshot, replay rewrites it
 
 
-def register_model(model_cost: str | dict, *, persist_across_reloads: bool = True):
+def register_model(
+    model_cost: str | dict,
+    *,
+    persist_across_reloads: bool = True,
+    warning_display_name: str | None = None,
+):
     """
     Register new / Override existing models (and their pricing) to specific providers.
     Provide EITHER a model cost dictionary or a url to a hosted json blob
@@ -2968,6 +2973,10 @@ def register_model(model_cost: str | dict, *, persist_across_reloads: bool = Tru
     registering a model is declaring durable intent. Pass False for a
     registration that only describes one request, so it is dropped rather than
     re-asserted over every future catalog.
+
+    ``warning_display_name`` names the model in the missing-cache-pricing
+    warning instead of the registered key, for callers that register under an
+    opaque key (e.g. the router's hashed deployment ids).
     """
 
     loaded_model_cost = {}
@@ -3014,10 +3023,15 @@ def register_model(model_cost: str | dict, *, persist_across_reloads: bool = Tru
                 elif (
                     value.get("cache_creation_input_token_cost") is None
                     and value.get("cache_read_input_token_cost") is None
+                    and value.get("tiered_pricing") is None
+                    and (
+                        value.get("input_cost_per_token") is not None
+                        or value.get("output_cost_per_token") is not None
+                    )
                 ):
                     verbose_logger.warning(
-                        "register_model: model=%s not in built-in cost map and no prefix/region variant matched; cache cost fields will default to 0. To track cache cost, add cache_creation_input_token_cost and cache_read_input_token_cost to model_info",
-                        key,
+                        "register_model: model=%s has custom pricing but not in built-in cost map and no prefix/region variant matched; cache_creation_input_token_cost and cache_read_input_token_cost will default to 0 for this model (input/output cost tracking is unaffected). To track cache cost, add them to model_info",
+                        warning_display_name or key,
                     )
         # ``get_model_info`` returns ``litellm_provider: None`` when the
         # provider is unknown (e.g. custom deployments registered via
@@ -8503,6 +8517,12 @@ class ProviderConfigManager:
             )
 
             return VertexAIAudioTranscriptionConfig()
+        elif litellm.LlmProviders.GEMINI == provider:
+            from litellm.llms.gemini.audio_transcription.transformation import (
+                GeminiAudioTranscriptionConfig,
+            )
+
+            return GeminiAudioTranscriptionConfig()
         return None
 
     @staticmethod

pyproject.toml

@@ -67,8 +67,8 @@ proxy = [
     "azure-identity>=1.25.2,<2.0",
     "azure-storage-blob>=12.28.0,<13.0",
     "mcp>=1.28.1,<2.0",
-    "litellm-proxy-extras==0.4.89",
-    "litellm-enterprise==0.1.60",
+    "litellm-proxy-extras==0.4.90",
+    "litellm-enterprise==0.1.61",
     "RestrictedPython>=8.1,<9.0",
     "rich>=13.9.4,<14.0",
     "InquirerPy>=0.3.4,<1.0",

scripts/pre_commit_lint.sh

@@ -13,7 +13,7 @@
 #   - tests/e2e Python -> `make lint-e2e-basedpyright` (test-linting.yml's e2e type-check step)
 #                         + raw HTTP client ban (test-code-quality.yml's check_e2e_no_raw_requests)
 #   - dashboard        -> prettier + eslint + lint budgets (test-litellm-ui-build.yml's frontend-lint)
-#   - proxy/types      -> regenerate dashboard API types and fail on drift (check-ui-api-types.yml)
+#   - proxy/types      -> regenerate the lazy OpenAPI snapshot and dashboard API types, fail on drift (check-ui-api-types.yml)
 #
 # Each block is skipped when no matching files are in scope, so unrelated commits
 # stay fast. This is intentionally not auto-installed as a git hook (see
@@ -244,7 +244,7 @@ fi
 
 genapi_checks() {
     local status=0
-    echo "check: checking dashboard API types are in sync (npm run gen:api)"
+    echo "check: checking the lazy OpenAPI snapshot and dashboard API types are in sync (npm run gen:api)"
     # gen-api-types.mjs imports litellm.proxy.proxy_server, which needs the proxy deps
     # and an up-to-date Prisma client; check-ui-api-types.yml installs those and runs
     # prisma generate before gen:api, so mirror that here or a stale client can mask
@@ -260,7 +260,14 @@ genapi_checks() {
     elif ! uv run --no-sync python scripts/prisma_generate_if_needed.py; then
         echo "✗ Could not regenerate Prisma client (prisma generate failed)." >&2
         status=1
+    elif ! uv run --no-sync python -m litellm.proxy._lazy_openapi_snapshot; then
+        echo "✗ Could not regenerate the lazy OpenAPI snapshot (python -m litellm.proxy._lazy_openapi_snapshot failed)." >&2
+        status=1
     elif ( cd ui/litellm-dashboard && LITELLM_PYTHON="uv run --no-sync python" npm run gen:api ); then
+        if ! git diff --quiet -- litellm/proxy/_lazy_openapi_snapshot.json; then
+            echo "✗ The lazy OpenAPI snapshot is stale; regenerated litellm/proxy/_lazy_openapi_snapshot.json. Stage it and commit; re-run make check only if other checks failed too." >&2
+            status=1
+        fi
         if ! git diff --quiet -- ui/litellm-dashboard/src/lib/http/schema.d.ts; then
             echo "✗ Dashboard API types are stale; regenerated src/lib/http/schema.d.ts. Stage it and commit; re-run make check only if other checks failed too." >&2
             status=1

tests/code_coverage_tests/check_licenses.py

@@ -5,8 +5,9 @@ import json
 from pathlib import Path
 import re
 import sys
+import time
 import tomllib
-from typing import Dict, List, Optional, Set, Tuple
+from typing import Callable, Dict, Final, List, Optional, Protocol, Set, Tuple
 
 from packaging.requirements import Requirement
 import requests
@@ -37,6 +38,13 @@ DEFAULT_TRANSITIVE_PIN_PACKAGES = (
 # of the identifier, not an operator.
 _SPDX_OPERATOR_SPLIT = re.compile(r"\s+(?:OR|AND)\s+")
 _SPDX_WITH_SUFFIX = re.compile(r"\s+WITH\s+.*", re.DOTALL)
+_PYPI_FETCH_ATTEMPTS: Final[int] = 3
+_PYPI_FETCH_BACKOFF_SECONDS: Final[float] = 0.5
+
+
+class _HttpGet(Protocol):
+    def __call__(self, url: str, *, timeout: float) -> requests.Response:
+        ...
 
 
 @dataclass
@@ -50,7 +58,10 @@ class PackageLicense:
 
 class LicenseChecker:
     def __init__(
-        self, config_file: Path = Path("./tests/code_coverage_tests/liccheck.ini")
+        self,
+        config_file: Path = Path("./tests/code_coverage_tests/liccheck.ini"),
+        http_get: Optional[_HttpGet] = None,
+        sleep: Optional[Callable[[float], None]] = None,
     ):
         if not config_file.exists():
             print(f"Error: Config file {config_file} not found")
@@ -79,6 +90,8 @@ class LicenseChecker:
 
         # Track package results
         self.package_results: List[PackageLicense] = []
+        self._http_get = http_get
+        self._sleep = sleep
 
     @staticmethod
     def _normalize_package_name(package_name: str) -> str:
@@ -123,21 +136,38 @@ class LicenseChecker:
         last resort derives the license from the ``License :: OSI Approved ::
         ...`` trove classifiers.
         """
-        try:
-            url = f"https://pypi.org/pypi/{package_name}/{version}/json"
-            response = requests.get(url, timeout=10)
-            response.raise_for_status()
-            info = response.json().get("info", {}) or {}
-            return (
-                info.get("license_expression")
-                or info.get("license")
-                or self._license_from_classifiers(info.get("classifiers") or [])
-            )
-        except Exception as e:
-            print(
-                f"Warning: Failed to fetch license for {package_name} {version}: {str(e)}"
-            )
-            return None
+        url = f"https://pypi.org/pypi/{package_name}/{version}/json"
+        http_get = self._http_get if self._http_get is not None else requests.get
+        sleep = self._sleep if self._sleep is not None else time.sleep
+
+        for attempt in range(_PYPI_FETCH_ATTEMPTS):
+            try:
+                response = http_get(url, timeout=10)
+                response.raise_for_status()
+                info = response.json().get("info", {}) or {}
+                return (
+                    info.get("license_expression")
+                    or info.get("license")
+                    or self._license_from_classifiers(info.get("classifiers") or [])
+                )
+            except Exception as error:
+                if self._is_retryable_pypi_error(error) and attempt < _PYPI_FETCH_ATTEMPTS - 1:
+                    sleep(_PYPI_FETCH_BACKOFF_SECONDS)
+                    continue
+                print(
+                    f"Warning: Failed to fetch license for {package_name} {version}: {str(error)}"
+                )
+                return None
+        return None
+
+    @staticmethod
+    def _is_retryable_pypi_error(error: Exception) -> bool:
+        if isinstance(error, (requests.ConnectionError, requests.Timeout)):
+            return True
+        if not isinstance(error, requests.HTTPError) or error.response is None:
+            return False
+        status_code = error.response.status_code
+        return status_code == 429 or status_code >= 500
 
     @staticmethod
     def _license_from_classifiers(classifiers: List[str]) -> Optional[str]:

tests/e2e/llm_translation/realtime/REALTIME_COVERAGE_MATRIX.md

@@ -42,7 +42,7 @@ at call time. The provider table below is the source of truth; edit `PROVIDERS`
 | openai | `openai-realtime` | `openai/gpt-realtime-2` |
 | azure | `azure-realtime` | `azure/gpt-realtime-2` (GA protocol) |
 | gemini | `gemini-realtime` | `gemini/gemini-3.1-flash-live-preview` |
-| vertex_ai | `vertex-realtime` | `vertex_ai/gemini-live-2.5-flash-preview-native-audio-09-2025` |
+| vertex_ai | `vertex-realtime` | `vertex_ai/gemini-live-2.5-flash-native-audio` |
 
 Bedrock and xai (`xai/grok-4-1-fast-non-reasoning`) are supported by the proxy but
 kept commented out in `PROVIDERS` until they pass end-to-end here; re-enable them by

tests/e2e/llm_translation/realtime/realtime_client.py

@@ -78,7 +78,7 @@ PROVIDERS = (
         "vertex_ai",
         "vertex-realtime",
         LiteLLMParamsBody(
-            model="vertex_ai/gemini-live-2.5-flash-preview-native-audio-09-2025",
+            model="vertex_ai/gemini-live-2.5-flash-native-audio",
             vertex_location="us-central1",
             vertex_credentials="os.environ/VERTEXAI_CREDENTIALS",
         ),

tests/e2e/llm_translation/test_chat_completions_regression_e2e.py

@@ -308,7 +308,8 @@ class TestGeminiChatCompletions:
                             content=f"Reply with the single word pong. marker={tag}",
                         )
                     ],
-                    max_tokens=32,
+                    max_tokens=64,
+                    reasoning_effort="none",
                 ),
             )
         )

tests/e2e/llm_translation/test_messages_mid_conversation_system_e2e.py

@@ -50,11 +50,14 @@ CACHE_WARM_CONSECUTIVE_READS = 3
 
 
 def _cacheable_system_block(marker: str) -> TextBlock:
-    """A system prompt comfortably above the 4096-token minimum cacheable size
-    of Haiku 4.5 (the smallest model here), unique per run so no other run's
-    cache entry can satisfy the read."""
-    text = " ".join(
-        f"Reference paragraph {index} for run {marker}." for index in range(300)
+    """A system prompt at roughly twice the 4096-token minimum cacheable size of
+    Haiku 4.5 (the smallest model here), unique per run so no other run's cache
+    entry can satisfy the read. The marker appears once instead of in every
+    paragraph: repeating it swung the block's size by ~1800 tokens with the
+    marker's own tokenization and left it under the minimum on ~15% of runs, so
+    the system breakpoint went uncached and the priming loop never saw a read."""
+    text = f"Run {marker}.\n" + " ".join(
+        f"Reference paragraph {index}." for index in range(1500)
     )
     return TextBlock(text=text, cache_control=CacheControl())
 
@@ -101,8 +104,8 @@ def _first_turn_user_text(marker: str) -> str:
     """A first user turn heavy enough (hundreds of tokens) that losing its cache
     entry is unambiguous in the usage numbers, unique per attempt so priming
     retries never depend on the proxy's response cache behavior."""
-    notes = " ".join(f"Session note {index} for attempt {marker}." for index in range(100))
-    return f"Reply with one word.\n{notes}"
+    notes = " ".join(f"Session note {index}." for index in range(100))
+    return f"Reply with one word. Attempt {marker}.\n{notes}"
 
 
 class PrimedCache(BaseModel):

tests/e2e/llm_translation/test_messages_mid_conversation_system_native_providers_e2e.py

@@ -70,9 +70,15 @@ def _vertex_params(model: str, location: str) -> LiteLLMParamsBody:
 
 
 def _cacheable_system_block(marker: str) -> TextBlock:
-    """A system prompt comfortably above the 1024-token minimum cacheable size,
-    unique per run so no other run's cache entry can satisfy the read."""
-    text = " ".join(f"Reference paragraph {index} for run {marker}." for index in range(300))
+    """A system prompt at roughly twice the 4096-token minimum cacheable size of
+    Haiku 4.5 (the smallest model here), unique per run so no other run's cache
+    entry can satisfy the read. The marker appears once instead of in every
+    paragraph: repeating it swung the block's size by ~1800 tokens with the
+    marker's own tokenization and left it under the minimum on ~15% of runs, so
+    the system breakpoint went uncached and the priming loop never saw a read."""
+    text = f"Run {marker}.\n" + " ".join(
+        f"Reference paragraph {index}." for index in range(1500)
+    )
     return TextBlock(text=text, cache_control=CacheControl())
 
 
@@ -110,8 +116,8 @@ def _first_turn_user_text(marker: str) -> str:
     """A first user turn heavy enough (hundreds of tokens) that losing its cache
     entry is unambiguous in the usage numbers, unique per attempt so priming
     retries never depend on the proxy's response cache behavior."""
-    notes = " ".join(f"Session note {index} for attempt {marker}." for index in range(100))
-    return f"Reply with one word.\n{notes}"
+    notes = " ".join(f"Session note {index}." for index in range(100))
+    return f"Reply with one word. Attempt {marker}.\n{notes}"
 
 
 class PrimedCache(BaseModel):

tests/e2e/management/management_client.py

@@ -9,8 +9,21 @@ from __future__ import annotations
 import time
 from dataclasses import dataclass
 
+import jwt
+
+from e2e_config import MASTER_KEY
 from proxy_client import ProxyClient
-from e2e_http import NoBody, ProbeResult, Result, StreamingResponse, Success, UnknownApiError, unwrap
+from e2e_http import (
+    AuthHeaders,
+    NetworkError,
+    NoBody,
+    ProbeResult,
+    Result,
+    StreamingResponse,
+    Success,
+    UnknownApiError,
+    unwrap,
+)
 from models import (
     ChatBody,
     ChatMessage,
@@ -50,6 +63,9 @@ from models import (
     TeamNewBody,
     TeamNewResponse,
     TeamUpdateBody,
+    UiLoginBody,
+    UiLoginResponse,
+    UiSessionClaims,
     UserDeleteBody,
     UserDeleteResponse,
     UserInfoParams,
@@ -63,38 +79,73 @@ from models import (
 
 MODEL_ACCESS_DENIED_MARKER = "key_model_access_denied"
 ROUTE_NOT_ALLOWED_MARKER = "not allowed to call this route"
+DASHBOARD_SESSION_TEAM_ID = "litellm-dashboard"
 _TEAM_READY_ATTEMPTS = 15
 _TEAM_READY_SLEEP_SECONDS = 0.4
+_KEY_WRITE_ATTEMPTS = 5
+_TRANSIENT_BACKEND_MARKERS = ("connecting to redis", "name resolution")
+
+
+@dataclass(frozen=True, slots=True)
+class DashboardSession:
+    """What a dashboard sign-in hands the Admin UI: the session key it sends as
+    its bearer on every subsequent call, the claims it renders the signed-in user
+    from, and where it lands the browser."""
+
+    session_key: str
+    claims: UiSessionClaims
+    redirect_url: str
 
 
 @dataclass(frozen=True, slots=True)
 class ManagementClient:
     proxy: ProxyClient
+    master_key: str
 
     def llm_only_key(self) -> str:
         return self.proxy.generate_key(KeyGenerateBody(models=[], allowed_routes=["llm_api_routes"]))
 
-    def update_key_models(self, key: str, models: list[str]) -> None:
-        last: Result[NoBody] | None = None
-        for attempt in range(5):
+    def generate_key(self, body: KeyGenerateBody, *, caller_key: str | None = None) -> Result[KeyGenerateResponse]:
+        """POST /key/generate. `caller_key` is who is creating the key: the master
+        key by default, or a virtual key (an admin filling in Create New Key on the
+        dashboard creates it under the session key their sign-in minted). Returns
+        the outcome rather than unwrapping it, so a caller can poll a route that is
+        only transiently refusing."""
+        headers = self.proxy.transport.master if caller_key is None else self.proxy.transport.bearer(caller_key)
+        return self.proxy.transport.post(
+            "/key/generate",
+            headers=headers,
+            json=body,
+            response_type=KeyGenerateResponse,
+        )
+
+    def update_key(self, body: KeyUpdateBody, *, caller_key: str | None = None) -> Result[NoBody]:
+        """POST /key/update. `caller_key` is who is editing: the master key by
+        default, or a virtual key (the dashboard edits under the session key its
+        sign-in minted, never the master key). Returns the outcome rather than
+        unwrapping it, so a caller can poll a route that is only transiently
+        refusing; `update_key_models` is the unwrapping shorthand."""
+        headers = self.proxy.transport.master if caller_key is None else self.proxy.transport.bearer(caller_key)
+        last: Result[NoBody] = NetworkError(message="/key/update was never attempted")
+        for attempt in range(_KEY_WRITE_ATTEMPTS):
             last = self.proxy.transport.post(
                 "/key/update",
-                headers=self.proxy.transport.master,
-                json=KeyUpdateBody(key=key, models=models),
+                headers=headers,
+                json=body,
                 response_type=NoBody,
             )
             match last:
-                case Success():
-                    return
-                case UnknownApiError(body=body) if (
-                    "connecting to redis" in body.lower() or "name resolution" in body.lower()
+                case UnknownApiError(body=error_body) if any(
+                    marker in error_body.lower() for marker in _TRANSIENT_BACKEND_MARKERS
                 ):
                     time.sleep(0.5 * (attempt + 1))
                     continue
                 case _:
                     break
-        assert last is not None
-        raise AssertionError(last)
+        return last
+
+    def update_key_models(self, key: str, models: list[str]) -> None:
+        _ = unwrap(self.update_key(KeyUpdateBody(key=key, models=models)))
 
     def delete_key_strict(self, key: str) -> None:
         """Strict delete for the act phase of a test: a failed delete is a hard
@@ -150,15 +201,42 @@ class ManagementClient:
             )
         ).key
 
+    def key_list(self, key_alias: str, *, caller_key: str | None = None) -> Result[KeyListResponse]:
+        """GET /key/list, the Virtual Keys page's own inventory call. `caller_key` is
+        who is asking: the master key by default, or a virtual key."""
+        headers = self.proxy.transport.master if caller_key is None else self.proxy.transport.bearer(caller_key)
+        return self.proxy.transport.get(
+            "/key/list",
+            headers=headers,
+            params=KeyListParams(key_alias=key_alias),
+            response_type=KeyListResponse,
+        )
+
     def key_alias_count(self, key_alias: str) -> int:
-        return unwrap(
-            self.proxy.transport.get(
-                "/key/list",
-                headers=self.proxy.transport.master,
-                params=KeyListParams(key_alias=key_alias),
-                response_type=KeyListResponse,
+        return unwrap(self.key_list(key_alias)).total_count
+
+    def dashboard_login(self, username: str, password: str) -> DashboardSession:
+        """POST /v2/login, the call the Admin UI's sign-in form makes.
+
+        The proxy authenticates the credentials, mints a UI session key for the
+        signed-in user, and hands it back inside a JWT signed with the master key.
+        Decoding that JWT is the only way to reach the session key, and it is what
+        the dashboard itself does before it can call a single management route."""
+        response = unwrap(
+            self.proxy.transport.post(
+                "/v2/login",
+                headers=AuthHeaders(),
+                json=UiLoginBody(username=username, password=password),
+                response_type=UiLoginResponse,
             )
-        ).total_count
+        )
+        decoded: object = jwt.decode(response.token, self.master_key, algorithms=["HS256"])
+        claims = UiSessionClaims.model_validate(decoded)
+        return DashboardSession(
+            session_key=claims.key,
+            claims=claims,
+            redirect_url=response.redirect_url,
+        )
 
     def create_team(self, body: TeamNewBody) -> str:
         team_id = unwrap(
@@ -465,4 +543,4 @@ class ManagementClient:
 
 
 def build_client(proxy: ProxyClient) -> ManagementClient:
-    return ManagementClient(proxy=proxy)
+    return ManagementClient(proxy=proxy, master_key=MASTER_KEY)

tests/e2e/management/test_management_e2e.py

@@ -15,15 +15,30 @@ from collections.abc import Callable
 
 import pytest
 
-from e2e_config import unique_marker
-from e2e_http import StreamingResponse
+from e2e_config import UI_PASSWORD, UI_USERNAME, unique_marker
+from e2e_http import StreamingResponse, Success
 from lifecycle import ResourceManager
 from management_client import (
+    DASHBOARD_SESSION_TEAM_ID,
     MODEL_ACCESS_DENIED_MARKER,
     ROUTE_NOT_ALLOWED_MARKER,
     ManagementClient,
 )
-from models import KeyGenerateBody, OrgInfoResponse, OrgNewBody, OrgUpdateBody, TagListEntry, TagNewBody, TeamNewBody, TeamUpdateBody, UserNewBody, UserUpdateBody, LiteLLMParamsBody, ModelInfoEntry
+from models import (
+    KeyGenerateBody,
+    KeyUpdateBody,
+    LiteLLMParamsBody,
+    ModelInfoEntry,
+    OrgInfoResponse,
+    OrgNewBody,
+    OrgUpdateBody,
+    TagListEntry,
+    TagNewBody,
+    TeamNewBody,
+    TeamUpdateBody,
+    UserNewBody,
+    UserUpdateBody,
+)
 
 pytestmark = pytest.mark.e2e
 
@@ -199,6 +214,132 @@ class TestKeyRoutes:
             return True if client.proxy.key_info(key).blocked else None
 
         _ = _poll(client, blocked, "/key/info never reported the key blocked after /key/block before the deadline")
+
+
+class TestDashboardKeyRoutes:
+    """The /key writes as the Admin UI makes them. Signing in mints the session key
+    the dashboard authenticates with, and every key an admin creates or edits in the
+    browser is written under that session key rather than the master key, so these
+    are the same routes the API-surface tests cover with a different caller."""
+
+    @pytest.mark.covers("mgmt.key.generate.happy_path")
+    def test_creating_a_key_from_the_dashboard_persists_and_works(
+        self, client: ManagementClient, resources: ResourceManager
+    ) -> None:
+        session = client.dashboard_login(UI_USERNAME, UI_PASSWORD)
+        resources.defer(lambda: client.proxy.delete_key(session.session_key))
+
+        assert session.claims.login_method == "username_password", (
+            f"/v2/login reports login_method {session.claims.login_method!r} for a username/password sign-in"
+        )
+        assert session.claims.user_role == "proxy_admin", (
+            f"/v2/login reports user_role {session.claims.user_role!r} for the admin credentials, "
+            "expected 'proxy_admin'"
+        )
+        assert session.redirect_url.endswith("/ui?login=success"), (
+            f"/v2/login sends the browser to {session.redirect_url!r} instead of the dashboard"
+        )
+
+        session_info = client.proxy.key_info(session.session_key)
+        assert session_info.team_id == DASHBOARD_SESSION_TEAM_ID, (
+            f"the minted session key reports team_id {session_info.team_id!r}, expected the dashboard's "
+            f"{DASHBOARD_SESSION_TEAM_ID!r}"
+        )
+
+        alias = f"e2e-mgmt-uicreate-{unique_marker()}"
+
+        def dashboard_creates_the_key() -> str | None:
+            match client.generate_key(
+                KeyGenerateBody(models=["gemini-2.5-flash"], key_alias=alias, tpm_limit=100),
+                caller_key=session.session_key,
+            ):
+                case Success(data=created):
+                    return created.key
+                case _:
+                    return None
+
+        created = _poll(
+            client,
+            dashboard_creates_the_key,
+            "the dashboard session key was never accepted on /key/generate before the deadline",
+        )
+        resources.defer(lambda: client.proxy.delete_key(created))
+
+        created_info = client.proxy.key_info(created)
+        assert created_info.key_alias == alias, (
+            f"/key/info reports key_alias {created_info.key_alias!r} for the key the dashboard created, "
+            f"expected {alias!r}"
+        )
+        assert created_info.models == ["gemini-2.5-flash"], (
+            f"/key/info reports models {created_info.models} for the key the dashboard created"
+        )
+        assert created_info.tpm_limit == 100, (
+            f"/key/info reports tpm_limit {created_info.tpm_limit} for the key the dashboard created, expected 100"
+        )
+
+        def dashboard_lists_the_key() -> bool | None:
+            match client.key_list(alias, caller_key=session.session_key):
+                case Success(data=listing) if listing.total_count == 1:
+                    return True
+                case _:
+                    return None
+
+        _ = _poll(
+            client,
+            dashboard_lists_the_key,
+            f"the session key never saw {alias!r} in /key/list before the deadline, so the dashboard "
+            "would render no keys",
+        )
+
+        _poll_chat_ok(client, created, "gemini-2.5-flash")
+        _assert_model_denied(client.chat_status(created, "gpt-5.5", f"say hi {unique_marker()}"), "gpt-5.5")
+
+    @pytest.mark.covers("mgmt.key.update.happy_path")
+    def test_editing_a_key_from_the_dashboard_persists_and_is_enforced(
+        self, client: ManagementClient, resources: ResourceManager
+    ) -> None:
+        alias = f"e2e-mgmt-uiedit-{unique_marker()}"
+        target = _generate_key(
+            client,
+            resources,
+            KeyGenerateBody(models=["gemini-2.5-flash"], key_alias=alias, tpm_limit=100, rpm_limit=200),
+        )
+        _poll_chat_ok(client, target, "gemini-2.5-flash")
+        _assert_model_denied(client.chat_status(target, "gpt-5.5", f"say hi {unique_marker()}"), "gpt-5.5")
+
+        session = client.dashboard_login(UI_USERNAME, UI_PASSWORD)
+        resources.defer(lambda: client.proxy.delete_key(session.session_key))
+
+        def dashboard_saves_the_edit() -> bool | None:
+            match client.update_key(
+                KeyUpdateBody(key=target, models=["gpt-5.5"], tpm_limit=300, rpm_limit=400),
+                caller_key=session.session_key,
+            ):
+                case Success():
+                    return True
+                case _:
+                    return None
+
+        _ = _poll(
+            client,
+            dashboard_saves_the_edit,
+            "the dashboard session key was never accepted on /key/update before the deadline",
+        )
+
+        info = client.proxy.key_info(target)
+        assert info.models == ["gpt-5.5"], (
+            f"/key/info reports models {info.models} after the dashboard edit to ['gpt-5.5']"
+        )
+        assert info.tpm_limit == 300, f"/key/info reports tpm_limit {info.tpm_limit} after the dashboard edit to 300"
+        assert info.rpm_limit == 400, f"/key/info reports rpm_limit {info.rpm_limit} after the dashboard edit to 400"
+        assert info.key_alias == alias, (
+            f"the dashboard edit renamed the key to {info.key_alias!r}, it should still be {alias!r}"
+        )
+
+        _poll_model_access_granted(client, target, "gpt-5.5")
+        _poll_chat_denied(client, target, "gemini-2.5-flash")
+
+
 class TestKeyRegeneration:
     @pytest.mark.covers("mgmt.key.regenerate.happy_path")
     def test_regenerate_rotates_to_a_working_new_key(

tests/e2e/models.py

@@ -256,6 +256,7 @@ class ChatBody(BaseModel):
     reasoning_effort: str | None = None
     thinking: ThinkingParam | None = None
     service_tier: str | None = None
+    prompt_cache_key: str | None = None
     tools: Sequence[ChatTool | McpChatTool] | None = None
     tool_choice: str | None = None
     guardrails: list[str] | None = None
@@ -892,7 +893,10 @@ class CredentialCreateResponse(BaseModel):
 
 class KeyUpdateBody(BaseModel):
     key: str
-    models: list[str]
+    models: list[str] | None = None
+    key_alias: str | None = None
+    tpm_limit: int | None = None
+    rpm_limit: int | None = None
 
 
 class KeyBlockBody(BaseModel):
@@ -907,6 +911,27 @@ class KeyListResponse(BaseModel):
     total_count: int
 
 
+# ---------- admin UI session ----------
+
+
+class UiLoginBody(BaseModel):
+    username: str
+    password: str
+
+
+class UiLoginResponse(BaseModel):
+    token: str
+    redirect_url: str
+
+
+class UiSessionClaims(BaseModel):
+    user_id: str
+    key: str
+    user_role: str
+    login_method: Literal["sso", "username_password"]
+    exp: int
+
+
 class TeamMemberEntry(BaseModel):
     role: Literal["admin", "user"]
     user_id: str

tests/e2e/quota_management/spend_tracking/test_cost_headers_e2e.py

@@ -12,11 +12,16 @@ header is exercised with a real nonzero value instead of passing vacuously. The
 backend is gpt-5.5 because it reports cached tokens on the second call; the
 gpt-5.6 line reports cache writes and never a read, which would leave the
 cache-read header at zero forever. The raw-transport send is used because the
-typed chat client validates bodies and drops headers. OpenAI caching is
-best-effort, so the prime+measure round retries with a fresh prefix before
-failing.
+typed chat client validates bodies and drops headers.
+
+OpenAI publishes a primed prefix asynchronously and routes lookups by
+prompt_cache_key, so a measure fired the instant the prime returns can miss a
+prefix that is about to become readable. Each round pins a cache key and re-reads
+the prefix it already paid to prime before spending a fresh one.
 """
 
+import time
+
 import pytest
 
 from cost_rows import approx_equal, cacheable_prefix, register_priced_model
@@ -31,6 +36,8 @@ pytestmark = pytest.mark.e2e
 BACKEND = "openai/gpt-5.5"
 OPENAI_API_KEY = "os.environ/OPENAI_API_KEY"
 CACHE_ATTEMPTS = 3
+CACHE_REREADS = 3
+CACHE_SETTLE_SECONDS = 2.0
 
 INPUT_RATE = 4e-05
 OUTPUT_RATE = 8e-05
@@ -70,7 +77,7 @@ class TestCostHeaders:
             ),
         )
 
-        def priced_call(content: str) -> StreamingResponse:
+        def priced_call(content: str, cache_key: str) -> StreamingResponse:
             response = client.proxy.transport.send(
                 "/chat/completions",
                 headers=client.proxy.transport.bearer(scoped_key),
@@ -78,21 +85,30 @@ class TestCostHeaders:
                     model=model,
                     messages=[ChatMessage(role="user", content=content)],
                     max_completion_tokens=4000,
+                    prompt_cache_key=cache_key,
                 ),
             )
             assert response.ok, f"chat failed (status {response.status_code}): {response.body[:300]}"
             return response
 
-        for _ in range(CACHE_ATTEMPTS):
-            prefix = cacheable_prefix(unique_marker())
-            priced_call(f"{prefix}\nReply with the single word ready.")
-            measured = priced_call(f"{prefix}\nReply with the single word measured.")
-            if _header_cost(measured, "x-litellm-response-cost-cache-read") > 0:
-                break
-        else:
+        def prime_then_reread() -> StreamingResponse | None:
+            marker = unique_marker()
+            prefix = cacheable_prefix(marker)
+            priced_call(f"{prefix}\nReply with the single word ready.", marker)
+            for _ in range(CACHE_REREADS):
+                time.sleep(CACHE_SETTLE_SECONDS)
+                response = priced_call(f"{prefix}\nReply with the single word measured.", marker)
+                if _header_cost(response, "x-litellm-response-cost-cache-read") > 0:
+                    return response
+            return None
+
+        rounds = (prime_then_reread() for _ in range(CACHE_ATTEMPTS))
+        measured = next((response for response in rounds if response is not None), None)
+        if measured is None:
             pytest.fail(
-                f"no cache read landed across {CACHE_ATTEMPTS} prime+measure rounds; "
-                "the cache-read cost header was never exercised with a nonzero value"
+                f"no cache read landed across {CACHE_ATTEMPTS} prime rounds of "
+                f"{CACHE_REREADS} re-reads each; the cache-read cost header was never "
+                "exercised with a nonzero value"
             )
 
         total = measured.response_cost

tests/e2e/ui/tests/modelsPage/autoRouterTemplateSelect.spec.ts

@@ -0,0 +1,59 @@
+import { expect, test, type Page as PlaywrightPage } from "@playwright/test";
+import { ADMIN_STORAGE_PATH } from "../../constants";
+import { navigateToPage } from "../../helpers/navigation";
+import { Page } from "../../fixtures/pages";
+
+/**
+ * Opens Add Auto Router and returns the Template select's trigger, which is the
+ * shallowest real page that renders SelectContent with tall multi-line options.
+ */
+async function openTemplateSelect(page: PlaywrightPage) {
+  await navigateToPage(page, Page.Models);
+  await page.getByRole("tab", { name: "Auto-Routers" }).click();
+  await page.getByRole("button", { name: "Add Auto Router" }).click();
+
+  const trigger = page.getByTestId("template-selector");
+  await expect(trigger).toBeVisible();
+  return trigger;
+}
+
+test.describe("Auto Router template select anchoring", () => {
+  test.use({ storageState: ADMIN_STORAGE_PATH });
+
+  test("opens the options below the trigger rather than over it", async ({ page }) => {
+    await page.setViewportSize({ width: 1280, height: 900 });
+    const trigger = await openTemplateSelect(page);
+    const triggerBox = await trigger.boundingBox();
+
+    await trigger.click();
+    const popup = page.locator('[data-slot="select-content"]');
+    await expect(popup).toBeVisible();
+    const popupBox = await popup.boundingBox();
+
+    expect(triggerBox).not.toBeNull();
+    expect(popupBox).not.toBeNull();
+
+    // Item-aligned mode reports "none" and puts the active item over the trigger.
+    await expect(popup).toHaveAttribute("data-side", "bottom");
+    expect(popupBox!.y).toBeGreaterThanOrEqual(triggerBox!.y + triggerBox!.height);
+  });
+
+  test("flips above the trigger instead of covering it when there is no room below", async ({ page }) => {
+    await page.setViewportSize({ width: 1280, height: 560 });
+    const trigger = await openTemplateSelect(page);
+    await trigger.scrollIntoViewIfNeeded();
+    const triggerBox = await trigger.boundingBox();
+
+    await trigger.click();
+    const popup = page.locator('[data-slot="select-content"]');
+    await expect(popup).toBeVisible();
+    const popupBox = await popup.boundingBox();
+
+    expect(triggerBox).not.toBeNull();
+    expect(popupBox).not.toBeNull();
+
+    const overlaps =
+      popupBox!.y < triggerBox!.y + triggerBox!.height && popupBox!.y + popupBox!.height > triggerBox!.y;
+    expect(overlaps).toBe(false);
+  });
+});

tests/e2e/ui/tests/settings/routerSettings.spec.ts

@@ -117,6 +117,11 @@ const ADMIN_AUTH = {
   Authorization: `Bearer ${users[Role.ProxyAdmin].password}`,
 };
 
+// Five probes 2s apart outlast the e2e stack's proxy_config_reload_interval_seconds of 7.
+const SETTLE_INTERVAL_MS = 2_000;
+const SETTLE_PROBES = 5;
+const SETTLE_TIMEOUT_MS = 60_000;
+
 /**
  * Apply a router_settings patch through the typed /config/update contract. The
  * server merges it over existing settings (request wins), so only the passed keys
@@ -133,6 +138,21 @@ async function patchRouterSettings(
   expect(res.ok(), `seed /config/update failed: ${res.status()} ${await res.text()}`).toBeTruthy();
 }
 
+/**
+ * Spreads its samples across more than one reload cycle: a single reply only proves the one
+ * replica that served it has reloaded, not the sibling still on the pre-update config.
+ */
+async function sampleStatuses(probe: () => Promise<number>): Promise<readonly number[]> {
+  return Array.from({ length: SETTLE_PROBES }).reduce<Promise<readonly number[]>>(
+    async (taken, _unused, index) => {
+      const sofar = await taken;
+      if (index > 0) await new Promise((resolve) => setTimeout(resolve, SETTLE_INTERVAL_MS));
+      return [...sofar, await probe()];
+    },
+    Promise.resolve([]),
+  );
+}
+
 test.describe("Router Settings - Loadbalancing", () => {
   test.use({ storageState: ADMIN_STORAGE_PATH });
 
@@ -252,28 +272,34 @@ test.describe("Router Settings - Fallbacks serve the request", () => {
   });
 
   test("a request to an unreachable model is answered by its fallback", async ({ page, request }) => {
-    const chat = async () =>
-      request.post("/v1/chat/completions", {
-        headers: { ...ADMIN_AUTH, "Content-Type": "application/json" },
-        data: {
-          model: BROKEN_PRIMARY,
-          messages: [{ role: "user", content: "fallback probe" }],
-        },
-      });
+    const chatStatus = async () =>
+      (
+        await request.post("/v1/chat/completions", {
+          headers: { ...ADMIN_AUTH, "Content-Type": "application/json" },
+          data: {
+            model: BROKEN_PRIMARY,
+            messages: [{ role: "user", content: "fallback probe" }],
+          },
+        })
+      ).status();
 
-    // The control: it proves the reply below could only have come from the fallback.
-    expect((await chat()).status(), "broken primary unexpectedly succeeded on its own").toBeGreaterThanOrEqual(400);
+    // The control: every replica must reject, or the reply below could have come from one
+    // that was still serving a fallback left behind by an earlier attempt.
+    await expect
+      .poll(async () => (await sampleStatuses(chatStatus)).every((status) => status >= 400), {
+        timeout: SETTLE_TIMEOUT_MS,
+        message: "broken primary unexpectedly succeeded on its own",
+      })
+      .toBe(true);
 
     await patchRouterSettings(request, {
       fallbacks: [{ [BROKEN_PRIMARY]: [PRIMARY] }],
     } as Partial<NonNullable<ConfigYAML["router_settings"]>>);
 
-    // Same call now succeeds, served by the fallback model.
+    // One success is the whole claim here, so this waits for a first sighting rather than
+    // for every replica: demanding a streak would also assert a fallback hit rate.
     await expect
-      .poll(async () => (await chat()).status(), {
-        timeout: 30_000,
-        message: "fallback never took effect",
-      })
+      .poll(chatStatus, { timeout: SETTLE_TIMEOUT_MS, message: "fallback never took effect" })
       .toBe(200);
 
     // And the playground renders a reply for a model whose own upstream is down.

tests/litellm-proxy-extras/test_litellm_proxy_extras_utils.py

@@ -12,7 +12,11 @@ sys.path.insert(
     ),
 )
 
-from litellm_proxy_extras.utils import ProxyExtrasDBManager
+from litellm_proxy_extras.utils import (
+    PARTITIONED_SPEND_LOGS_PUSH_ERROR,
+    ProxyExtrasDBManager,
+    filter_partitioned_spend_logs_diff,
+)
 
 # Path to the migrations directory
 _MIGRATIONS_DIR = os.path.abspath(
@@ -475,3 +479,205 @@ class TestMigrationGuardScope:
             if not self._run_rules([(TestMigrationGuardScope._NEW, by_name[name])])
         ]
         assert not redundant, f"these no longer violate and should be removed: {redundant}"
+
+
+_PARTITIONED_DRIFT_SQL = """-- AlterTable
+ALTER TABLE "LiteLLM_BudgetTable" ADD COLUMN     "updated_by" TEXT;
+
+-- AlterTable
+ALTER TABLE "LiteLLM_SpendLogs" DROP CONSTRAINT "LiteLLM_SpendLogs_pkey",
+ADD COLUMN     "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+ADD COLUMN     "updated_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+ADD CONSTRAINT "LiteLLM_SpendLogs_pkey" PRIMARY KEY ("request_id");
+
+-- DropTable
+DROP TABLE "LiteLLM_SpendLogs_legacy";
+"""
+
+
+class TestPartitionedSpendLogsDriftFilter:
+    """A doc-partitioned LiteLLM_SpendLogs (db_scripts/partition_spend_logs.sql) has a
+    composite primary key that schema.prisma cannot express, so `prisma migrate diff`
+    emits a primary-key rewrite that Postgres rejects, aborting the whole drift script
+    before its legitimate statements run."""
+
+    def test_pk_rewrite_and_runbook_artifact_drops_are_removed(self):
+        filtered = filter_partitioned_spend_logs_diff(_PARTITIONED_DRIFT_SQL)
+        assert 'DROP CONSTRAINT "LiteLLM_SpendLogs_pkey"' not in filtered
+        assert 'PRIMARY KEY ("request_id")' not in filtered
+        assert "LiteLLM_SpendLogs_legacy" not in filtered
+
+    def test_legitimate_statements_in_the_same_script_are_kept(self):
+        filtered = filter_partitioned_spend_logs_diff(_PARTITIONED_DRIFT_SQL)
+        assert 'ALTER TABLE "LiteLLM_BudgetTable" ADD COLUMN     "updated_by" TEXT;' in filtered
+        assert 'ADD COLUMN     "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP' in filtered
+        assert 'ADD COLUMN     "updated_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP' in filtered
+        assert filtered.count('ALTER TABLE "LiteLLM_SpendLogs"') == 1
+
+    def test_an_alter_containing_only_the_pk_rewrite_is_dropped_entirely(self):
+        sql = (
+            'ALTER TABLE "LiteLLM_SpendLogs" DROP CONSTRAINT "LiteLLM_SpendLogs_pkey",\n'
+            'ADD CONSTRAINT "LiteLLM_SpendLogs_pkey" PRIMARY KEY ("request_id");\n'
+        )
+        assert filter_partitioned_spend_logs_diff(sql).strip() == ""
+
+    def test_other_tables_pk_changes_are_untouched(self):
+        sql = (
+            'ALTER TABLE "LiteLLM_TeamTable" DROP CONSTRAINT "LiteLLM_TeamTable_pkey",\n'
+            'ADD CONSTRAINT "LiteLLM_TeamTable_pkey" PRIMARY KEY ("team_id");\n'
+        )
+        filtered = filter_partitioned_spend_logs_diff(sql)
+        assert 'DROP CONSTRAINT "LiteLLM_TeamTable_pkey"' in filtered
+        assert 'PRIMARY KEY ("team_id")' in filtered
+
+
+class _FakeCompleted:
+    stdout = ""
+    stderr = ""
+
+
+class TestResolveAllMigrationsLedger:
+    def _run(self, monkeypatch, tmp_path, partitioned, execute_fails):
+        import subprocess as subprocess_module
+
+        import litellm_proxy_extras.utils as utils_module
+
+        monkeypatch.setenv("DATABASE_URL", "postgresql://u:p@localhost:5432/db")
+        monkeypatch.delenv("DIRECT_URL", raising=False)
+        monkeypatch.setattr(
+            ProxyExtrasDBManager, "spend_logs_is_partitioned", staticmethod(lambda: partitioned)
+        )
+        monkeypatch.setattr(
+            ProxyExtrasDBManager,
+            "_get_migration_names",
+            staticmethod(lambda migrations_dir: ["20250326162113_baseline"]),
+        )
+        calls = []
+
+        def fake_run(cmd, **kwargs):
+            calls.append(cmd)
+            if "diff" in cmd:
+                kwargs["stdout"].write(_PARTITIONED_DRIFT_SQL)
+                return _FakeCompleted()
+            if "execute" in cmd:
+                executed_sql = open(cmd[cmd.index("--file") + 1]).read()
+                calls.append(("executed_sql", executed_sql))
+                if execute_fails:
+                    raise subprocess_module.CalledProcessError(1, cmd, stderr="boom")
+                return _FakeCompleted()
+            return _FakeCompleted()
+
+        monkeypatch.setattr(utils_module.subprocess, "run", fake_run)
+        ProxyExtrasDBManager._resolve_all_migrations(str(tmp_path), "schema.prisma")
+        return calls
+
+    def _resolved(self, calls):
+        return [c for c in calls if isinstance(c, list) and "resolve" in c]
+
+    def _executed_sql(self, calls):
+        return next(c[1] for c in calls if isinstance(c, tuple) and c[0] == "executed_sql")
+
+    def test_failed_drift_apply_does_not_mark_migrations_applied(self, monkeypatch, tmp_path):
+        calls = self._run(monkeypatch, tmp_path, partitioned=False, execute_fails=True)
+        assert self._resolved(calls) == []
+
+    def test_successful_drift_apply_still_marks_migrations_applied(self, monkeypatch, tmp_path):
+        calls = self._run(monkeypatch, tmp_path, partitioned=False, execute_fails=False)
+        assert len(self._resolved(calls)) == 1
+
+    def test_partitioned_spend_logs_gets_the_filtered_drift_script(self, monkeypatch, tmp_path):
+        calls = self._run(monkeypatch, tmp_path, partitioned=True, execute_fails=False)
+        executed_sql = self._executed_sql(calls)
+        assert 'PRIMARY KEY ("request_id")' not in executed_sql
+        assert "LiteLLM_SpendLogs_legacy" not in executed_sql
+        assert 'ALTER TABLE "LiteLLM_BudgetTable" ADD COLUMN     "updated_by" TEXT;' in executed_sql
+        assert 'ADD COLUMN     "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP' in executed_sql
+        assert len(self._resolved(calls)) == 1
+
+    def test_unpartitioned_spend_logs_drift_script_is_untouched(self, monkeypatch, tmp_path):
+        calls = self._run(monkeypatch, tmp_path, partitioned=False, execute_fails=False)
+        assert self._executed_sql(calls) == _PARTITIONED_DRIFT_SQL
+
+
+class TestPartitionedSpendLogsPushGuard:
+    def _forbid_subprocess(self, monkeypatch):
+        import litellm_proxy_extras.utils as utils_module
+
+        def fail_run(cmd, **kwargs):
+            raise AssertionError(f"subprocess.run should not be called, got: {cmd}")
+
+        monkeypatch.setattr(utils_module.subprocess, "run", fail_run)
+
+    def test_v1_db_push_fails_fast_with_guidance(self, monkeypatch):
+        monkeypatch.setattr(
+            ProxyExtrasDBManager, "spend_logs_is_partitioned", staticmethod(lambda: True)
+        )
+        self._forbid_subprocess(monkeypatch)
+        with pytest.raises(RuntimeError) as err:
+            ProxyExtrasDBManager._run_migrations(use_migrate=False, use_v2_resolver=False)
+        assert str(err.value) == PARTITIONED_SPEND_LOGS_PUSH_ERROR
+
+    def test_v2_db_push_fails_fast_with_guidance(self, monkeypatch):
+        monkeypatch.setattr(
+            ProxyExtrasDBManager, "spend_logs_is_partitioned", staticmethod(lambda: True)
+        )
+        self._forbid_subprocess(monkeypatch)
+        with pytest.raises(RuntimeError) as err:
+            ProxyExtrasDBManager._setup_database_v2(use_migrate=False)
+        assert str(err.value) == PARTITIONED_SPEND_LOGS_PUSH_ERROR
+
+
+class _FakeCursor:
+    def fetchone(self):
+        return (1,)
+
+
+class _FakePsycopgConn:
+    def __init__(self, executed):
+        self._executed = executed
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, *args):
+        return False
+
+    def execute(self, query, params):
+        self._executed.append((query, params))
+        return _FakeCursor()
+
+
+class TestSpendLogsPartitionDetectionSchemaScope:
+    """A same-named LiteLLM_SpendLogs in another schema must not trip the
+    detector: the catalog lookup has to be scoped to Prisma's target schema."""
+
+    def _detect(self, monkeypatch, database_url):
+        import sys
+        import types
+
+        executed = []
+        fake_psycopg = types.ModuleType("psycopg")
+        fake_psycopg.connect = lambda url, **kwargs: _FakePsycopgConn(executed)
+        fake_psycopg.OperationalError = type("OperationalError", (Exception,), {})
+        fake_psycopg.DatabaseError = type("DatabaseError", (Exception,), {})
+        monkeypatch.setitem(sys.modules, "psycopg", fake_psycopg)
+        monkeypatch.setenv("DATABASE_URL", database_url)
+        assert ProxyExtrasDBManager.spend_logs_is_partitioned() is True
+        return executed[0]
+
+    def test_lookup_is_scoped_to_the_schema_url_param(self, monkeypatch):
+        query, params = self._detect(
+            monkeypatch, "postgresql://u:p@localhost:5432/db?schema=tenant_a"
+        )
+        assert "pg_namespace" in query
+        assert "n.nspname = %s" in query
+        assert params == ("tenant_a",)
+
+    def test_lookup_falls_back_to_public_without_a_schema_param(self, monkeypatch):
+        query, params = self._detect(monkeypatch, "postgresql://u:p@localhost:5432/db")
+        assert "n.nspname = %s" in query
+        assert params == ("public",)
+
+    def test_only_partitioned_relations_match(self, monkeypatch):
+        query, _ = self._detect(monkeypatch, "postgresql://u:p@localhost:5432/db")
+        assert "pg_partitioned_table" in query

tests/mcp_tests/conftest.py

@@ -45,6 +45,22 @@ def setup_and_teardown():
     asyncio.set_event_loop(None)  # Remove the reference to the loop
 
 
+@pytest.fixture(scope="function", autouse=True)
+async def drain_logging_worker():
+    """
+    The logging queue is bound to the running loop, so anything left queued when a test's loop
+    goes away is carried onto the next test's loop and fires against its callbacks.
+    """
+    from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER
+
+    yield
+
+    try:
+        await asyncio.wait_for(GLOBAL_LOGGING_WORKER.clear_queue(), timeout=10)
+    except asyncio.TimeoutError:
+        pass
+
+
 def pytest_collection_modifyitems(config, items):
     # Separate tests in 'test_amazing_proxy_custom_logger.py' and other tests
     custom_logger_tests = [

tests/proxy_unit_tests/test_proxy_server.py

@@ -2661,7 +2661,7 @@ async def test_run_direct_health_check_drops_only_the_rejected_kwarg(monkeypatch
     rejected argument alongside working ones would probe deployments the operator opted out."""
     import litellm.proxy.proxy_server as proxy_server
 
-    seen: list = []
+    seen: list[tuple[dict[str, str] | None, bool]] = []
 
     async def fake_perform_health_check(
         model_list,

tests/test_litellm/conftest.py

@@ -375,6 +375,9 @@ def isolate_litellm_state():
         litellm.in_memory_llm_clients_cache.flush_cache()
     image_handling_module.in_memory_cache.flush_cache()
     _reset_module_level_aws_auth_caches()
+    # litellm.get_model_info() memoizes ModelInfo built from litellm.model_cost, so a
+    # test that rebinds the cost map leaves later tests pricing against the old map.
+    litellm_utils_module._invalidate_model_cost_lowercase_map()
 
     # Clear all callback lists to prevent cross-test contamination
     if hasattr(litellm, "callbacks"):
@@ -418,6 +421,7 @@ def isolate_litellm_state():
 
     litellm_utils_module._runtime_registered_model_cost.clear()
     litellm_utils_module._runtime_registered_model_cost.update(original_runtime_registered_model_cost)
+    litellm_utils_module._invalidate_model_cost_lowercase_map()
 
     for _router in tuple(litellm_router_module._live_routers):
         litellm_router_module._live_routers.discard(_router)

tests/test_litellm/integrations/newrelic/test_newrelic_metrics.py

@@ -0,0 +1,825 @@
+"""
+Batching tests for NewRelicMetricsLogger: flush-window interval computation,
+dimension-bucket aggregation, the 4xx-drop vs 5xx/network-requeue policy, the
+retry-queue cap, and the stop flag that ends the periodic flush loop.
+"""
+
+import asyncio
+import gzip
+import json
+from unittest.mock import AsyncMock, patch
+
+import pytest
+from httpx import HTTPStatusError, Request, Response
+
+from litellm.integrations.newrelic.newrelic_metrics import (
+    NewRelicMetricsLogger,
+    _bucket_metrics,
+    build_metric_payload,
+)
+from litellm.types.integrations.newrelic import (
+    NEWRELIC_METRIC_COMPLETION_TOKENS,
+    NEWRELIC_METRIC_COST_USD,
+    NEWRELIC_METRIC_ENDPOINT_BY_REGION,
+    NEWRELIC_METRIC_PROMPT_TOKENS,
+    NEWRELIC_METRIC_REQUEST_DURATION_MS,
+    NEWRELIC_METRIC_REQUESTS,
+    NEWRELIC_METRIC_TOTAL_TOKENS,
+    NewRelicMetricRecord,
+)
+
+
+def _record(
+    team_id="team-a",
+    team_alias=None,
+    model="gpt-4o",
+    model_group=None,
+    status="success",
+    response_cost=0.5,
+    prompt_tokens=10,
+    completion_tokens=20,
+    total_tokens=30,
+    duration_ms=100.0,
+) -> NewRelicMetricRecord:
+    return NewRelicMetricRecord(
+        team_id=team_id,
+        team_alias=team_alias if team_alias is not None else f"{team_id}-alias",
+        model_group=model_group if model_group is not None else f"{model}-group",
+        model=model,
+        custom_llm_provider="openai",
+        status=status,
+        response_cost=response_cost,
+        prompt_tokens=prompt_tokens,
+        completion_tokens=completion_tokens,
+        total_tokens=total_tokens,
+        duration_ms=duration_ms,
+    )
+
+
+def _standard_logging_object(team_id="team-a", response_cost=0.25) -> dict:
+    return {
+        "metadata": {"user_api_key_team_id": team_id, "user_api_key_team_alias": f"{team_id}-alias"},
+        "model_group": "gpt-4o-group",
+        "model": "gpt-4o",
+        "custom_llm_provider": "openai",
+        "status": "success",
+        "response_cost": response_cost,
+        "prompt_tokens": 10,
+        "completion_tokens": 20,
+        "total_tokens": 30,
+        "response_time": 0.1,
+    }
+
+
+def _make_logger(**kwargs) -> NewRelicMetricsLogger:
+    with patch("asyncio.create_task"):
+        return NewRelicMetricsLogger(newrelic_api_key="test-key", **kwargs)
+
+
+def _response(status_code: int, text: str = "") -> Response:
+    return Response(status_code, request=Request("POST", "https://example.com"), text=text)
+
+
+def _raises(status_code: int):
+    """Mock the way AsyncHTTPHandler.post really behaves: raise_for_status() turns
+    every non-2xx into an HTTPStatusError rather than returning the response."""
+    resp = _response(status_code)
+    return AsyncMock(side_effect=HTTPStatusError("err", request=resp.request, response=resp))
+
+
+def _metrics_by_name(payload, name):
+    return [m for m in payload[0]["metrics"] if m["name"] == name]
+
+
+class TestBuildMetricPayload:
+    def test_interval_and_timestamp_reflect_flush_window(self):
+        payload = build_metric_payload((_record(),), window_start=1_000.0, now=1_007.5)
+
+        assert payload[0]["common"]["timestamp"] == 1_000_000
+        assert payload[0]["common"]["interval.ms"] == 7_500
+
+    def test_interval_is_at_least_one_ms(self):
+        payload = build_metric_payload((_record(),), window_start=1_000.0, now=1_000.0)
+
+        assert payload[0]["common"]["interval.ms"] == 1
+
+    def test_single_record_metric_values(self):
+        payload = build_metric_payload(
+            (_record(response_cost=0.5, prompt_tokens=10, completion_tokens=20, total_tokens=30, duration_ms=100.0),),
+            window_start=1_000.0,
+            now=1_005.0,
+        )
+
+        by_name = {m["name"]: m for m in payload[0]["metrics"]}
+        assert by_name[NEWRELIC_METRIC_REQUESTS]["value"] == 1.0
+        assert by_name[NEWRELIC_METRIC_REQUESTS]["type"] == "count"
+        assert by_name[NEWRELIC_METRIC_COST_USD]["value"] == 0.5
+        assert by_name[NEWRELIC_METRIC_PROMPT_TOKENS]["value"] == 10.0
+        assert by_name[NEWRELIC_METRIC_COMPLETION_TOKENS]["value"] == 20.0
+        assert by_name[NEWRELIC_METRIC_TOTAL_TOKENS]["value"] == 30.0
+        duration = by_name[NEWRELIC_METRIC_REQUEST_DURATION_MS]
+        assert duration["type"] == "summary"
+        assert duration["value"] == {"count": 1, "sum": 100.0, "min": 100.0, "max": 100.0}
+        assert by_name[NEWRELIC_METRIC_REQUESTS]["attributes"] == {
+            "team_id": "team-a",
+            "team_alias": "team-a-alias",
+            "model_group": "gpt-4o-group",
+            "model": "gpt-4o",
+            "custom_llm_provider": "openai",
+            "status": "success",
+        }
+
+    def test_aggregates_across_dimension_buckets(self):
+        """Two teams x two models in one queue land in the right bucket sums.
+
+        team_alias and model_group are held constant so bucketing provably keys on
+        team_id and model themselves, not on correlated fields.
+        """
+        shared = {"team_alias": "shared-alias", "model_group": "shared-group"}
+        records = (
+            _record(team_id="team-a", model="gpt-4o", response_cost=0.1, total_tokens=10, duration_ms=50.0, **shared),
+            _record(team_id="team-a", model="gpt-4o", response_cost=0.2, total_tokens=20, duration_ms=150.0, **shared),
+            _record(
+                team_id="team-a", model="claude-4", response_cost=0.4, total_tokens=40, duration_ms=200.0, **shared
+            ),
+            _record(team_id="team-b", model="gpt-4o", response_cost=0.8, total_tokens=80, duration_ms=300.0, **shared),
+        )
+        payload = build_metric_payload(records, window_start=1_000.0, now=1_005.0)
+
+        cost_by_bucket = {
+            (m["attributes"]["team_id"], m["attributes"]["model"]): m["value"]
+            for m in _metrics_by_name(payload, NEWRELIC_METRIC_COST_USD)
+        }
+        assert cost_by_bucket == {
+            ("team-a", "gpt-4o"): pytest.approx(0.3),
+            ("team-a", "claude-4"): pytest.approx(0.4),
+            ("team-b", "gpt-4o"): pytest.approx(0.8),
+        }
+
+        requests_by_bucket = {
+            (m["attributes"]["team_id"], m["attributes"]["model"]): m["value"]
+            for m in _metrics_by_name(payload, NEWRELIC_METRIC_REQUESTS)
+        }
+        assert requests_by_bucket == {
+            ("team-a", "gpt-4o"): 2.0,
+            ("team-a", "claude-4"): 1.0,
+            ("team-b", "gpt-4o"): 1.0,
+        }
+
+        duration_by_bucket = {
+            (m["attributes"]["team_id"], m["attributes"]["model"]): m["value"]
+            for m in _metrics_by_name(payload, NEWRELIC_METRIC_REQUEST_DURATION_MS)
+        }
+        assert duration_by_bucket[("team-a", "gpt-4o")] == {"count": 2, "sum": 200.0, "min": 50.0, "max": 150.0}
+
+    def test_status_is_a_bucket_dimension(self):
+        records = (
+            _record(status="success", response_cost=0.1),
+            _record(status="failure", response_cost=0.0),
+        )
+        payload = build_metric_payload(records, window_start=1_000.0, now=1_005.0)
+
+        statuses = {m["attributes"]["status"] for m in _metrics_by_name(payload, NEWRELIC_METRIC_REQUESTS)}
+        assert statuses == {"success", "failure"}
+
+    def test_empty_attribute_values_are_omitted(self):
+        record = NewRelicMetricRecord(
+            team_id="",
+            team_alias="",
+            model_group="",
+            model="gpt-4o",
+            custom_llm_provider="openai",
+            status="success",
+            response_cost=0.0,
+            prompt_tokens=0,
+            completion_tokens=0,
+            total_tokens=0,
+            duration_ms=0.0,
+        )
+        payload = build_metric_payload((record,), window_start=1_000.0, now=1_005.0)
+
+        attributes = payload[0]["metrics"][0]["attributes"]
+        assert "team_id" not in attributes
+        assert "team_alias" not in attributes
+        assert "model_group" not in attributes
+
+
+class TestQueueAndFlush:
+    @pytest.mark.asyncio
+    async def test_log_event_queues_record_from_standard_logging_object(self):
+        logger = _make_logger()
+
+        await logger.async_log_success_event(
+            kwargs={"standard_logging_object": _standard_logging_object()},
+            response_obj={},
+            start_time=None,
+            end_time=None,
+        )
+
+        assert len(logger.log_queue) == 1
+        record = logger.log_queue[0]
+        assert record.team_id == "team-a"
+        assert record.response_cost == 0.25
+        assert record.duration_ms == pytest.approx(100.0)
+
+    @pytest.mark.asyncio
+    async def test_failure_event_queues_record(self):
+        logger = _make_logger()
+
+        slo = _standard_logging_object()
+        slo["status"] = "failure"
+        await logger.async_log_failure_event(
+            kwargs={"standard_logging_object": slo},
+            response_obj={},
+            start_time=None,
+            end_time=None,
+        )
+
+        assert len(logger.log_queue) == 1
+        assert logger.log_queue[0].status == "failure"
+
+    @pytest.mark.asyncio
+    async def test_threshold_flush_uses_flush_queue(self):
+        logger = _make_logger()
+        logger.batch_size = 1
+        logger.flush_queue = AsyncMock()
+
+        await logger.async_log_success_event(
+            kwargs={"standard_logging_object": _standard_logging_object()},
+            response_obj={},
+            start_time=None,
+            end_time=None,
+        )
+
+        logger.flush_queue.assert_awaited_once()
+
+    @pytest.mark.asyncio
+    async def test_flush_queue_updates_last_flush_time_on_success(self):
+        logger = _make_logger()
+        logger.log_queue = [_record()]
+        logger.last_flush_time = 0
+        logger.async_client.post = AsyncMock(return_value=_response(202))
+
+        await logger.flush_queue()
+
+        assert logger.log_queue == []
+        assert logger.last_flush_time > 0
+
+    @pytest.mark.asyncio
+    async def test_flush_advances_window_even_on_requeue(self):
+        # The window start advances every flush cycle so requeued records report
+        # in the next window instead of freezing interval.ms under sustained
+        # failure, and an idle gap never inflates the next batch's window
+        logger = _make_logger()
+        logger.log_queue = [_record()]
+        logger.last_flush_time = 123.0
+        logger.async_client.post = _raises(500)
+
+        await logger.flush_queue()
+
+        assert logger.last_flush_time > 123.0
+        assert len(logger.log_queue) == 1
+
+    @pytest.mark.asyncio
+    async def test_sent_payload_window_starts_at_last_flush_time(self):
+        logger = _make_logger()
+        logger.log_queue = [_record()]
+        logger.last_flush_time = 2_000.0
+        logger.async_client.post = AsyncMock(return_value=_response(202))
+
+        with patch("litellm.integrations.newrelic.newrelic_metrics.time.time", return_value=2_010.0):
+            await logger.async_send_batch()
+
+        sent = logger.async_client.post.await_args.kwargs
+        body = json.loads(gzip.decompress(sent["data"]).decode("utf-8"))
+        assert body[0]["common"]["timestamp"] == 2_000_000
+        assert body[0]["common"]["interval.ms"] == 10_000
+        assert sent["headers"]["Api-Key"] == "test-key"
+        assert sent["headers"]["Content-Encoding"] == "gzip"
+        assert sent["url"] == NEWRELIC_METRIC_ENDPOINT_BY_REGION["us"]
+
+
+class TestBatchSizeCap:
+    @pytest.mark.asyncio
+    async def test_flush_sends_at_most_batch_size_records_per_request(self):
+        """A queue grown past the batch size by requeues must go out in chunks:
+        one oversized request would breach the Metric API data point cap and get
+        the whole retry backlog dropped as a 4xx."""
+        logger = _make_logger()
+        logger.batch_size = 2
+        logger.log_queue = [_record(model=f"model-{i}") for i in range(5)]
+        logger.async_client.post = AsyncMock(return_value=_response(202))
+
+        await logger.flush_queue()
+
+        sent_counts = [
+            sum(
+                metric["value"]
+                for metric in json.loads(gzip.decompress(call.kwargs["data"]).decode("utf-8"))[0]["metrics"]
+                if metric["name"] == NEWRELIC_METRIC_REQUESTS
+            )
+            for call in logger.async_client.post.await_args_list
+        ]
+        assert sent_counts == [2.0, 2.0, 1.0]
+        assert logger.log_queue == []
+
+    @pytest.mark.asyncio
+    async def test_failed_chunk_stops_the_flush_and_keeps_order(self):
+        """A 5xx on the first chunk ends the flush instead of hammering the same
+        failing endpoint with the rest of the backlog, and the requeue keeps the
+        records in chronological order."""
+        logger = _make_logger()
+        logger.batch_size = 2
+        records = [_record(model=f"model-{i}") for i in range(5)]
+        logger.log_queue = list(records)
+        logger.async_client.post = _raises(500)
+
+        await logger.flush_queue()
+
+        assert logger.async_client.post.await_count == 1
+        assert logger.log_queue == records
+
+
+class TestFlushConcurrency:
+    @pytest.mark.asyncio
+    async def test_records_appended_during_flush_await_survive(self):
+        """A record appended by a concurrent request while the POST is in flight
+        must survive the flush, not be clobbered by a queue replacement."""
+        logger = _make_logger()
+        logger.log_queue = [_record(team_id="team-a")]
+        interleaved = _record(team_id="team-interleaved")
+
+        async def _post_appending_mid_flight(**kwargs):
+            logger.log_queue.append(interleaved)
+            return _response(202)
+
+        logger.async_client.post = AsyncMock(side_effect=_post_appending_mid_flight)
+
+        await logger.async_send_batch()
+
+        assert logger.log_queue == [interleaved]
+        body = json.loads(gzip.decompress(logger.async_client.post.await_args.kwargs["data"]).decode("utf-8"))
+        team_ids = {m["attributes"]["team_id"] for m in body[0]["metrics"]}
+        assert team_ids == {"team-a"}
+
+    @pytest.mark.asyncio
+    async def test_records_appended_during_failed_flush_await_survive_requeue(self):
+        """The requeue path must also preserve interleaved records: batch is
+        prepended in place, never assigned over the live queue."""
+        logger = _make_logger()
+        original = _record(team_id="team-a")
+        logger.log_queue = [original]
+        interleaved = _record(team_id="team-interleaved")
+
+        async def _post_appending_mid_flight(**kwargs):
+            logger.log_queue.append(interleaved)
+            raise HTTPStatusError('e', request=_response(500).request, response=_response(500))
+
+        logger.async_client.post = AsyncMock(side_effect=_post_appending_mid_flight)
+
+        await logger.async_send_batch()
+
+        assert logger.log_queue == [original, interleaved]
+
+
+class TestErrorPolicy:
+    @pytest.mark.asyncio
+    async def test_4xx_drops_batch(self):
+        logger = _make_logger()
+        logger.log_queue = [_record(), _record(team_id="team-b")]
+        logger.async_client.post = AsyncMock(return_value=_response(400, text="bad request"))
+
+        await logger.async_send_batch()
+
+        assert logger.log_queue == []
+        assert logger.async_client.post.await_count == 1
+
+    @pytest.mark.asyncio
+    async def test_403_drops_batch_and_names_permanent_credential_failure(self):
+        logger = _make_logger()
+        logger.log_queue = [_record()]
+        logger.async_client.post = _raises(403)
+
+        with patch("litellm.integrations.newrelic.newrelic_metrics.verbose_logger") as mock_logger:
+            await logger.async_send_batch()
+
+        assert logger.log_queue == []
+        warning_text = " ".join(str(arg) for call in mock_logger.warning.call_args_list for arg in call.args)
+        assert "permanent credential failure" in warning_text
+
+    @pytest.mark.asyncio
+    async def test_5xx_requeues_batch(self):
+        records = [_record(), _record(team_id="team-b")]
+        logger = _make_logger()
+        logger.log_queue = list(records)
+        logger.async_client.post = _raises(500)
+
+        await logger.async_send_batch()
+
+        assert logger.log_queue == records
+
+    @pytest.mark.asyncio
+    async def test_network_error_requeues_batch(self):
+        records = [_record()]
+        logger = _make_logger()
+        logger.log_queue = list(records)
+        logger.async_client.post = AsyncMock(side_effect=ConnectionError("boom"))
+
+        await logger.async_send_batch()
+
+        assert logger.log_queue == records
+
+    @pytest.mark.asyncio
+    async def test_requeue_is_capped_dropping_oldest(self):
+        logger = _make_logger()
+        logger.max_queue_size = 3
+        oldest = _record(team_id="oldest")
+        rest = [_record(team_id=f"team-{i}") for i in range(3)]
+        logger.log_queue = [oldest, *rest]
+        logger.async_client.post = _raises(500)
+
+        await logger.async_send_batch()
+
+        assert logger.log_queue == rest
+
+    @pytest.mark.asyncio
+    async def test_requeued_records_are_resent_with_new_records(self):
+        logger = _make_logger()
+        logger.log_queue = [_record()]
+        logger.async_client.post = _raises(500)
+
+        await logger.async_send_batch()
+        logger.log_queue.append(_record(team_id="team-b"))
+        logger.async_client.post = AsyncMock(return_value=_response(202))
+
+        await logger.async_send_batch()
+
+        sent = logger.async_client.post.await_args.kwargs
+        body = json.loads(gzip.decompress(sent["data"]).decode("utf-8"))
+        team_ids = {m["attributes"]["team_id"] for m in body[0]["metrics"]}
+        assert team_ids == {"team-a", "team-b"}
+        assert logger.log_queue == []
+
+
+class TestStopFlag:
+    @pytest.mark.asyncio
+    async def test_stop_ends_periodic_flush_loop(self):
+        logger = _make_logger()
+        logger.flush_interval = 0.01
+        logger.flush_queue = AsyncMock()
+
+        task = asyncio.create_task(logger.periodic_flush())
+        await asyncio.sleep(0.05)
+        assert not task.done()
+
+        logger.stop()
+        await asyncio.wait_for(task, timeout=1.0)
+
+        assert task.done()
+
+    @pytest.mark.asyncio
+    async def test_stopped_logger_exits_after_one_final_drain(self):
+        logger = _make_logger()
+        logger.flush_interval = 0.01
+        logger._final_drain = AsyncMock()
+        logger._stopped = True
+
+        await asyncio.wait_for(logger.periodic_flush(), timeout=1.0)
+
+        logger._final_drain.assert_awaited_once()
+
+    @pytest.mark.asyncio
+    async def test_eviction_drains_queued_records(self):
+        """Eviction must post what is already queued, not silently discard it."""
+        from litellm.litellm_core_utils.specialty_caches.dynamic_logging_cache import (
+            DynamicLoggingCache,
+        )
+
+        cache = DynamicLoggingCache()
+        logger = _make_logger()
+        logger.log_queue = [_record(), _record(team_id="team-b")]
+        logger.async_client.post = AsyncMock(return_value=_response(202))
+        credentials = {"newrelic_api_key": "test-key", "newrelic_region": None}
+        cache.set_cache(credentials=credentials, service_name="newrelic", logging_obj=logger)
+
+        key = cache.get_cache_key(args={**credentials, "service_name": "newrelic"})
+        cache.cache._remove_key(key)
+        for _ in range(10):
+            await asyncio.sleep(0)
+
+        logger.async_client.post.assert_awaited_once()
+        body = json.loads(gzip.decompress(logger.async_client.post.await_args.kwargs["data"]).decode("utf-8"))
+        team_ids = {m["attributes"]["team_id"] for m in body[0]["metrics"]}
+        assert team_ids == {"team-a", "team-b"}
+        assert logger.log_queue == []
+
+    @pytest.mark.asyncio
+    async def test_dynamic_logging_cache_eviction_calls_stop(self):
+        from litellm.litellm_core_utils.specialty_caches.dynamic_logging_cache import (
+            DynamicLoggingCache,
+        )
+
+        cache = DynamicLoggingCache()
+        logger = _make_logger()
+        credentials = {"newrelic_api_key": "test-key", "newrelic_region": None}
+        cache.set_cache(credentials=credentials, service_name="newrelic", logging_obj=logger)
+
+        key = cache.get_cache_key(args={**credentials, "service_name": "newrelic"})
+        cache.cache._remove_key(key)
+
+        assert logger._stopped is True
+        assert cache.get_cache(credentials=credentials, service_name="newrelic") is None
+
+
+@pytest.mark.asyncio
+async def test_append_after_eviction_drain_self_flushes():
+    """An in-flight callback holding an evicted (stopped) logger still delivers
+    its record: with no periodic loop left, the append itself drains."""
+    logger = _make_logger()
+    with patch.object(
+        logger.async_client, "post", new=AsyncMock(return_value=_response(202))
+    ) as mock_post:
+        logger.stop()
+        await logger.async_log_success_event(
+            {"standard_logging_object": _standard_logging_object()}, None, None, None
+        )
+    assert mock_post.await_count >= 1, "record appended after stop() must be flushed, not stranded"
+    assert logger.log_queue == []
+
+
+@pytest.mark.asyncio
+async def test_final_drain_retries_transient_failure_then_delivers():
+    """A transient 5xx during the eviction drain must not strand the last
+    batch: the final drain retries on its own (no periodic loop is left)."""
+    logger = _make_logger()
+    err = _response(500)
+    responses = [HTTPStatusError('e', request=err.request, response=err), HTTPStatusError('e', request=err.request, response=err), _response(202)]
+    post_mock = AsyncMock(side_effect=responses)
+    with patch.object(logger, "async_client") as client, patch("asyncio.sleep", new=AsyncMock()):
+        client.post = post_mock
+        await logger._log_async_event(standard_logging_object=_standard_logging_object())
+        await logger._final_drain()
+    assert post_mock.await_count == 3
+    assert logger.log_queue == []
+
+
+@pytest.mark.asyncio
+async def test_final_drain_drops_after_bounded_passes_under_lock():
+    """A permanently failing destination is retried across bounded passes, then
+    the remainder is dropped under flush_lock and logged, never stranded. A
+    second drain over the now-empty queue is a no-op."""
+    logger = _make_logger()
+    post_mock = _raises(500)
+    with patch.object(logger, "async_client") as client, patch("asyncio.sleep", new=AsyncMock()):
+        client.post = post_mock
+        await logger._log_async_event(standard_logging_object=_standard_logging_object())
+        await logger._final_drain()
+        after_first = post_mock.await_count
+        await logger._final_drain()
+    assert after_first >= 1, "the failing destination was retried before the drop"
+    assert post_mock.await_count == after_first, "second drain over an empty queue is a no-op"
+    assert logger.log_queue == [], "exhausted retries end in a logged drop, not a stranded queue"
+
+
+def test_attribute_values_bounded_against_payload_bombs():
+    """A caller-controlled high-entropy model string is truncated in metric
+    attributes so one record cannot inflate the shared batch past the Metric
+    API payload cap and take out other users' metrics."""
+    record = _record(model="m" * 5000)
+    metrics = _bucket_metrics((record,))
+    for metric in metrics:
+        assert len(metric["attributes"]["model"]) == 255
+
+
+@pytest.mark.asyncio
+async def test_idle_gap_does_not_inflate_next_window():
+    """Empty flush cycles advance the window start, so a burst after idling
+    reports an interval close to the flush cadence, not the whole idle gap."""
+    logger = _make_logger()
+    logger.last_flush_time = 100.0
+    with patch.object(logger, "async_client") as client:
+        client.post = AsyncMock(return_value=_response(202))
+        await logger.flush_queue()
+        assert logger.last_flush_time > 100.0
+
+
+@pytest.mark.asyncio
+async def test_mid_drain_append_delivered_against_healthy_destination():
+    """A record a callback appends while a drain is running is picked up by a
+    later pass and delivered when the destination is healthy; nothing stranded."""
+    logger = _make_logger()
+    logger.stop()
+    late_record = _record(model="late-model")
+    injected = {"done": False}
+    posted = []
+
+    async def _capture(url, headers=None, content=None, **kw):
+        posted.append(content)
+        if not injected["done"]:
+            injected["done"] = True
+            logger.log_queue.append(late_record)
+        return _response(202)
+
+    with patch.object(logger, "async_client") as client, patch("asyncio.sleep", new=AsyncMock()):
+        client.post = _capture
+        logger.log_queue.append(_record(model="first"))
+        await logger._drain_with_retry()
+    assert logger.log_queue == [], "the mid-drain append was drained too, nothing stranded"
+    assert len(posted) >= 2, "both the original and the mid-drain record were sent"
+
+
+@pytest.mark.asyncio
+async def test_drain_attempts_every_chunk_not_just_the_head_under_failure():
+    """Regression: with more than batch_size records queued on a stopped logger
+    and a persistently failing destination, every record must be attempted before
+    the bounded terminal drop. The periodic path stops at the first failing chunk,
+    so a drain that reused it would drop the un-sent tail (records past the head
+    chunk) as if it had tried them, silently undercounting the team's usage."""
+    logger = _make_logger()
+    logger.stop()
+    logger.batch_size = 2
+    logger.log_queue = [_record(model=f"m{i}") for i in range(5)]
+    sent_models = []
+
+    async def _capture_then_fail(url, data=None, headers=None, **kw):
+        body = json.loads(gzip.decompress(data).decode("utf-8"))
+        sent_models.extend(
+            m["attributes"]["model"] for m in body[0]["metrics"] if m["name"] == NEWRELIC_METRIC_REQUESTS
+        )
+        resp = _response(503)
+        raise HTTPStatusError("err", request=resp.request, response=resp)
+
+    with patch("asyncio.sleep", new=AsyncMock()):
+        logger.async_client.post = _capture_then_fail
+        await logger._drain_with_retry()
+
+    assert set(sent_models) == {"m0", "m1", "m2", "m3", "m4"}, "every chunk, including the tail, was attempted"
+    assert logger.log_queue == [], "the exhausted batch is dropped after bounded passes, nothing stranded"
+
+
+@pytest.mark.asyncio
+async def test_drain_delivers_the_tail_once_the_destination_recovers():
+    """The tail beyond the head chunk must be delivered, not stranded, once a
+    transiently failing destination recovers within the drain's passes."""
+    logger = _make_logger()
+    logger.stop()
+    logger.batch_size = 2
+    logger.log_queue = [_record(model=f"m{i}") for i in range(5)]
+    delivered_models = []
+    posts = {"n": 0}
+
+    async def _fail_first_pass_then_recover(url, data=None, headers=None, **kw):
+        posts["n"] += 1
+        if posts["n"] <= 3:  # the first pass's three chunks all fail
+            resp = _response(503)
+            raise HTTPStatusError("err", request=resp.request, response=resp)
+        body = json.loads(gzip.decompress(data).decode("utf-8"))
+        delivered_models.extend(
+            m["attributes"]["model"] for m in body[0]["metrics"] if m["name"] == NEWRELIC_METRIC_REQUESTS
+        )
+        return _response(202)
+
+    with patch("asyncio.sleep", new=AsyncMock()):
+        logger.async_client.post = _fail_first_pass_then_recover
+        await logger._drain_with_retry()
+
+    assert set(delivered_models) == {"m0", "m1", "m2", "m3", "m4"}, "all chunks delivered after recovery"
+    assert logger.log_queue == [], "nothing left stranded once the destination recovered"
+
+
+@pytest.mark.asyncio
+async def test_terminal_drop_leaves_untried_late_arrival_for_next_drain():
+    """Against a permanently failing destination, the terminal drop clears only
+    the records this drain actually tried; a record a callback appends during the
+    final pass, after that pass's snapshot, is left in the queue for its own
+    serialized drain, never wiped un-tried."""
+    logger = _make_logger()
+    logger.stop()
+    from litellm.types.integrations.newrelic import NEWRELIC_METRICS_MAX_DRAIN_PASSES
+
+    late_record = _record(model="late-arrival")
+    posts = {"n": 0}
+
+    async def _fail_and_append_on_final_pass(url, data=None, headers=None, **kw):
+        posts["n"] += 1
+        # One record means one post per pass, so the final pass's post is the
+        # Nth; append then, after the drain has already snapshotted the queue.
+        if posts["n"] == NEWRELIC_METRICS_MAX_DRAIN_PASSES:
+            logger.log_queue.append(late_record)
+        resp = _response(503)
+        raise HTTPStatusError("err", request=resp.request, response=resp)
+
+    with patch("asyncio.sleep", new=AsyncMock()):
+        logger.async_client.post = _fail_and_append_on_final_pass
+        logger.log_queue.append(_record(model="doomed"))
+        await logger._drain_with_retry()
+    assert logger.log_queue == [late_record], "the un-tried late arrival is left for its own drain, not dropped"
+
+
+@pytest.mark.asyncio
+async def test_record_appended_on_an_early_pass_is_not_dropped_short_of_the_retry_budget():
+    """A record a callback appends during an early drain pass entered the queue
+    after this drain's snapshot, so it has not seen the full retry budget. The
+    terminal drop must clear only records queued when the drain began, leaving
+    the early-pass arrival for its own serialized drain instead of dropping it
+    after fewer than the configured attempts."""
+    logger = _make_logger()
+    logger.stop()
+    early_record = _record(model="early-pass-arrival")
+    posts = {"n": 0}
+
+    async def _fail_and_append_on_first_pass(url, data=None, headers=None, **kw):
+        posts["n"] += 1
+        # One record queued at start means the first pass's post is the 1st;
+        # append during it, before this drain's later passes.
+        if posts["n"] == 1:
+            logger.log_queue.append(early_record)
+        resp = _response(503)
+        raise HTTPStatusError("err", request=resp.request, response=resp)
+
+    with patch("asyncio.sleep", new=AsyncMock()):
+        logger.async_client.post = _fail_and_append_on_first_pass
+        logger.log_queue.append(_record(model="doomed"))
+        await logger._drain_with_retry()
+    assert logger.log_queue == [early_record], "the early-pass arrival is left for its own drain, not dropped short"
+
+
+@pytest.mark.asyncio
+async def test_post_stop_drains_are_serialized():
+    """A callback that appends to a stopped logger and starts its own drain must
+    queue behind an already-running drain, not race it: otherwise one drain's
+    terminal clear could wipe a record the other is still responsible for.
+    Proven by holding the first drain inside its flush and asserting the second
+    has not entered its own flush until the first releases."""
+    logger = _make_logger()
+    logger._stopped = True  # stopped without scheduling a background drain
+    logger.log_queue.append(_record(model="r1"))
+    entered = []
+    release = asyncio.Event()
+
+    async def blocking_flush():
+        entered.append(len(entered) + 1)
+        if len(entered) == 1:
+            await release.wait()
+        logger.log_queue.clear()
+
+    logger._drain_flush_once = blocking_flush
+    t1 = asyncio.create_task(logger._drain_with_retry())
+    await asyncio.sleep(0.02)  # let t1 acquire the drain lock and enter flush
+    assert entered == [1], f"first drain did not enter flush: {entered}"
+    t2 = asyncio.create_task(logger._drain_with_retry())
+    await asyncio.sleep(0.02)  # t2 must block on the drain lock, not enter flush
+    assert entered == [1], f"second drain raced the first: {entered}"
+    release.set()
+    await asyncio.gather(t1, t2)
+    assert logger.log_queue == []
+
+
+@pytest.mark.asyncio
+async def test_raised_403_is_dropped_not_requeued():
+    """AsyncHTTPHandler.post raises HTTPStatusError on 4xx, so a 403 (permanent
+    bad key) arrives as an exception, not a response. It must be dropped, never
+    requeued, or a revoked key retries forever."""
+    logger = _make_logger()
+    logger.log_queue.append(_record())
+    logger.async_client.post = _raises(403)
+    await logger.async_send_batch()
+    assert logger.log_queue == [], "a permanent 403 must drop, not requeue"
+
+
+@pytest.mark.asyncio
+async def test_raised_500_is_requeued():
+    """A raised 5xx is transient and must be requeued for retry."""
+    logger = _make_logger()
+    record = _record()
+    logger.log_queue.append(record)
+    logger.async_client.post = _raises(503)
+    await logger.async_send_batch()
+    assert logger.log_queue == [record], "a transient 5xx must requeue"
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("status", [429, 408])
+async def test_transient_4xx_is_requeued_not_dropped(status):
+    """The Metric API returns 429 when it throttles (and 408 on a request
+    timeout); both are transient and expect a retry, so the batch must be
+    requeued rather than permanently dropped like a 400/403."""
+    logger = _make_logger()
+    record = _record()
+    logger.log_queue.append(record)
+    logger.async_client.post = _raises(status)
+    await logger.async_send_batch()
+    assert logger.log_queue == [record], f"a transient {status} must requeue, not drop"
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("status", [200, 201, 204])
+async def test_any_2xx_is_treated_as_delivered_not_requeued(status):
+    """The Metric API answers 202, but any 2xx means the destination accepted the
+    batch. Treating a non-202 2xx as a failure would re-queue and re-send data
+    New Relic already stored, duplicating the team's metrics until the cap drops."""
+    logger = _make_logger()
+    logger.log_queue.append(_record())
+    logger.async_client.post = AsyncMock(return_value=_response(status))
+    await logger.async_send_batch()
+    assert logger.log_queue == [], f"a {status} success must drop, not requeue and duplicate"

tests/test_litellm/integrations/newrelic/test_newrelic_team_handler.py

@@ -0,0 +1,274 @@
+"""
+Tests for team-scoped New Relic metrics callback support.
+
+Verifies that NewRelicMetricsLogger is instantiated with per-team credentials
+(newrelic_api_key, newrelic_region) with no environment fallback, and that
+NewRelicHandler correctly resolves and caches per-team loggers.
+"""
+
+import copy
+from unittest.mock import patch
+
+import pytest
+
+from litellm.integrations.newrelic.newrelic_metrics import NewRelicMetricsLogger
+from litellm.integrations.newrelic.newrelic_team_handler import NewRelicHandler
+from litellm.litellm_core_utils.initialize_dynamic_callback_params import (
+    TRUSTED_CALLBACK_VARS_FIELD,
+)
+from litellm.litellm_core_utils.specialty_caches.dynamic_logging_cache import (
+    DynamicLoggingCache,
+)
+from litellm.types.integrations.newrelic import NEWRELIC_METRIC_ENDPOINT_BY_REGION
+from litellm.types.utils import StandardCallbackDynamicParams
+
+US_ENDPOINT = NEWRELIC_METRIC_ENDPOINT_BY_REGION["us"]
+EU_ENDPOINT = NEWRELIC_METRIC_ENDPOINT_BY_REGION["eu"]
+
+
+class TestNewRelicMetricsLoggerCredentialKwargs:
+    """The logger takes credentials by injection only; env vars never leak in."""
+
+    def test_init_with_explicit_credentials(self):
+        with patch("asyncio.create_task"):
+            logger = NewRelicMetricsLogger(newrelic_api_key="team_key", newrelic_region="eu")
+
+        assert logger.newrelic_api_key == "team_key"
+        assert logger.metric_api_url == EU_ENDPOINT
+
+    def test_init_defaults_to_us_region(self):
+        with patch("asyncio.create_task"):
+            logger = NewRelicMetricsLogger(newrelic_api_key="team_key")
+
+        assert logger.metric_api_url == US_ENDPOINT
+
+    def test_unknown_region_falls_back_to_us(self):
+        with patch("asyncio.create_task"):
+            logger = NewRelicMetricsLogger(newrelic_api_key="team_key", newrelic_region="mars")
+
+        assert logger.metric_api_url == US_ENDPOINT
+
+    def test_region_is_case_insensitive(self):
+        with patch("asyncio.create_task"):
+            logger = NewRelicMetricsLogger(newrelic_api_key="team_key", newrelic_region="EU")
+
+        assert logger.metric_api_url == EU_ENDPOINT
+
+    def test_init_raises_without_api_key(self):
+        with pytest.raises(ValueError, match="newrelic_api_key"):
+            with patch("asyncio.create_task"):
+                NewRelicMetricsLogger(newrelic_api_key="")
+
+    def test_init_never_falls_back_to_env_license_key(self, monkeypatch):
+        """A missing team key must fail, never silently reuse the operator's key."""
+        monkeypatch.setenv("NEW_RELIC_LICENSE_KEY", "operator-license-key")
+
+        with pytest.raises(ValueError, match="newrelic_api_key"):
+            with patch("asyncio.create_task"):
+                NewRelicMetricsLogger(newrelic_api_key="")
+
+
+class TestNewRelicHandler:
+    """The handler resolves the correct logger per team."""
+
+    def test_creates_team_logger_with_dynamic_credentials(self):
+        cache = DynamicLoggingCache()
+        params = StandardCallbackDynamicParams(newrelic_api_key="team_a_key", newrelic_region="eu")
+
+        with patch("asyncio.create_task"):
+            result = NewRelicHandler.get_newrelic_logger_for_request(
+                standard_callback_dynamic_params=params,
+                in_memory_dynamic_logger_cache=cache,
+            )
+
+        assert result.newrelic_api_key == "team_a_key"
+        assert result.metric_api_url == EU_ENDPOINT
+
+    def test_caches_team_logger(self):
+        cache = DynamicLoggingCache()
+        params = StandardCallbackDynamicParams(newrelic_api_key="team_b_key")
+
+        with patch("asyncio.create_task"):
+            result1 = NewRelicHandler.get_newrelic_logger_for_request(
+                standard_callback_dynamic_params=params,
+                in_memory_dynamic_logger_cache=cache,
+            )
+            result2 = NewRelicHandler.get_newrelic_logger_for_request(
+                standard_callback_dynamic_params=params,
+                in_memory_dynamic_logger_cache=cache,
+            )
+
+        assert result1 is result2
+
+    def test_different_teams_get_different_loggers(self):
+        cache = DynamicLoggingCache()
+        params_a = StandardCallbackDynamicParams(newrelic_api_key="team_a_key")
+        params_b = StandardCallbackDynamicParams(newrelic_api_key="team_b_key")
+
+        with patch("asyncio.create_task"):
+            result_a = NewRelicHandler.get_newrelic_logger_for_request(
+                standard_callback_dynamic_params=params_a,
+                in_memory_dynamic_logger_cache=cache,
+            )
+            result_b = NewRelicHandler.get_newrelic_logger_for_request(
+                standard_callback_dynamic_params=params_b,
+                in_memory_dynamic_logger_cache=cache,
+            )
+
+        assert result_a is not result_b
+        assert result_a.newrelic_api_key == "team_a_key"
+        assert result_b.newrelic_api_key == "team_b_key"
+
+    def test_region_is_part_of_cache_key(self):
+        """Same key, different region must not share a logger (different endpoints)."""
+        cache = DynamicLoggingCache()
+
+        with patch("asyncio.create_task"):
+            result_us = NewRelicHandler.get_newrelic_logger_for_request(
+                standard_callback_dynamic_params=StandardCallbackDynamicParams(newrelic_api_key="key"),
+                in_memory_dynamic_logger_cache=cache,
+            )
+            result_eu = NewRelicHandler.get_newrelic_logger_for_request(
+                standard_callback_dynamic_params=StandardCallbackDynamicParams(
+                    newrelic_api_key="key", newrelic_region="eu"
+                ),
+                in_memory_dynamic_logger_cache=cache,
+            )
+
+        assert result_us is not result_eu
+        assert result_us.metric_api_url == US_ENDPOINT
+        assert result_eu.metric_api_url == EU_ENDPOINT
+
+    def test_request_blocked_callback_params_includes_newrelic(self):
+        from litellm.litellm_core_utils.initialize_dynamic_callback_params import (
+            _request_blocked_callback_params,
+        )
+
+        assert "newrelic_api_key" in _request_blocked_callback_params
+        assert "newrelic_region" in _request_blocked_callback_params
+
+
+class TestDynamicCredentialDetection:
+    def test_no_credentials(self):
+        params = StandardCallbackDynamicParams()
+        assert NewRelicHandler._dynamic_newrelic_credentials_are_passed(params) is False
+
+    def test_region_only_is_not_credentials(self):
+        params = StandardCallbackDynamicParams(newrelic_region="eu")
+        assert NewRelicHandler._dynamic_newrelic_credentials_are_passed(params) is False
+
+    def test_api_key_is_credentials(self):
+        params = StandardCallbackDynamicParams(newrelic_api_key="key")
+        assert NewRelicHandler._dynamic_newrelic_credentials_are_passed(params) is True
+
+
+class TestStandardCallbackDynamicParamsIncludesNewRelic:
+    def test_newrelic_params_in_annotations(self):
+        annotations = StandardCallbackDynamicParams.__annotations__
+        assert "newrelic_api_key" in annotations
+        assert "newrelic_region" in annotations
+
+
+def _build_logging_obj(kwargs: dict, *, with_newrelic_callback: bool = True):
+    from litellm.litellm_core_utils.litellm_logging import Logging
+
+    with patch("asyncio.create_task"):
+        return Logging(
+            model="gpt-4",
+            messages=[{"role": "user", "content": "hi"}],
+            stream=False,
+            call_type="completion",
+            start_time="2026-01-01",
+            litellm_call_id="test-call-id",
+            function_id="test-func",
+            dynamic_success_callbacks=["newrelic"] if with_newrelic_callback else None,
+            kwargs=kwargs,
+        )
+
+
+def _metrics_loggers(logging_obj) -> list[NewRelicMetricsLogger]:
+    return [cb for cb in (logging_obj.dynamic_success_callbacks or []) if isinstance(cb, NewRelicMetricsLogger)]
+
+
+class TestTeamCallbackFlowPassesNewRelicCredentials:
+    """
+    newrelic_* credentials reach NewRelicHandler only from the proxy-stamped trusted
+    field. Anything the caller put in the request body must not, or a caller could
+    pair its own newrelic_region with the team's ingest key.
+    """
+
+    def test_trusted_callback_vars_reach_newrelic_handler(self):
+        logging_obj = _build_logging_obj(
+            {
+                TRUSTED_CALLBACK_VARS_FIELD: {"newrelic_api_key": "team-nr-key-123", "newrelic_region": "eu"},
+                "model": "gpt-4",
+                "litellm_params": {"metadata": {}},
+            }
+        )
+
+        metrics_loggers = _metrics_loggers(logging_obj)
+        assert len(metrics_loggers) == 1, "NewRelicMetricsLogger should be initialized from team callback_vars"
+        assert metrics_loggers[0].newrelic_api_key == "team-nr-key-123"
+        assert metrics_loggers[0].metric_api_url == EU_ENDPOINT
+
+    def test_trace_logger_still_dispatched_alongside_metrics(self):
+        """The metrics logger must not displace the trace logger for the same name."""
+        logging_obj = _build_logging_obj(
+            {
+                TRUSTED_CALLBACK_VARS_FIELD: {"newrelic_api_key": "team-nr-key-123"},
+                "model": "gpt-4",
+                "litellm_params": {"metadata": {}},
+            }
+        )
+
+        non_metrics = [
+            cb for cb in (logging_obj.dynamic_success_callbacks or []) if not isinstance(cb, NewRelicMetricsLogger)
+        ]
+        assert len(non_metrics) == 1, "trace logger (OTel v2 or legacy agent) must remain in the dynamic list"
+        assert len(_metrics_loggers(logging_obj)) == 1
+        async_non_metrics = [
+            cb
+            for cb in (logging_obj.dynamic_async_success_callbacks or [])
+            if not isinstance(cb, NewRelicMetricsLogger)
+        ]
+        assert len(async_non_metrics) == 1
+
+    def test_request_kwargs_newrelic_params_are_ignored(self):
+        logging_obj = _build_logging_obj(
+            {
+                "newrelic_api_key": "caller-nr-key",
+                "newrelic_region": "eu",
+                "model": "gpt-4",
+                "litellm_params": {"metadata": {}},
+            }
+        )
+
+        assert _metrics_loggers(logging_obj) == []
+
+    def test_logging_object_stays_deepcopyable(self):
+        logging_obj = _build_logging_obj(
+            {
+                TRUSTED_CALLBACK_VARS_FIELD: {"newrelic_api_key": "team-nr-key-123"},
+                "model": "gpt-4",
+                "litellm_params": {"metadata": {}},
+            },
+            with_newrelic_callback=False,
+        )
+
+        assert copy.deepcopy(logging_obj)._trusted_callback_vars == logging_obj._trusted_callback_vars
+
+    def test_caller_cannot_redirect_team_credentials(self):
+        """The exfil shape: caller's newrelic_region paired with the team's key."""
+        logging_obj = _build_logging_obj(
+            {
+                TRUSTED_CALLBACK_VARS_FIELD: {"newrelic_api_key": "team-nr-key-123"},
+                "newrelic_region": "eu",
+                "model": "gpt-4",
+                "litellm_params": {"metadata": {}},
+            }
+        )
+
+        metrics_loggers = _metrics_loggers(logging_obj)
+        assert len(metrics_loggers) == 1
+        assert metrics_loggers[0].newrelic_api_key == "team-nr-key-123"
+        assert metrics_loggers[0].metric_api_url == US_ENDPOINT

tests/test_litellm/integrations/otel/test_otel_v2_dynamic.py

@@ -357,6 +357,92 @@ def test_release_without_eviction_keeps_provider_alive(monkeypatch):
     cache.release(None)  # default-route release is a no-op
 
 
+# --- per-request service.name routing from trusted key/team config --- #
+
+
+def test_tenant_service_name_precedence_and_blanks():
+    from litellm.integrations.otel.plumbing.routing import tenant_service_name
+
+    assert tenant_service_name({"otel_service_name": "team-svc"}) == "team-svc"
+    assert tenant_service_name({"otel_service_name_override": "override", "otel_service_name": "base"}) == "override"
+    assert tenant_service_name({"otel_service_name": "   "}) is None
+    assert tenant_service_name({"logging_setting": "x"}) is None
+    assert tenant_service_name(None) is None
+
+
+def test_key_override_survives_team_metadata_merge():
+    from litellm.integrations.otel.plumbing.routing import tenant_service_name
+
+    # Request setup merges team metadata over key metadata (last writer wins),
+    # so a key keeps its own destination via ``otel_service_name_override``,
+    # which a team defining only ``otel_service_name`` never touches.
+    merged = {"otel_service_name_override": "key-svc"}
+    merged.update({"otel_service_name": "team-svc"})
+    assert tenant_service_name(merged) == "key-svc"
+
+
+def test_provider_cached_per_service_name():
+    cache = _cache("otel")
+    default = NoOpTracer()
+    routed = cache.route_for(default, None, {"otel_service_name": "payments-gateway"})
+    assert routed.tracer is not default
+    assert routed.detached is False  # stays parented into the request trace
+    assert routed.provider is not None
+    assert routed.provider.resource.attributes["service.name"] == "payments-gateway"
+    cache.route_for(default, None, {"otel_service_name": "payments-gateway"})
+    assert len(cache._providers) == 1
+    cache.route_for(default, None, {"otel_service_name": "search-gateway"})
+    assert len(cache._providers) == 2
+    for provider in cache._providers.values():
+        provider.shutdown()
+
+
+def test_service_name_routed_span_carries_team_service_name(monkeypatch):
+    # The artifact the exporter receives: the finished span's Resource must
+    # carry the team's service.name, not the env-configured default.
+    monkeypatch.setenv("OTEL_SERVICE_NAME", "proxy-default")
+    cache = _cache("otel")
+    default = NoOpTracer()
+    route = cache.route_for(default, None, {"otel_service_name": "payments-gateway"})
+    with route.tracer.start_as_current_span("chat gpt-4o-mini") as span:
+        pass
+    assert span.resource.attributes["service.name"] == "payments-gateway"
+    cache.release(route.provider)
+
+    unrouted = cache.route_for(default, None, {"logging_setting": "x"})
+    assert unrouted.tracer is default  # env fallback: no scoped provider built
+
+
+def test_client_dynamic_params_cannot_choose_service_name():
+    # ``StandardCallbackDynamicParams`` is populated from client-supplied
+    # request metadata; the service name may only come from server-set
+    # key/team config (the ``auth_metadata`` argument).
+    cache = _cache("otel")
+    default = NoOpTracer()
+    assert cache.route_for(default, {"otel_service_name": "attacker"}).tracer is default
+    assert cache.route_for(default, {"otel_service_name_override": "attacker"}).tracer is default
+    assert cache._providers == {}
+
+
+def test_service_name_override_leaves_exporters_untouched():
+    cache = _cache(
+        "otel",
+        exporters=[
+            ExporterSpec(
+                kind="otlp_http",
+                endpoint="http://collector:4318",
+                headers="x=base-collector",
+                owner=None,
+            ),
+        ],
+    )
+    cfg = cache._routed_config({}, {}, None, "payments-gateway")
+    assert cfg.service_name == "payments-gateway"
+    (spec,) = cfg.exporters
+    assert spec.headers == "x=base-collector"
+    assert spec.endpoint == "http://collector:4318"
+
+
 # --- New Relic: per-team api-key header + fixed-table region endpoint --- #
 
 

tests/test_litellm/integrations/otel/test_otel_v2_logger.py

@@ -778,11 +778,15 @@ def test_mcp_span_roots_without_transport_or_propagated_context(
 
 
 @pytest.mark.parametrize("make_payload, span_name", _MCP_SPAN_CASES)
-def test_mcp_span_parents_to_propagated_meta_trace_context(make_payload, span_name):
+def test_mcp_span_links_propagated_meta_trace_context_and_nests_under_transport(
+    make_payload, span_name
+):
     """When the client propagates W3C trace context in the request's
-    ``params._meta`` (SEP-414), the MCP span parents to it (one distributed trace)
-    and still links the transport span — never falling through to the
-    ambient/session span."""
+    ``params._meta`` (SEP-414), the MCP span still nests under the gateway's own
+    transport span — one renderable trace — and records the client's context as a
+    span *link*. Parenting to the remote context instead would root the span in a
+    trace whose root span never reaches the gateway's tracing backend, leaving the
+    span unreachable from the trace view."""
     logger, exporter = _logger()
     transport = logger._emitter.start_span(
         SpanRole.PROXY_REQUEST, LITELLM_PROXY_REQUEST_SPAN_NAME
@@ -801,12 +805,65 @@ def test_mcp_span_parents_to_propagated_meta_trace_context(make_payload, span_na
         reset_mcp_message_trace_carrier(token)
     transport.end()
     span = next(s for s in exporter.get_finished_spans() if s.name == span_name)
-    assert span.context.trace_id == 0x11111111111111111111111111111111
     assert span.parent is not None
-    assert span.parent.span_id == 0x2222222222222222
-    assert [link.context.span_id for link in span.links] == [
-        transport.get_span_context().span_id
+    assert span.parent.span_id == transport.get_span_context().span_id
+    assert span.context.trace_id == transport.get_span_context().trace_id
+    assert [link.context.trace_id for link in span.links] == [
+        0x11111111111111111111111111111111
     ]
+    assert [link.context.span_id for link in span.links] == [0x2222222222222222]
+
+
+@pytest.mark.parametrize("make_payload, span_name", _MCP_SPAN_CASES)
+def test_mcp_span_without_transport_roots_and_links_propagated_context(
+    make_payload, span_name
+):
+    """With no transport span at all there is nothing of the gateway's to anchor
+    to, so the span starts its own root trace — and the client context stays a
+    span link there too, so the event keeps one shape everywhere."""
+    logger, exporter = _logger()
+    token = set_mcp_message_trace_carrier(
+        {"traceparent": "00-11111111111111111111111111111111-2222222222222222-01"}
+    )
+    try:
+        asyncio.run(
+            logger.async_log_success_event(
+                {"standard_logging_object": make_payload()}, None, None, None
+            )
+        )
+    finally:
+        reset_mcp_message_trace_carrier(token)
+    span = next(s for s in exporter.get_finished_spans() if s.name == span_name)
+    assert span.parent is None
+    assert span.context.trace_id != 0x11111111111111111111111111111111
+    assert [link.context.span_id for link in span.links] == [0x2222222222222222]
+
+
+def test_mcp_span_links_unsampled_client_traceparent():
+    """A client traceparent with the sampled flag off ('-00') still yields a valid
+    remote context, so the link is recorded; the span's own recording follows the
+    transport's sampling decision, never the client's flag."""
+    logger, exporter = _logger()
+    transport = logger._emitter.start_span(
+        SpanRole.PROXY_REQUEST, LITELLM_PROXY_REQUEST_SPAN_NAME
+    )
+    set_request_root_span(transport)
+    token = set_mcp_message_trace_carrier(
+        {"traceparent": "00-11111111111111111111111111111111-2222222222222222-00"}
+    )
+    try:
+        asyncio.run(
+            logger.async_log_success_event(
+                {"standard_logging_object": _mcp_list_payload()}, None, None, None
+            )
+        )
+    finally:
+        reset_mcp_message_trace_carrier(token)
+    transport.end()
+    span = next(s for s in exporter.get_finished_spans() if s.name == "tools/list")
+    assert span.parent is not None
+    assert span.parent.span_id == transport.get_span_context().span_id
+    assert [link.context.span_id for link in span.links] == [0x2222222222222222]
 
 
 @pytest.mark.parametrize("make_payload, span_name", _MCP_SPAN_CASES)
@@ -839,8 +896,11 @@ def test_mcp_span_ignores_client_supplied_baggage(make_payload, span_name):
         reset_mcp_message_trace_carrier(token)
     transport.end()
     span = next(s for s in exporter.get_finished_spans() if s.name == span_name)
-    # Trace context still honored: proves the carrier was processed, not dropped wholesale.
-    assert span.parent is not None and span.parent.span_id == 0x2222222222222222
+    # Trace context still honored (as a link): proves the carrier was processed,
+    # not dropped wholesale.
+    assert [link.context.span_id for link in span.links] == [0x2222222222222222]
+    assert span.parent is not None
+    assert span.parent.span_id == transport.get_span_context().span_id
     # Identity is the authenticated payload's team, never the client's spoofed value.
     assert span.attributes[LiteLLM.TEAM_ID] == "t1"
     assert "litellm.metadata.user_api_key_user_id" not in span.attributes
@@ -888,10 +948,10 @@ def test_mcp_span_malformed_traceparent_nests_under_transport():
     assert span.links == ()
 
 
-def test_mcp_span_links_this_messages_transport_when_context_is_propagated():
-    """On the semconv path the transport is recorded as a link, and that link must
-    point at the POST carrying this message too. Reading the stale session anchor
-    would attribute the tool call to whichever request opened the session."""
+def test_mcp_span_with_propagated_context_nests_under_this_messages_transport():
+    """With client context propagated, the span must still anchor to the POST
+    carrying this message, not the stale session anchor — otherwise the tool call
+    is attributed to whichever request opened the session."""
     logger, exporter = _logger()
     session_opener = logger._emitter.start_span(
         SpanRole.PROXY_REQUEST, LITELLM_PROXY_REQUEST_SPAN_NAME
@@ -916,10 +976,10 @@ def test_mcp_span_links_this_messages_transport_when_context_is_propagated():
     session_opener.end()
     this_message.end()
     span = next(s for s in exporter.get_finished_spans() if s.name == "tools/list")
-    assert span.parent is not None and span.parent.span_id == 0x2222222222222222
-    assert [link.context.span_id for link in span.links] == [
-        this_message.get_span_context().span_id
-    ]
+    assert span.parent is not None
+    assert span.parent.span_id == this_message.get_span_context().span_id
+    assert span.context.trace_id == this_message.get_span_context().trace_id
+    assert [link.context.span_id for link in span.links] == [0x2222222222222222]
 
 
 def test_pre_call_idempotent_keeps_first_span():

tests/test_litellm/integrations/otel/test_otel_v2_sources_of_truth.py

@@ -107,32 +107,29 @@ def test_registry_parent_integrity_no_orphans():
 
 
 def test_registry_hierarchy_shape():
-    # MCP roles have no in-process parent: per the MCP semconv they root (or adopt
-    # the client's propagated _meta context), so they sit alongside PROXY_REQUEST.
-    assert set(root_roles()) == {
-        SpanRole.PROXY_REQUEST,
-        SpanRole.MCP_TOOL_CALL,
-        SpanRole.MCP_LIST_TOOLS,
-    }
+    assert set(root_roles()) == {SpanRole.PROXY_REQUEST}
     # Guardrails parent to the request span, not the LLM call: a pre-call
-    # guardrail runs before the LLM call exists, so it's a sibling of it.
+    # guardrail runs before the LLM call exists, so it's a sibling of it. MCP
+    # spans nest under the transport span of the request carrying that message.
     assert set(child_roles(SpanRole.PROXY_REQUEST)) == {
         SpanRole.LLM_CALL,
         SpanRole.GUARDRAIL,
         SpanRole.DB_CALL,
         SpanRole.SERVICE,
+        SpanRole.MCP_TOOL_CALL,
+        SpanRole.MCP_LIST_TOOLS,
     }
     assert SPAN_REGISTRY[SpanRole.LLM_CALL].kind is LiteLLMSpanKind.CLIENT
     # The proxy is an MCP client to the upstream tool server: CLIENT span. Listing
     # tools is the same client relationship, so it's a CLIENT span too.
     assert SPAN_REGISTRY[SpanRole.MCP_TOOL_CALL].kind is LiteLLMSpanKind.CLIENT
     assert SPAN_REGISTRY[SpanRole.MCP_LIST_TOOLS].kind is LiteLLMSpanKind.CLIENT
-    # MCP spans don't nest under the transport: they link the PROXY_REQUEST span
-    # instead of parenting to it (OTel GenAI MCP semconv).
-    assert SPAN_REGISTRY[SpanRole.MCP_TOOL_CALL].parent is None
-    assert SPAN_REGISTRY[SpanRole.MCP_LIST_TOOLS].parent is None
-    assert SPAN_REGISTRY[SpanRole.MCP_TOOL_CALL].links is SpanRole.PROXY_REQUEST
-    assert SPAN_REGISTRY[SpanRole.MCP_LIST_TOOLS].links is SpanRole.PROXY_REQUEST
+    # MCP spans nest under the transport span of the request carrying that
+    # message (resolved per message at emit time); a client-propagated context
+    # becomes a span link to that remote context, which is not a registry role
+    # (SpanSpec declares no link field at all).
+    assert SPAN_REGISTRY[SpanRole.MCP_TOOL_CALL].parent is SpanRole.PROXY_REQUEST
+    assert SPAN_REGISTRY[SpanRole.MCP_LIST_TOOLS].parent is SpanRole.PROXY_REQUEST
     assert SPAN_REGISTRY[SpanRole.PROXY_REQUEST].kind is LiteLLMSpanKind.SERVER
     assert SPAN_REGISTRY[SpanRole.GUARDRAIL].parent is SpanRole.PROXY_REQUEST
     # An outbound datastore call is a CLIENT span; an internal service is INTERNAL.

tests/test_litellm/litellm_core_utils/llm_cost_calc/test_tool_call_cost_tracking_dict_safety.py

@@ -8,11 +8,10 @@ See https://github.com/BerriAI/litellm/issues/26153.
 
 import pytest
 
-
 from litellm.litellm_core_utils.llm_cost_calc.tool_call_cost_tracking import (
     StandardBuiltInToolCostTracking,
-    _get_web_search_requests,
 )
+from litellm.litellm_core_utils.llm_cost_calc.utils import get_web_search_requests
 from litellm.types.utils import ModelResponse, ServerToolUse, Usage
 
 
@@ -28,25 +27,25 @@ class _UsageWithDictServerToolUse:
 
 
 def test_get_web_search_requests_handles_none():
-    assert _get_web_search_requests(None) is None
+    assert get_web_search_requests(None) is None
 
 
 def test_get_web_search_requests_handles_dict():
-    assert _get_web_search_requests({"web_search_requests": 5}) == 5
+    assert get_web_search_requests({"web_search_requests": 5}) == 5
 
 
 def test_get_web_search_requests_handles_dict_missing_key():
-    assert _get_web_search_requests({}) is None
+    assert get_web_search_requests({}) is None
 
 
 def test_get_web_search_requests_handles_pydantic():
     stu = ServerToolUse(web_search_requests=7)
-    assert _get_web_search_requests(stu) == 7
+    assert get_web_search_requests(stu) == 7
 
 
 def test_get_web_search_requests_handles_pydantic_with_none_value():
     stu = ServerToolUse()
-    assert _get_web_search_requests(stu) is None
+    assert get_web_search_requests(stu) is None
 
 
 def test_response_object_includes_web_search_call_with_dict_server_tool_use():

tests/test_litellm/litellm_core_utils/test_exception_mapping_utils.py

@@ -1002,6 +1002,17 @@ def test_an_exception_without_a_status_is_still_a_connection_error(quiet_excepti
         )
 
 
+def test_an_unmapped_exception_with_no_model_or_provider_is_a_connection_error(quiet_exception_mapping):
+    with pytest.raises(litellm.APIConnectionError) as raised:
+        exception_type(
+            model=None,
+            original_exception=ValueError("boom"),
+            custom_llm_provider=None,
+        )
+
+    assert "boom" in raised.value.message
+
+
 CONTEXT_WINDOW_MESSAGE = "This model's maximum context length is 4096 tokens."
 CONTENT_POLICY_MESSAGE = (
     '{"error": {"type": "invalid_request_error", "code": "content_policy_violation"}}'

tests/test_litellm/litellm_core_utils/test_realtime_streaming.py

@@ -2957,3 +2957,157 @@ async def test_log_messages_routes_async_logging_through_bounded_worker():
         logging_obj.success_handler.assert_not_called()
         # the bare create_task path must no longer be used for success logging
         mock_create_task.assert_not_called()
+
+
+@pytest.mark.asyncio
+async def test_provider_config_path_captures_transcription_usage():
+    """A transcription.completed event with usage from the provider transform must
+    land in the logged messages so realtime cost calculation can bill it."""
+    from typing import Final
+
+    from litellm.types.realtime import RealtimeInputAudioTranscriptionUsage, RealtimeResponseTypedDict
+
+    client_ws: Final = MagicMock()
+    client_ws.send_text = AsyncMock()
+    backend_ws: Final = MagicMock()
+    backend_ws.send = AsyncMock()
+    logging_obj: Final = MagicMock()
+
+    usage: Final[RealtimeInputAudioTranscriptionUsage] = {
+        "type": "tokens",
+        "input_tokens": 50,
+        "output_tokens": 6,
+        "total_tokens": 56,
+        "input_token_details": {"text_tokens": 0, "audio_tokens": 50},
+    }
+    transform_output: Final[RealtimeResponseTypedDict] = {
+        "response": {
+            "type": "conversation.item.input_audio_transcription.completed",
+            "event_id": "event_1",
+            "transcript": "ahoy",
+            "item_id": "item_1",
+            "content_index": 0,
+            "usage": usage,
+        },
+        "current_output_item_id": None,
+        "current_response_id": None,
+        "current_delta_chunks": None,
+        "current_conversation_id": None,
+        "current_item_chunks": None,
+        "current_delta_type": None,
+        "session_configuration_request": None,
+    }
+    provider_config: Final = MagicMock()
+    provider_config.transform_realtime_request = MagicMock(return_value=())
+    provider_config.transform_realtime_response = MagicMock(return_value=transform_output)
+
+    streaming: Final = RealTimeStreaming(
+        client_ws,
+        backend_ws,
+        logging_obj,
+        provider_config=provider_config,
+        model="gemini-3.5-transcribe-live",
+    )
+
+    await streaming._handle_provider_config_message("{}")
+
+    usage_events: Final = tuple(
+        message
+        for message in streaming.messages
+        if isinstance(message, dict)
+        and message.get("type") == "conversation.item.input_audio_transcription.completed"
+        and message.get("usage") == usage
+    )
+    assert len(usage_events) == 1
+
+
+@pytest.mark.asyncio
+async def test_session_close_flushes_unbilled_transcription_usage():
+    """Trailing audio appended after the last transcript frame must still be billed:
+    on session close the provider's unbilled estimate is flushed into the logged
+    messages before log_messages runs, and never forwarded to the client."""
+    from typing import Final
+
+    from litellm.types.realtime import RealtimeInputAudioTranscriptionUsage
+
+    client_ws: Final = MagicMock()
+    client_ws.send_text = AsyncMock()
+    backend_ws: Final = MagicMock()
+    backend_ws.recv = AsyncMock(side_effect=ConnectionClosed(None, None))
+    logging_obj: Final = MagicMock()
+    logging_obj.async_success_handler = AsyncMock()
+    logging_obj.success_handler = MagicMock()
+
+    usage: Final[RealtimeInputAudioTranscriptionUsage] = {
+        "type": "tokens",
+        "input_tokens": 153,
+        "output_tokens": 18,
+        "total_tokens": 171,
+        "input_token_details": {"text_tokens": 0, "audio_tokens": 153},
+    }
+    provider_config: Final = MagicMock()
+    provider_config.unbilled_usage_on_session_close = MagicMock(return_value=usage)
+
+    streaming: Final = RealTimeStreaming(
+        client_ws,
+        backend_ws,
+        logging_obj,
+        provider_config=provider_config,
+        model="gemini-3.5-transcribe-live",
+    )
+    logged_snapshots: Final[list[tuple]] = []
+
+    original_log_messages: Final = streaming.log_messages
+
+    async def _snapshot_then_log():
+        logged_snapshots.append(tuple(streaming.messages))
+        await original_log_messages()
+
+    streaming.log_messages = _snapshot_then_log
+
+    await streaming.backend_to_client_send_messages()
+
+    provider_config.unbilled_usage_on_session_close.assert_called_once_with("gemini-3.5-transcribe-live")
+    flushed: Final = tuple(
+        message
+        for message in streaming.messages
+        if isinstance(message, dict)
+        and message.get("type") == "conversation.item.input_audio_transcription.completed"
+        and message.get("usage") == usage
+    )
+    assert len(flushed) == 1
+    assert flushed[0] in logged_snapshots[0]
+    assert not client_ws.send_text.called
+
+
+@pytest.mark.asyncio
+async def test_session_close_flush_noop_without_unbilled_usage():
+    """Everything already billed mid-stream: the session-close flush must not append
+    a duplicate transcription event."""
+    from typing import Final
+
+    client_ws: Final = MagicMock()
+    client_ws.send_text = AsyncMock()
+    backend_ws: Final = MagicMock()
+    backend_ws.recv = AsyncMock(side_effect=ConnectionClosed(None, None))
+    logging_obj: Final = MagicMock()
+    logging_obj.async_success_handler = AsyncMock()
+    logging_obj.success_handler = MagicMock()
+
+    provider_config: Final = MagicMock()
+    provider_config.unbilled_usage_on_session_close = MagicMock(return_value=None)
+
+    streaming: Final = RealTimeStreaming(
+        client_ws,
+        backend_ws,
+        logging_obj,
+        provider_config=provider_config,
+        model="gemini-3.5-transcribe-live",
+    )
+
+    await streaming.backend_to_client_send_messages()
+
+    assert not any(
+        isinstance(message, dict) and message.get("type") == "conversation.item.input_audio_transcription.completed"
+        for message in streaming.messages
+    )

tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py

@@ -3997,3 +3997,98 @@ def test_translate_anthropic_messages_to_openai_carries_midturn_system_prompt_ca
     assert result == [
         {"role": "system", "content": [{"type": "text", "text": "fix", "prompt_cache_breakpoint": explicit}]}
     ]
+
+
+def _openai_response_with_usage(usage: Usage) -> ModelResponse:
+    return ModelResponse(
+        id="resp_web_search",
+        model="gemini-3-flash-preview",
+        choices=[
+            Choices(
+                finish_reason="stop",
+                index=0,
+                message=Message(role="assistant", content="searched"),
+            )
+        ],
+        usage=usage,
+    )
+
+
+def test_translate_openai_response_to_anthropic_maps_gemini_web_search_usage():
+    from litellm.types.utils import PromptTokensDetailsWrapper
+
+    usage = Usage(
+        prompt_tokens=385,
+        completion_tokens=566,
+        total_tokens=951,
+        prompt_tokens_details=PromptTokensDetailsWrapper(text_tokens=385, web_search_requests=2),
+    )
+
+    anthropic_response = LiteLLMAnthropicMessagesAdapter().translate_openai_response_to_anthropic(
+        response=_openai_response_with_usage(usage)
+    )
+
+    assert anthropic_response["usage"]["server_tool_use"] == {"web_search_requests": 2}
+
+
+def test_translate_openai_response_to_anthropic_maps_server_tool_use_web_search_usage():
+    from litellm.types.utils import ServerToolUse
+
+    usage = Usage(
+        prompt_tokens=100,
+        completion_tokens=40,
+        total_tokens=140,
+        server_tool_use=ServerToolUse(web_search_requests=3),
+    )
+
+    anthropic_response = LiteLLMAnthropicMessagesAdapter().translate_openai_response_to_anthropic(
+        response=_openai_response_with_usage(usage)
+    )
+
+    assert anthropic_response["usage"]["server_tool_use"] == {"web_search_requests": 3}
+
+
+def test_translate_openai_response_to_anthropic_omits_server_tool_use_without_web_search():
+    usage = Usage(prompt_tokens=100, completion_tokens=40, total_tokens=140)
+
+    anthropic_response = LiteLLMAnthropicMessagesAdapter().translate_openai_response_to_anthropic(
+        response=_openai_response_with_usage(usage)
+    )
+
+    assert "server_tool_use" not in anthropic_response["usage"]
+
+
+def test_completion_cost_on_translated_anthropic_response_includes_web_search():
+    from litellm.types.utils import PromptTokensDetailsWrapper
+
+    adapter = LiteLLMAnthropicMessagesAdapter()
+    with_search = adapter.translate_openai_response_to_anthropic(
+        response=_openai_response_with_usage(
+            Usage(
+                prompt_tokens=385,
+                completion_tokens=566,
+                total_tokens=951,
+                prompt_tokens_details=PromptTokensDetailsWrapper(text_tokens=385, web_search_requests=2),
+            )
+        )
+    )
+    without_search = adapter.translate_openai_response_to_anthropic(
+        response=_openai_response_with_usage(Usage(prompt_tokens=385, completion_tokens=566, total_tokens=951))
+    )
+
+    cost_with_search = litellm.completion_cost(
+        completion_response=with_search,
+        model="gemini/gemini-3-flash-preview",
+        call_type="anthropic_messages",
+    )
+    cost_without_search = litellm.completion_cost(
+        completion_response=without_search,
+        model="gemini/gemini-3-flash-preview",
+        call_type="anthropic_messages",
+    )
+
+    per_query_cost = litellm.model_cost["gemini/gemini-3-flash-preview"]["search_context_cost_per_query"][
+        "search_context_size_medium"
+    ]
+    assert per_query_cost > 0
+    assert cost_with_search - cost_without_search == pytest.approx(2 * per_query_cost)

tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_anthropic_experimental_pass_through_messages_handler.py

@@ -17,7 +17,12 @@ from litellm.anthropic_interface import messages
 from litellm.integrations.custom_logger import CustomLogger
 from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER
 from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
-from litellm.types.utils import Delta, ModelResponse, StreamingChoices
+from litellm.types.utils import (
+    Delta,
+    ModelResponse,
+    StandardLoggingPayloadErrorInformation,
+    StreamingChoices,
+)
 
 
 def test_anthropic_experimental_pass_through_messages_handler():
@@ -1292,7 +1297,7 @@ class TestMessagesStreamingSuccessLogging:
 class _FailureCapture(CustomLogger):
     def __init__(self):
         super().__init__()
-        self.error_information: List[Dict[str, Any]] = []
+        self.error_information: list[StandardLoggingPayloadErrorInformation] = []
 
     async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time):
         payload = kwargs.get("standard_logging_object") or {}

tests/test_litellm/llms/anthropic/test_cost_calculation_dict_safety.py

@@ -8,11 +8,8 @@ See https://github.com/BerriAI/litellm/issues/26153.
 
 import pytest
 
-
-from litellm.llms.anthropic.cost_calculation import (
-    _get_web_search_requests,
-    get_cost_for_anthropic_web_search,
-)
+from litellm.litellm_core_utils.llm_cost_calc.utils import get_web_search_requests
+from litellm.llms.anthropic.cost_calculation import get_cost_for_anthropic_web_search
 from litellm.types.utils import ModelInfo, ServerToolUse
 
 
@@ -33,19 +30,19 @@ def _make_model_info(cost_per_query: float = 0.01) -> ModelInfo:
 
 
 def test_get_web_search_requests_handles_none():
-    assert _get_web_search_requests(None) is None
+    assert get_web_search_requests(None) is None
 
 
 def test_get_web_search_requests_handles_dict():
-    assert _get_web_search_requests({"web_search_requests": 4}) == 4
+    assert get_web_search_requests({"web_search_requests": 4}) == 4
 
 
 def test_get_web_search_requests_handles_dict_missing_key():
-    assert _get_web_search_requests({}) is None
+    assert get_web_search_requests({}) is None
 
 
 def test_get_web_search_requests_handles_pydantic():
-    assert _get_web_search_requests(ServerToolUse(web_search_requests=2)) == 2
+    assert get_web_search_requests(ServerToolUse(web_search_requests=2)) == 2
 
 
 def test_get_cost_for_anthropic_web_search_with_dict_server_tool_use():

tests/test_litellm/llms/bedrock/rerank/test_bedrock_rerank_header_forwarding.py

@@ -12,6 +12,7 @@ import pytest
 
 import litellm
 from litellm.llms.bedrock.base_aws_llm import Boto3CredentialsInfo
+from litellm.llms.bedrock.rerank.handler import BedrockRerankHandler
 from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
 
 # Mock response for Bedrock rerank
@@ -402,6 +403,66 @@ def test_bedrock_rerank_extra_headers_and_headers_merge():
             pytest.fail(f"Failed to merge and forward headers: {str(e)}")
 
 
+def test_bedrock_rerank_forwarded_headers_excluded_from_sigv4_signature():
+    """
+    A forwarded header like x-forwarded-for can be rewritten between LiteLLM
+    signing the request and AWS receiving it (e.g. by an intermediate load
+    balancer), which invalidates the signature if that header was part of
+    the signed set. It must still reach Bedrock, just unsigned.
+    """
+    handler = BedrockRerankHandler()
+
+    prepared_request = handler._prepare_request(
+        model="cohere.rerank-v3-5:0",
+        api_base=None,
+        extra_headers={"x-forwarded-for": "203.0.113.5"},
+        data={"query": test_query, "documents": test_documents},
+        optional_params={
+            "aws_access_key_id": "test-access-key",
+            "aws_secret_access_key": "test-secret-key",
+            "aws_region_name": "us-east-1",
+        },
+    )
+
+    headers = prepared_request["prepped"].headers
+    signed_headers = headers["Authorization"].split("SignedHeaders=")[1].split(",")[0].split(";")
+
+    assert "x-forwarded-for" not in signed_headers, (
+        f"x-forwarded-for must not be part of the SigV4 signature, got SignedHeaders={signed_headers}"
+    )
+    assert headers["x-forwarded-for"] == "203.0.113.5", "forwarded header must still reach Bedrock, unsigned"
+
+
+def test_bedrock_rerank_signs_with_sigv4_even_when_bedrock_api_key_is_set(monkeypatch):
+    """
+    Bedrock API keys are only valid for Bedrock and Bedrock Runtime actions, not for
+    Agents for Amazon Bedrock Runtime ones. Rerank is served by bedrock-agent-runtime,
+    so it has to keep signing with SigV4 even when AWS_BEARER_TOKEN_BEDROCK is set.
+    """
+    monkeypatch.setenv("AWS_BEARER_TOKEN_BEDROCK", "test-bedrock-api-key")
+
+    handler = BedrockRerankHandler()
+
+    prepared_request = handler._prepare_request(
+        model="cohere.rerank-v3-5:0",
+        api_base=None,
+        extra_headers=None,
+        data={"query": test_query, "documents": test_documents},
+        optional_params={
+            "aws_access_key_id": "test-access-key",
+            "aws_secret_access_key": "test-secret-key",
+            "aws_region_name": "us-east-1",
+        },
+    )
+
+    assert prepared_request["endpoint_url"].startswith("https://bedrock-agent-runtime.")
+
+    authorization = prepared_request["prepped"].headers["Authorization"]
+    assert authorization.startswith("AWS4-HMAC-SHA256"), (
+        f"rerank must sign with SigV4, got Authorization={authorization[:30]}"
+    )
+
+
 @pytest.mark.asyncio
 async def test_bedrock_rerank_records_llm_api_duration():
     """The bedrock rerank handler must feed httpx timing into the logging obj, so the

tests/test_litellm/llms/bedrock/test_cross_region_inference_profile_mapping.py

@@ -59,17 +59,17 @@ class GptProfile(NamedTuple):
 GPT_5_6_PROFILES = [
     GptProfile(
         model_id="us.openai.gpt-5.6-sol",
-        input_cost=5.5e-06, input_cost_above_272k=1.1e-05,
-        cache_write=6.875e-06, cache_write_above_272k=1.375e-05,
-        cache_read=5.5e-07, cache_read_above_272k=1.1e-06,
-        output_cost=3.3e-05, output_cost_above_272k=4.95e-05,
+        input_cost=4.4e-06, input_cost_above_272k=8.8e-06,
+        cache_write=5.5e-06, cache_write_above_272k=1.1e-05,
+        cache_read=4.4e-07, cache_read_above_272k=8.8e-07,
+        output_cost=2.2e-05, output_cost_above_272k=3.3e-05,
     ),
     GptProfile(
         model_id="global.openai.gpt-5.6-sol",
-        input_cost=5e-06, input_cost_above_272k=1e-05,
-        cache_write=6.25e-06, cache_write_above_272k=1.25e-05,
-        cache_read=5e-07, cache_read_above_272k=1e-06,
-        output_cost=3e-05, output_cost_above_272k=4.5e-05,
+        input_cost=4e-06, input_cost_above_272k=8e-06,
+        cache_write=5e-06, cache_write_above_272k=1e-05,
+        cache_read=4e-07, cache_read_above_272k=8e-07,
+        output_cost=2e-05, output_cost_above_272k=3e-05,
     ),
     GptProfile(
         model_id="us.openai.gpt-5.6-terra",
@@ -221,7 +221,7 @@ def test_bedrock_gpt_5_6_above_272k_tier_applies_to_cost(local_model_cost_map):
         custom_llm_provider="bedrock",
     )
 
-    assert cost == pytest.approx((300000 * 1.1e-05) + (1000 * 4.95e-05), rel=1e-9)
+    assert cost == pytest.approx((300000 * 8.8e-06) + (1000 * 3.3e-05), rel=1e-9)
 
 
 def test_bedrock_gpt_5_6_bills_cache_read_tokens(local_model_cost_map):
@@ -241,10 +241,10 @@ def test_bedrock_gpt_5_6_bills_cache_read_tokens(local_model_cost_map):
         custom_llm_provider="bedrock",
     )
 
-    expected = (2 * 5.5e-06) + (15609 * 5.5e-07) + (5 * 3.3e-05)
+    expected = (2 * 4.4e-06) + (15609 * 4.4e-07) + (5 * 2.2e-05)
     assert cost == pytest.approx(expected, rel=1e-9)
     # Without cache_read_input_token_cost the cached prefix bills at zero.
-    assert cost > (15611 * 5.5e-06) * 0.1
+    assert cost > (15611 * 4.4e-06) * 0.1
 
 
 def test_bedrock_gpt_5_6_bills_cache_write_tokens(local_model_cost_map):
@@ -263,7 +263,7 @@ def test_bedrock_gpt_5_6_bills_cache_write_tokens(local_model_cost_map):
         custom_llm_provider="bedrock",
     )
 
-    expected = (2 * 5.5e-06) + (15609 * 6.875e-06) + (5 * 3.3e-05)
+    expected = (2 * 4.4e-06) + (15609 * 5.5e-06) + (5 * 2.2e-05)
     assert cost == pytest.approx(expected, rel=1e-9)
 
 

tests/test_litellm/llms/bedrock_mantle/test_bedrock_mantle_responses_transformation.py

@@ -1683,10 +1683,19 @@ class TestBedrockMantleResponsesPricing:
         assert info["cache_read_input_token_cost"] == pytest.approx(2.75e-07)
         assert info["max_input_tokens"] == 1050000
 
+    def test_gpt_5_6_cyber_pricing_and_mode(self, local_cost_map):
+        info = litellm.get_model_info("bedrock_mantle/openai.gpt-5.6-cyber")
+        assert info["mode"] == "responses"
+        assert info["input_cost_per_token"] == pytest.approx(1.375e-05)
+        assert info["cache_creation_input_token_cost"] == pytest.approx(1.71875e-05)
+        assert info["cache_read_input_token_cost"] == pytest.approx(1.375e-06)
+        assert info["output_cost_per_token"] == pytest.approx(8.25e-05)
+        assert info["max_input_tokens"] == 272000
+
     @pytest.mark.parametrize(
         "model, input_cost, cache_creation_cost, cache_read_cost, output_cost",
         [
-            ("openai.gpt-5.6-sol", 5.5e-06, 6.875e-06, 5.5e-07, 3.3e-05),
+            ("openai.gpt-5.6-sol", 4.4e-06, 5.5e-06, 4.4e-07, 2.2e-05),
             ("openai.gpt-5.6-terra", 2.2e-06, 2.75e-06, 2.2e-07, 1.32e-05),
             ("openai.gpt-5.6-luna", 2.2e-07, 2.75e-07, 2.2e-08, 1.32e-06),
         ],
@@ -1709,7 +1718,7 @@ class TestBedrockMantleResponsesPricing:
     @pytest.mark.parametrize(
         "model, input_cost, output_cost",
         [
-            ("openai.gpt-5.6-sol", 5.5e-06, 3.3e-05),
+            ("openai.gpt-5.6-sol", 4.4e-06, 2.2e-05),
             ("openai.gpt-5.6-terra", 2.2e-06, 1.32e-05),
             ("openai.gpt-5.6-luna", 2.2e-07, 1.32e-06),
         ],

tests/test_litellm/llms/databricks/test_databricks_cost_calculator.py

@@ -61,6 +61,8 @@ PUBLISHED_DBU_PER_MILLION: Final = {
     "databricks/databricks-gemini-3-1-flash-lite": ("4.464", "26.786", "4.464", "0.446"),
     "databricks/databricks-gemini-2-5-pro": ("22.321", "178.571", "22.321", "2.232"),
     "databricks/databricks-gemini-2-5-flash": ("5.357", "44.643", "5.357", "0.536"),
+    "databricks/databricks-kimi-k3": ("42.857", "214.286", "42.857", "4.286"),
+    "databricks/databricks-glm-5-2": ("20.000", "62.857", "20.000", "3.714"),
 }
 PROMOTIONAL_DISCOUNT: Final = 0.80
 PROMOTION_EXPIRES: Final = "2027-01-31"