feat: add in_flight_requests metric to /health/backlog + prometheus

litellm/model_prices_and_context_window_backup.json

@@ -17357,6 +17357,39 @@
         "supports_tool_choice": true,
         "supports_vision": false
     },
+    "gpt-audio-1.5": {
+        "input_cost_per_audio_token": 3.2e-05,
+        "input_cost_per_token": 2.5e-06,
+        "litellm_provider": "openai",
+        "max_input_tokens": 128000,
+        "max_output_tokens": 16384,
+        "max_tokens": 16384,
+        "mode": "chat",
+        "output_cost_per_audio_token": 6.4e-05,
+        "output_cost_per_token": 1e-05,
+        "supported_endpoints": [
+            "/v1/chat/completions"
+        ],
+        "supported_modalities": [
+            "text",
+            "audio"
+        ],
+        "supported_output_modalities": [
+            "text",
+            "audio"
+        ],
+        "supports_audio_input": true,
+        "supports_audio_output": true,
+        "supports_function_calling": true,
+        "supports_native_streaming": true,
+        "supports_parallel_function_calling": true,
+        "supports_prompt_caching": false,
+        "supports_reasoning": false,
+        "supports_response_schema": false,
+        "supports_system_messages": true,
+        "supports_tool_choice": true,
+        "supports_vision": false
+    },
     "gpt-audio-2025-08-28": {
         "input_cost_per_audio_token": 3.2e-05,
         "input_cost_per_token": 2.5e-06,
@@ -19041,6 +19074,38 @@
         "supports_system_messages": true,
         "supports_tool_choice": true
     },
+    "gpt-realtime-1.5": {
+        "cache_creation_input_audio_token_cost": 4e-07,
+        "cache_read_input_token_cost": 4e-07,
+        "input_cost_per_audio_token": 3.2e-05,
+        "input_cost_per_image": 5e-06,
+        "input_cost_per_token": 4e-06,
+        "litellm_provider": "openai",
+        "max_input_tokens": 32000,
+        "max_output_tokens": 4096,
+        "max_tokens": 4096,
+        "mode": "chat",
+        "output_cost_per_audio_token": 6.4e-05,
+        "output_cost_per_token": 1.6e-05,
+        "supported_endpoints": [
+            "/v1/realtime"
+        ],
+        "supported_modalities": [
+            "text",
+            "image",
+            "audio"
+        ],
+        "supported_output_modalities": [
+            "text",
+            "audio"
+        ],
+        "supports_audio_input": true,
+        "supports_audio_output": true,
+        "supports_function_calling": true,
+        "supports_parallel_function_calling": true,
+        "supports_system_messages": true,
+        "supports_tool_choice": true
+    },
     "gpt-realtime-mini": {
         "cache_creation_input_audio_token_cost": 3e-07,
         "cache_read_input_audio_token_cost": 3e-07,
@@ -25001,8 +25066,8 @@
         "mode": "chat",
         "output_cost_per_token": 0.0,
         "source": "https://platform.publicai.co/docs",
-        "supports_function_calling": true,
-        "supports_tool_choice": true
+        "supports_function_calling": false,
+        "supports_tool_choice": false
     },
     "publicai/swiss-ai/apertus-70b-instruct": {
         "input_cost_per_token": 0.0,
@@ -25013,8 +25078,8 @@
         "mode": "chat",
         "output_cost_per_token": 0.0,
         "source": "https://platform.publicai.co/docs",
-        "supports_function_calling": true,
-        "supports_tool_choice": true
+        "supports_function_calling": false,
+        "supports_tool_choice": false
     },
     "publicai/aisingapore/Gemma-SEA-LION-v4-27B-IT": {
         "input_cost_per_token": 0.0,
@@ -30640,6 +30705,7 @@
         "supports_web_search": true
     },
     "xai/grok-2-vision-1212": {
+        "deprecation_date": "2026-02-28",
         "input_cost_per_image": 2e-06,
         "input_cost_per_token": 2e-06,
         "litellm_provider": "xai",
@@ -30744,6 +30810,7 @@
     },
     "xai/grok-3-mini": {
         "cache_read_input_token_cost": 7.5e-08,
+        "deprecation_date": "2026-02-28",
         "input_cost_per_token": 3e-07,
         "litellm_provider": "xai",
         "max_input_tokens": 131072,
@@ -30760,6 +30827,7 @@
     },
     "xai/grok-3-mini-beta": {
         "cache_read_input_token_cost": 7.5e-08,
+        "deprecation_date": "2026-02-28",
         "input_cost_per_token": 3e-07,
         "litellm_provider": "xai",
         "max_input_tokens": 131072,

litellm/proxy/health_endpoints/_health_endpoints.py

@@ -31,6 +31,9 @@ from litellm.proxy.health_check import (
     perform_health_check,
     run_with_timeout,
 )
+from litellm.proxy.middleware.in_flight_requests_middleware import (
+    get_in_flight_requests,
+)
 from litellm.secret_managers.main import get_secret
 from litellm.litellm_core_utils.custom_logger_registry import CustomLoggerRegistry
 
@@ -1276,6 +1279,23 @@ async def health_readiness():
         raise HTTPException(status_code=503, detail=f"Service Unhealthy ({str(e)})")
 
 
+@router.get(
+    "/health/backlog",
+    tags=["health"],
+    dependencies=[Depends(user_api_key_auth)],
+)
+async def health_backlog():
+    """
+    Returns the number of HTTP requests currently in-flight on this uvicorn worker.
+
+    Use this to measure per-pod queue depth. A high value means the worker is
+    processing many concurrent requests — requests arriving now will have to wait
+    for the event loop to get to them, adding latency before LiteLLM even starts
+    its own timer.
+    """
+    return {"in_flight_requests": get_in_flight_requests()}
+
+
 @router.get(
     "/health/liveliness",  # Historical LiteLLM name; doesn't match k8s terminology but kept for backwards compatibility
     tags=["health"],

litellm/proxy/middleware/in_flight_requests_middleware.py

@@ -0,0 +1,75 @@
+"""
+Tracks the number of HTTP requests currently in-flight on this uvicorn worker.
+
+Used by /health/backlog to expose per-pod queue depth, and emitted as the
+Prometheus gauge `litellm_in_flight_requests`.
+"""
+
+import os
+from typing import Optional
+
+from starlette.types import ASGIApp, Receive, Scope, Send
+
+_in_flight: int = 0
+
+# Lazily created on first request so PROMETHEUS_MULTIPROC_DIR is already set
+# by the time we register the metric.
+_gauge: Optional[object] = None
+
+
+def _get_gauge() -> Optional[object]:
+    global _gauge
+    if _gauge is not None:
+        return _gauge
+    try:
+        from prometheus_client import Gauge
+
+        kwargs = {}
+        if "PROMETHEUS_MULTIPROC_DIR" in os.environ:
+            # livesum aggregates across all worker processes in the scrape response
+            kwargs["multiprocess_mode"] = "livesum"
+        _gauge = Gauge(
+            "litellm_in_flight_requests",
+            "Number of HTTP requests currently in-flight on this uvicorn worker",
+            **kwargs,
+        )
+    except Exception:
+        pass
+    return _gauge
+
+
+def get_in_flight_requests() -> int:
+    return _in_flight
+
+
+class InFlightRequestsMiddleware:
+    """
+    ASGI middleware that increments a counter when a request arrives
+    and decrements it when the response is sent (or an error occurs).
+
+    The counter is module-level and therefore scoped to a single uvicorn
+    worker process — exactly the per-pod granularity we want.
+
+    Also updates the `litellm_in_flight_requests` Prometheus gauge if
+    prometheus_client is installed.
+    """
+
+    def __init__(self, app: ASGIApp) -> None:
+        self.app = app
+
+    async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None:
+        if scope["type"] != "http":
+            await self.app(scope, receive, send)
+            return
+
+        global _in_flight
+        _in_flight += 1
+        gauge = _get_gauge()
+        if gauge is not None:
+            gauge.inc()  # type: ignore[union-attr]
+        try:
+            await self.app(scope, receive, send)
+        finally:
+            _in_flight -= 1
+            if gauge is not None:
+                gauge.dec()  # type: ignore[union-attr]

litellm/proxy/proxy_server.py

@@ -398,6 +398,9 @@ from litellm.proxy.management_endpoints.user_agent_analytics_endpoints import (
     router as user_agent_analytics_router,
 )
 from litellm.proxy.management_helpers.audit_logs import create_audit_log_for_update
+from litellm.proxy.middleware.in_flight_requests_middleware import (
+    InFlightRequestsMiddleware,
+)
 from litellm.proxy.middleware.prometheus_auth_middleware import PrometheusAuthMiddleware
 from litellm.proxy.ocr_endpoints.endpoints import router as ocr_router
 from litellm.proxy.openai_files_endpoints.files_endpoints import (
@@ -1198,6 +1201,7 @@ app.add_middleware(
 )
 
 app.add_middleware(PrometheusAuthMiddleware)
+app.add_middleware(InFlightRequestsMiddleware)
 
 
 def mount_swagger_ui():

litellm/types/integrations/prometheus.py

@@ -238,6 +238,7 @@ DEFINED_PROMETHEUS_METRICS = Literal[
     "litellm_remaining_api_key_tokens_for_model",
     "litellm_llm_api_failed_requests_metric",
     "litellm_callback_logging_failures_metric",
+    "litellm_in_flight_requests",
 ]