diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 0f84bba941d..539f694c850 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -17357,6 +17357,39 @@ "supports_tool_choice": true, "supports_vision": false }, + "gpt-audio-1.5": { + "input_cost_per_audio_token": 3.2e-05, + "input_cost_per_token": 2.5e-06, + "litellm_provider": "openai", + "max_input_tokens": 128000, + "max_output_tokens": 16384, + "max_tokens": 16384, + "mode": "chat", + "output_cost_per_audio_token": 6.4e-05, + "output_cost_per_token": 1e-05, + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supported_modalities": [ + "text", + "audio" + ], + "supported_output_modalities": [ + "text", + "audio" + ], + "supports_audio_input": true, + "supports_audio_output": true, + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_parallel_function_calling": true, + "supports_prompt_caching": false, + "supports_reasoning": false, + "supports_response_schema": false, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": false + }, "gpt-audio-2025-08-28": { "input_cost_per_audio_token": 3.2e-05, "input_cost_per_token": 2.5e-06, @@ -19041,6 +19074,38 @@ "supports_system_messages": true, "supports_tool_choice": true }, + "gpt-realtime-1.5": { + "cache_creation_input_audio_token_cost": 4e-07, + "cache_read_input_token_cost": 4e-07, + "input_cost_per_audio_token": 3.2e-05, + "input_cost_per_image": 5e-06, + "input_cost_per_token": 4e-06, + "litellm_provider": "openai", + "max_input_tokens": 32000, + "max_output_tokens": 4096, + "max_tokens": 4096, + "mode": "chat", + "output_cost_per_audio_token": 6.4e-05, + "output_cost_per_token": 1.6e-05, + "supported_endpoints": [ + "/v1/realtime" + ], + "supported_modalities": [ + "text", + "image", + "audio" + ], + "supported_output_modalities": [ + "text", + "audio" + ], + "supports_audio_input": true, + "supports_audio_output": true, + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, "gpt-realtime-mini": { "cache_creation_input_audio_token_cost": 3e-07, "cache_read_input_audio_token_cost": 3e-07, @@ -25001,8 +25066,8 @@ "mode": "chat", "output_cost_per_token": 0.0, "source": "https://platform.publicai.co/docs", - "supports_function_calling": true, - "supports_tool_choice": true + "supports_function_calling": false, + "supports_tool_choice": false }, "publicai/swiss-ai/apertus-70b-instruct": { "input_cost_per_token": 0.0, @@ -25013,8 +25078,8 @@ "mode": "chat", "output_cost_per_token": 0.0, "source": "https://platform.publicai.co/docs", - "supports_function_calling": true, - "supports_tool_choice": true + "supports_function_calling": false, + "supports_tool_choice": false }, "publicai/aisingapore/Gemma-SEA-LION-v4-27B-IT": { "input_cost_per_token": 0.0, @@ -30640,6 +30705,7 @@ "supports_web_search": true }, "xai/grok-2-vision-1212": { + "deprecation_date": "2026-02-28", "input_cost_per_image": 2e-06, "input_cost_per_token": 2e-06, "litellm_provider": "xai", @@ -30744,6 +30810,7 @@ }, "xai/grok-3-mini": { "cache_read_input_token_cost": 7.5e-08, + "deprecation_date": "2026-02-28", "input_cost_per_token": 3e-07, "litellm_provider": "xai", "max_input_tokens": 131072, @@ -30760,6 +30827,7 @@ }, "xai/grok-3-mini-beta": { "cache_read_input_token_cost": 7.5e-08, + "deprecation_date": "2026-02-28", "input_cost_per_token": 3e-07, "litellm_provider": "xai", "max_input_tokens": 131072, diff --git a/litellm/proxy/health_endpoints/_health_endpoints.py b/litellm/proxy/health_endpoints/_health_endpoints.py index da90696ec2d..5c39fd395b8 100644 --- a/litellm/proxy/health_endpoints/_health_endpoints.py +++ b/litellm/proxy/health_endpoints/_health_endpoints.py @@ -31,6 +31,9 @@ from litellm.proxy.health_check import ( perform_health_check, run_with_timeout, ) +from litellm.proxy.middleware.in_flight_requests_middleware import ( + get_in_flight_requests, +) from litellm.secret_managers.main import get_secret from litellm.litellm_core_utils.custom_logger_registry import CustomLoggerRegistry @@ -1276,6 +1279,23 @@ async def health_readiness(): raise HTTPException(status_code=503, detail=f"Service Unhealthy ({str(e)})") +@router.get( + "/health/backlog", + tags=["health"], + dependencies=[Depends(user_api_key_auth)], +) +async def health_backlog(): + """ + Returns the number of HTTP requests currently in-flight on this uvicorn worker. + + Use this to measure per-pod queue depth. A high value means the worker is + processing many concurrent requests — requests arriving now will have to wait + for the event loop to get to them, adding latency before LiteLLM even starts + its own timer. + """ + return {"in_flight_requests": get_in_flight_requests()} + + @router.get( "/health/liveliness", # Historical LiteLLM name; doesn't match k8s terminology but kept for backwards compatibility tags=["health"], diff --git a/litellm/proxy/middleware/in_flight_requests_middleware.py b/litellm/proxy/middleware/in_flight_requests_middleware.py new file mode 100644 index 00000000000..0c0476373eb --- /dev/null +++ b/litellm/proxy/middleware/in_flight_requests_middleware.py @@ -0,0 +1,75 @@ +""" +Tracks the number of HTTP requests currently in-flight on this uvicorn worker. + +Used by /health/backlog to expose per-pod queue depth, and emitted as the +Prometheus gauge `litellm_in_flight_requests`. +""" + +import os +from typing import Optional + +from starlette.types import ASGIApp, Receive, Scope, Send + +_in_flight: int = 0 + +# Lazily created on first request so PROMETHEUS_MULTIPROC_DIR is already set +# by the time we register the metric. +_gauge: Optional[object] = None + + +def _get_gauge() -> Optional[object]: + global _gauge + if _gauge is not None: + return _gauge + try: + from prometheus_client import Gauge + + kwargs = {} + if "PROMETHEUS_MULTIPROC_DIR" in os.environ: + # livesum aggregates across all worker processes in the scrape response + kwargs["multiprocess_mode"] = "livesum" + _gauge = Gauge( + "litellm_in_flight_requests", + "Number of HTTP requests currently in-flight on this uvicorn worker", + **kwargs, + ) + except Exception: + pass + return _gauge + + +def get_in_flight_requests() -> int: + return _in_flight + + +class InFlightRequestsMiddleware: + """ + ASGI middleware that increments a counter when a request arrives + and decrements it when the response is sent (or an error occurs). + + The counter is module-level and therefore scoped to a single uvicorn + worker process — exactly the per-pod granularity we want. + + Also updates the `litellm_in_flight_requests` Prometheus gauge if + prometheus_client is installed. + """ + + def __init__(self, app: ASGIApp) -> None: + self.app = app + + async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None: + if scope["type"] != "http": + await self.app(scope, receive, send) + return + + global _in_flight + _in_flight += 1 + gauge = _get_gauge() + if gauge is not None: + gauge.inc() # type: ignore[union-attr] + try: + await self.app(scope, receive, send) + finally: + _in_flight -= 1 + if gauge is not None: + gauge.dec() # type: ignore[union-attr] diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 9959918b76c..a54cc5fa471 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -398,6 +398,9 @@ from litellm.proxy.management_endpoints.user_agent_analytics_endpoints import ( router as user_agent_analytics_router, ) from litellm.proxy.management_helpers.audit_logs import create_audit_log_for_update +from litellm.proxy.middleware.in_flight_requests_middleware import ( + InFlightRequestsMiddleware, +) from litellm.proxy.middleware.prometheus_auth_middleware import PrometheusAuthMiddleware from litellm.proxy.ocr_endpoints.endpoints import router as ocr_router from litellm.proxy.openai_files_endpoints.files_endpoints import ( @@ -1198,6 +1201,7 @@ app.add_middleware( ) app.add_middleware(PrometheusAuthMiddleware) +app.add_middleware(InFlightRequestsMiddleware) def mount_swagger_ui(): diff --git a/litellm/types/integrations/prometheus.py b/litellm/types/integrations/prometheus.py index a0471a2dfa1..0c2f64dcb4d 100644 --- a/litellm/types/integrations/prometheus.py +++ b/litellm/types/integrations/prometheus.py @@ -238,6 +238,7 @@ DEFINED_PROMETHEUS_METRICS = Literal[ "litellm_remaining_api_key_tokens_for_model", "litellm_llm_api_failed_requests_metric", "litellm_callback_logging_failures_metric", + "litellm_in_flight_requests", ]