emit deployment_partial_outage on prometheus

litellm/integrations/prometheus.py

@@ -279,6 +279,8 @@ class PrometheusLogger:
                     user_api_team_alias,
                     user_id,
                 ).inc()
+
+                self.set_llm_deployment_failure_metrics(kwargs)
         except Exception as e:
             verbose_logger.error(
                 "prometheus Layer Error(): Exception occured - {}".format(str(e))
@@ -286,6 +288,33 @@ class PrometheusLogger:
             verbose_logger.debug(traceback.format_exc())
             pass
 
+    def set_llm_deployment_failure_metrics(self, request_kwargs: dict):
+        try:
+            verbose_logger.debug("setting remaining tokens requests metric")
+            _response_headers = request_kwargs.get("response_headers")
+            _litellm_params = request_kwargs.get("litellm_params", {}) or {}
+            _metadata = _litellm_params.get("metadata", {})
+            litellm_model_name = request_kwargs.get("model", None)
+            api_base = _metadata.get("api_base", None)
+            llm_provider = _litellm_params.get("custom_llm_provider", None)
+            model_id = _metadata.get("model_id")
+
+            """
+            log these labels
+            ["litellm_model_name", "model_id", "api_base", "api_provider"]
+            """
+            self.deployment_partial_outage.labels(
+                litellm_model_name, model_id, api_base, llm_provider
+            ).set(1)
+
+            self.deployment_healthy.labels(
+                litellm_model_name, model_id, api_base, llm_provider
+            ).set(0)
+
+            pass
+        except:
+            pass
+
     def set_llm_deployment_success_metrics(self, request_kwargs: dict):
         try:
             verbose_logger.debug("setting remaining tokens requests metric")

litellm/proxy/proxy_config.yaml

@@ -3,7 +3,7 @@ model_list:
     litellm_params:
       model: openai/fake
       api_key: fake-key
-      api_base: https://exampleopenaiendpoint-production.up.railway.app/
+      api_base: https://exampleopenaiendpoint-production.up.railwaz.app/
   - model_name: fireworks-llama-v3-70b-instruct
     litellm_params:
       model: fireworks_ai/accounts/fireworks/models/llama-v3-70b-instruct
@@ -50,4 +50,6 @@ general_settings:
 
 
 litellm_settings:
-  callbacks: ["otel"] # 👈 KEY CHANGE
+  callbacks: ["otel"] # 👈 KEY CHANGE
+  success_callback: ["prometheus"]
+  failure_callback: ["prometheus"]