diff --git a/litellm/integrations/prometheus.py b/litellm/integrations/prometheus.py index 66e64aa2f7d..0865f64eed5 100644 --- a/litellm/integrations/prometheus.py +++ b/litellm/integrations/prometheus.py @@ -279,6 +279,8 @@ class PrometheusLogger: user_api_team_alias, user_id, ).inc() + + self.set_llm_deployment_failure_metrics(kwargs) except Exception as e: verbose_logger.error( "prometheus Layer Error(): Exception occured - {}".format(str(e)) @@ -286,6 +288,33 @@ class PrometheusLogger: verbose_logger.debug(traceback.format_exc()) pass + def set_llm_deployment_failure_metrics(self, request_kwargs: dict): + try: + verbose_logger.debug("setting remaining tokens requests metric") + _response_headers = request_kwargs.get("response_headers") + _litellm_params = request_kwargs.get("litellm_params", {}) or {} + _metadata = _litellm_params.get("metadata", {}) + litellm_model_name = request_kwargs.get("model", None) + api_base = _metadata.get("api_base", None) + llm_provider = _litellm_params.get("custom_llm_provider", None) + model_id = _metadata.get("model_id") + + """ + log these labels + ["litellm_model_name", "model_id", "api_base", "api_provider"] + """ + self.deployment_partial_outage.labels( + litellm_model_name, model_id, api_base, llm_provider + ).set(1) + + self.deployment_healthy.labels( + litellm_model_name, model_id, api_base, llm_provider + ).set(0) + + pass + except: + pass + def set_llm_deployment_success_metrics(self, request_kwargs: dict): try: verbose_logger.debug("setting remaining tokens requests metric") diff --git a/litellm/proxy/proxy_config.yaml b/litellm/proxy/proxy_config.yaml index 97cd407d324..36b191c90a5 100644 --- a/litellm/proxy/proxy_config.yaml +++ b/litellm/proxy/proxy_config.yaml @@ -3,7 +3,7 @@ model_list: litellm_params: model: openai/fake api_key: fake-key - api_base: https://exampleopenaiendpoint-production.up.railway.app/ + api_base: https://exampleopenaiendpoint-production.up.railwaz.app/ - model_name: fireworks-llama-v3-70b-instruct litellm_params: model: fireworks_ai/accounts/fireworks/models/llama-v3-70b-instruct @@ -50,4 +50,6 @@ general_settings: litellm_settings: - callbacks: ["otel"] # 👈 KEY CHANGE \ No newline at end of file + callbacks: ["otel"] # 👈 KEY CHANGE + success_callback: ["prometheus"] + failure_callback: ["prometheus"] \ No newline at end of file