mirror of
https://github.com/BerriAI/litellm.git
synced 2026-08-28 05:25:59 +00:00
feat(alerting): add native Microsoft Teams alerting destination (#38367)
* feat(alerting): add native Microsoft Teams alerting destination Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(alerting): preserve active destinations on MS Teams save and confirm health test delivery Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(ui): read persisted alerting destinations at MS Teams save time Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: yassin <yassin@berri.ai> Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
devin-ai-integration[bot]
GitHub
parent
44d84360fb
commit
d392e7faae
12 changed files with 627 additions and 164 deletions
|
|
@ -10,6 +10,8 @@ from typing import TYPE_CHECKING, Any, Final
|
|||
|
||||
from litellm._logging import verbose_proxy_logger
|
||||
|
||||
from .ms_teams import MS_TEAMS_ALERTING_DESTINATION, build_ms_teams_payload
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from .slack_alerting import SlackAlerting as _SlackAlerting
|
||||
|
||||
|
|
@ -62,14 +64,17 @@ async def send_to_webhook(slackAlertingInstance: SlackAlertingType, item, count)
|
|||
if count > 1:
|
||||
payload["text"] = f"[Num Alerts: {count}]\n\n{payload['text']}"
|
||||
|
||||
request_body: Final = (
|
||||
build_ms_teams_payload(payload["text"]) if item.get("format") == MS_TEAMS_ALERTING_DESTINATION else payload
|
||||
)
|
||||
response: Final = await slackAlertingInstance.async_http_handler.post(
|
||||
url=item["url"],
|
||||
headers=item["headers"],
|
||||
data=json.dumps(payload),
|
||||
data=json.dumps(request_body),
|
||||
)
|
||||
if response.status_code != 200:
|
||||
verbose_proxy_logger.debug("Error sending slack alert to url=%s. Error=%s", item["url"], response.text)
|
||||
verbose_proxy_logger.debug("Error sending alert to url=%s. Error=%s", item["url"], response.text)
|
||||
except Exception as e:
|
||||
verbose_proxy_logger.debug("Error sending slack alert: %s", e)
|
||||
verbose_proxy_logger.debug("Error sending alert: %s", e)
|
||||
finally:
|
||||
_print_alerting_payload_warning(payload, slackAlertingInstance=slackAlertingInstance)
|
||||
|
|
|
|||
75
litellm/integrations/SlackAlerting/ms_teams.py
Normal file
75
litellm/integrations/SlackAlerting/ms_teams.py
Normal file
|
|
@ -0,0 +1,75 @@
|
|||
"""Microsoft Teams alert delivery helpers.
|
||||
|
||||
Teams incoming webhooks (Workflows and legacy connectors) accept an Adaptive
|
||||
Card wrapped in a message attachment, so alert text is delivered as a single
|
||||
wrapped TextBlock.
|
||||
"""
|
||||
|
||||
import os
|
||||
from collections.abc import Mapping
|
||||
from types import MappingProxyType
|
||||
from typing import Final
|
||||
|
||||
from typing_extensions import ReadOnly, TypedDict
|
||||
|
||||
from litellm.types.integrations.slack_alerting import AlertType
|
||||
|
||||
MS_TEAMS_WEBHOOK_URL_ENV: Final = "MS_TEAMS_WEBHOOK_URL"
|
||||
|
||||
MS_TEAMS_ALERTING_DESTINATION: Final = "ms_teams"
|
||||
|
||||
MS_TEAMS_ALERT_HEADERS: Final[Mapping[str, str]] = MappingProxyType({"Content-type": "application/json"})
|
||||
|
||||
|
||||
class MSTeamsTextBlock(TypedDict):
|
||||
type: ReadOnly[str]
|
||||
text: ReadOnly[str]
|
||||
wrap: ReadOnly[bool]
|
||||
|
||||
|
||||
class MSTeamsAdaptiveCard(TypedDict):
|
||||
type: ReadOnly[str]
|
||||
version: ReadOnly[str]
|
||||
body: ReadOnly[tuple[MSTeamsTextBlock, ...]]
|
||||
|
||||
|
||||
class MSTeamsAttachment(TypedDict):
|
||||
contentType: ReadOnly[str]
|
||||
content: ReadOnly[MSTeamsAdaptiveCard]
|
||||
|
||||
|
||||
class MSTeamsMessage(TypedDict):
|
||||
type: ReadOnly[str]
|
||||
attachments: ReadOnly[tuple[MSTeamsAttachment, ...]]
|
||||
|
||||
|
||||
class MSTeamsAlertText(TypedDict):
|
||||
text: ReadOnly[str]
|
||||
|
||||
|
||||
class MSTeamsQueueItem(TypedDict):
|
||||
url: ReadOnly[str]
|
||||
headers: ReadOnly[Mapping[str, str]]
|
||||
payload: ReadOnly[MSTeamsAlertText]
|
||||
alert_type: ReadOnly[AlertType]
|
||||
format: ReadOnly[str]
|
||||
|
||||
|
||||
def get_ms_teams_webhook_url() -> str | None:
|
||||
return os.getenv(MS_TEAMS_WEBHOOK_URL_ENV)
|
||||
|
||||
|
||||
def build_ms_teams_payload(text: str) -> MSTeamsMessage:
|
||||
return MSTeamsMessage(
|
||||
type="message",
|
||||
attachments=(
|
||||
MSTeamsAttachment(
|
||||
contentType="application/vnd.microsoft.card.adaptive",
|
||||
content=MSTeamsAdaptiveCard(
|
||||
type="AdaptiveCard",
|
||||
version="1.4",
|
||||
body=(MSTeamsTextBlock(type="TextBlock", text=text, wrap=True),),
|
||||
),
|
||||
),
|
||||
),
|
||||
)
|
||||
|
|
@ -57,6 +57,13 @@ from litellm.types.proxy.model_deprecation import (
|
|||
|
||||
from ..email_templates.templates import *
|
||||
from .batching_handler import send_to_webhook, squash_payloads
|
||||
from .ms_teams import (
|
||||
MS_TEAMS_ALERT_HEADERS,
|
||||
MS_TEAMS_ALERTING_DESTINATION,
|
||||
MSTeamsAlertText,
|
||||
MSTeamsQueueItem,
|
||||
get_ms_teams_webhook_url,
|
||||
)
|
||||
from .utils import process_slack_alerting_variables
|
||||
|
||||
if TYPE_CHECKING:
|
||||
|
|
@ -1431,13 +1438,45 @@ Model Info:
|
|||
# only send budget alerts over Email
|
||||
await self.send_email_alert_using_smtp(webhook_event=user_info, alert_type=alert_type)
|
||||
|
||||
if "slack" not in self.alerting:
|
||||
send_to_slack: Final = "slack" in self.alerting
|
||||
send_to_ms_teams: Final = MS_TEAMS_ALERTING_DESTINATION in self.alerting
|
||||
if not send_to_slack and not send_to_ms_teams:
|
||||
return
|
||||
if alert_type not in self.alert_types:
|
||||
return
|
||||
|
||||
from datetime import datetime
|
||||
|
||||
# Get the current timestamp
|
||||
current_time: Final = datetime.now().strftime("%H:%M:%S")
|
||||
_proxy_base_url: Final = os.getenv("PROXY_BASE_URL", None)
|
||||
# Use .name if it's an enum, otherwise use as is
|
||||
alert_type_name: Final = getattr(alert_type, "name", alert_type)
|
||||
alert_type_formatted: Final = f"Alert type: `{alert_type_name}`"
|
||||
if alert_type == "daily_reports" or alert_type == "new_model_added":
|
||||
formatted_message = alert_type_formatted + message
|
||||
else:
|
||||
formatted_message = (
|
||||
f"{alert_type_formatted}\nLevel: `{level}`\nTimestamp: `{current_time}`\n\nMessage: {message}"
|
||||
)
|
||||
|
||||
if kwargs:
|
||||
for key, value in kwargs.items():
|
||||
formatted_message += f"\n\n{key}: `{value}`\n\n"
|
||||
if alerting_metadata:
|
||||
for key, value in alerting_metadata.items():
|
||||
formatted_message += f"\n\n*Alerting Metadata*: \n{key}: `{value}`\n\n"
|
||||
if _proxy_base_url is not None:
|
||||
formatted_message += f"\n\nProxy URL: `{_proxy_base_url}`"
|
||||
|
||||
if send_to_ms_teams:
|
||||
self._enqueue_ms_teams_alert(formatted_message=formatted_message, alert_type=alert_type)
|
||||
|
||||
if not send_to_slack:
|
||||
if len(self.log_queue) >= self.batch_size:
|
||||
await self.flush_queue()
|
||||
return
|
||||
|
||||
# Check if digest mode is enabled for this alert type
|
||||
alert_type_name_str: Final = getattr(alert_type, "value", str(alert_type))
|
||||
_atc: Final = self.alert_type_config.get(alert_type_name_str)
|
||||
|
|
@ -1473,28 +1512,6 @@ Model Info:
|
|||
)
|
||||
return # Suppress immediate alert; will be emitted by _flush_digest_buckets
|
||||
|
||||
# Get the current timestamp
|
||||
current_time: Final = datetime.now().strftime("%H:%M:%S")
|
||||
_proxy_base_url: Final = os.getenv("PROXY_BASE_URL", None)
|
||||
# Use .name if it's an enum, otherwise use as is
|
||||
alert_type_name: Final = getattr(alert_type, "name", alert_type)
|
||||
alert_type_formatted: Final = f"Alert type: `{alert_type_name}`"
|
||||
if alert_type == "daily_reports" or alert_type == "new_model_added":
|
||||
formatted_message = alert_type_formatted + message
|
||||
else:
|
||||
formatted_message = (
|
||||
f"{alert_type_formatted}\nLevel: `{level}`\nTimestamp: `{current_time}`\n\nMessage: {message}"
|
||||
)
|
||||
|
||||
if kwargs:
|
||||
for key, value in kwargs.items():
|
||||
formatted_message += f"\n\n{key}: `{value}`\n\n"
|
||||
if alerting_metadata:
|
||||
for key, value in alerting_metadata.items():
|
||||
formatted_message += f"\n\n*Alerting Metadata*: \n{key}: `{value}`\n\n"
|
||||
if _proxy_base_url is not None:
|
||||
formatted_message += f"\n\nProxy URL: `{_proxy_base_url}`"
|
||||
|
||||
# check if we find the slack webhook url in self.alert_to_webhook_url
|
||||
if self.alert_to_webhook_url is not None and alert_type in self.alert_to_webhook_url:
|
||||
slack_webhook_url: str | list[str] | None = self.alert_to_webhook_url[alert_type]
|
||||
|
|
@ -1531,6 +1548,24 @@ Model Info:
|
|||
if len(self.log_queue) >= self.batch_size:
|
||||
await self.flush_queue()
|
||||
|
||||
def _enqueue_ms_teams_alert(self, formatted_message: str, alert_type: AlertType) -> None:
|
||||
ms_teams_webhook_url: Final = get_ms_teams_webhook_url()
|
||||
if ms_teams_webhook_url is None:
|
||||
verbose_proxy_logger.error(
|
||||
"MS Teams alerting is enabled but MS_TEAMS_WEBHOOK_URL is not set. Dropping alert type=%s",
|
||||
alert_type,
|
||||
)
|
||||
return
|
||||
payload: Final[MSTeamsAlertText] = {"text": formatted_message}
|
||||
item: Final[MSTeamsQueueItem] = {
|
||||
"url": ms_teams_webhook_url,
|
||||
"headers": MS_TEAMS_ALERT_HEADERS,
|
||||
"payload": payload,
|
||||
"alert_type": alert_type,
|
||||
"format": MS_TEAMS_ALERTING_DESTINATION,
|
||||
}
|
||||
self.log_queue.append(item)
|
||||
|
||||
async def async_send_batch(self):
|
||||
if not self.log_queue:
|
||||
return
|
||||
|
|
|
|||
|
|
@ -25,3 +25,7 @@ EMAIL_DESCRIPTORS: Final[tuple[FieldDescriptor, ...]] = (
|
|||
SLACK_DESCRIPTORS: Final[tuple[FieldDescriptor, ...]] = (
|
||||
FieldDescriptor("SLACK_WEBHOOK_URL", "SLACK_WEBHOOK_URL", "SLACK_WEBHOOK_URL", is_secret=True),
|
||||
)
|
||||
|
||||
MS_TEAMS_DESCRIPTORS: Final[tuple[FieldDescriptor, ...]] = (
|
||||
FieldDescriptor("MS_TEAMS_WEBHOOK_URL", "MS_TEAMS_WEBHOOK_URL", "MS_TEAMS_WEBHOOK_URL", is_secret=True),
|
||||
)
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
import asyncio
|
||||
import copy
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import secrets
|
||||
|
|
@ -11,10 +12,16 @@ from typing import Any, Final, Literal, TypedDict, cast
|
|||
|
||||
import fastapi
|
||||
from fastapi import APIRouter, Depends, HTTPException, Request, Response, status
|
||||
from typing_extensions import ReadOnly
|
||||
|
||||
import litellm
|
||||
from litellm._logging import verbose_logger, verbose_proxy_logger
|
||||
from litellm.constants import HEALTH_CHECK_TIMEOUT_SECONDS
|
||||
from litellm.integrations.SlackAlerting.ms_teams import (
|
||||
MS_TEAMS_ALERT_HEADERS,
|
||||
build_ms_teams_payload,
|
||||
get_ms_teams_webhook_url,
|
||||
)
|
||||
from litellm.litellm_core_utils.custom_logger_registry import CustomLoggerRegistry
|
||||
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
|
||||
from litellm.proxy._types import (
|
||||
|
|
@ -164,6 +171,7 @@ services = (
|
|||
"langfuse",
|
||||
"langfuse_otel",
|
||||
"slack",
|
||||
"ms_teams",
|
||||
"openmeter",
|
||||
"webhook",
|
||||
"email",
|
||||
|
|
@ -180,6 +188,15 @@ services = (
|
|||
)
|
||||
|
||||
|
||||
class _ServiceTestErrorDetail(TypedDict):
|
||||
error: ReadOnly[str]
|
||||
|
||||
|
||||
class _ServiceTestSuccessResponse(TypedDict):
|
||||
status: ReadOnly[str]
|
||||
message: ReadOnly[str]
|
||||
|
||||
|
||||
@router.get(
|
||||
"/test",
|
||||
tags=["health"],
|
||||
|
|
@ -238,6 +255,7 @@ async def health_services_endpoint(
|
|||
"langfuse",
|
||||
"langfuse_otel",
|
||||
"slack",
|
||||
"ms_teams",
|
||||
"openmeter",
|
||||
"webhook",
|
||||
"braintrust",
|
||||
|
|
@ -448,6 +466,38 @@ async def health_services_endpoint(
|
|||
status_code=422,
|
||||
detail={"error": f'"{service}" not in proxy config: general_settings. Unable to test this.'},
|
||||
)
|
||||
if service == "ms_teams":
|
||||
if "ms_teams" not in general_settings.get("alerting", ()):
|
||||
not_configured_detail: Final[_ServiceTestErrorDetail] = {
|
||||
"error": f'"{service}" not in proxy config: general_settings. Unable to test this.'
|
||||
}
|
||||
raise HTTPException(status_code=422, detail=not_configured_detail)
|
||||
ms_teams_webhook_url: Final = get_ms_teams_webhook_url()
|
||||
if ms_teams_webhook_url is None:
|
||||
missing_webhook_detail: Final[_ServiceTestErrorDetail] = {
|
||||
"error": "MS_TEAMS_WEBHOOK_URL not set. Unable to test this."
|
||||
}
|
||||
raise HTTPException(status_code=422, detail=missing_webhook_detail)
|
||||
ms_teams_test_message: Final = (
|
||||
f"Alert type: `{AlertType.budget_alerts.value}`\nLevel: `Low`\n"
|
||||
f"Timestamp: `{datetime.now().strftime('%H:%M:%S')}`\n\n"
|
||||
"Message: This is a test MS Teams alert message"
|
||||
)
|
||||
ms_teams_response: Final = await proxy_logging_obj.slack_alerting_instance.async_http_handler.post(
|
||||
url=ms_teams_webhook_url,
|
||||
headers=dict(MS_TEAMS_ALERT_HEADERS), # mutable-ok: async_http_handler.post only accepts dict headers
|
||||
data=json.dumps(build_ms_teams_payload(ms_teams_test_message)),
|
||||
)
|
||||
if ms_teams_response.status_code >= 400:
|
||||
delivery_failed_detail: Final[_ServiceTestErrorDetail] = {
|
||||
"error": f"MS Teams webhook returned status {ms_teams_response.status_code}: {ms_teams_response.text}"
|
||||
}
|
||||
raise HTTPException(status_code=500, detail=delivery_failed_detail)
|
||||
ms_teams_success: Final[_ServiceTestSuccessResponse] = {
|
||||
"status": "success",
|
||||
"message": "Mock MS Teams Alert sent, verify MS Teams Alert Received in your channel",
|
||||
}
|
||||
return ms_teams_success
|
||||
if service == "email":
|
||||
webhook_event: Final = WebhookEvent(
|
||||
event="key_created",
|
||||
|
|
|
|||
|
|
@ -40,7 +40,7 @@ import anyio
|
|||
import websockets
|
||||
import websockets.exceptions
|
||||
from pydantic import BaseModel, Json, JsonValue
|
||||
from typing_extensions import NotRequired, assert_never
|
||||
from typing_extensions import NotRequired, ReadOnly, assert_never
|
||||
|
||||
from litellm._uuid import uuid
|
||||
from litellm.constants import (
|
||||
|
|
@ -381,6 +381,7 @@ from litellm.proxy.common_utils.user_api_key_cache import (
|
|||
from litellm.proxy.config_resolvers import resolve_fields
|
||||
from litellm.proxy.config_resolvers.alerting import (
|
||||
EMAIL_DESCRIPTORS,
|
||||
MS_TEAMS_DESCRIPTORS,
|
||||
SLACK_DESCRIPTORS,
|
||||
)
|
||||
from litellm.proxy.container_endpoints.endpoints import router as container_router
|
||||
|
|
@ -16300,6 +16301,11 @@ def _apply_callback_role_gate(entries: list, is_full_admin: bool) -> list:
|
|||
return [{**entry, "variables": _redact_callback_env_vars(entry.get("variables") or {})} for entry in entries]
|
||||
|
||||
|
||||
class _AlertingDestinationEntry(TypedDict):
|
||||
name: ReadOnly[str]
|
||||
variables: ReadOnly[Mapping[str, str | None]]
|
||||
|
||||
|
||||
def _apply_alerting_env_role_gate(env_vars: dict, is_full_admin: bool) -> dict:
|
||||
if is_full_admin:
|
||||
return mask_sensitive_keys(env_vars, _ALERTING_SENSITIVE_VARS)
|
||||
|
|
@ -16949,6 +16955,17 @@ async def get_config(
|
|||
}
|
||||
)
|
||||
|
||||
_ms_teams_values, _ = resolve_fields(
|
||||
MS_TEAMS_DESCRIPTORS, environment_variables, os.environ, empty_db_is_set=True
|
||||
)
|
||||
_ms_teams_env_vars: Final = _apply_alerting_env_role_gate(_ms_teams_values, is_full_admin)
|
||||
|
||||
ms_teams_alerting_entry: Final[_AlertingDestinationEntry] = {
|
||||
"name": "ms_teams",
|
||||
"variables": _ms_teams_env_vars,
|
||||
}
|
||||
alerting_data.append(ms_teams_alerting_entry)
|
||||
|
||||
if llm_router is None:
|
||||
_router_settings = {}
|
||||
else:
|
||||
|
|
@ -16958,6 +16975,7 @@ async def get_config(
|
|||
"status": "success",
|
||||
"callbacks": _data_to_return,
|
||||
"alerts": alerting_data,
|
||||
"active_alerting_destinations": tuple(_alerting),
|
||||
"router_settings": _router_settings,
|
||||
"available_callbacks": all_available_callbacks,
|
||||
}
|
||||
|
|
|
|||
|
|
@ -765,7 +765,7 @@ class ProxyLogging:
|
|||
alert_type_config=alert_type_config,
|
||||
)
|
||||
|
||||
if self.alerting is not None and "slack" in self.alerting:
|
||||
if self.alerting is not None and ("slack" in self.alerting or "ms_teams" in self.alerting):
|
||||
# NOTE: ENSURE we only add callbacks when alerting is on
|
||||
# We should NOT add callbacks when alerting is off
|
||||
if (
|
||||
|
|
@ -2236,7 +2236,7 @@ class ProxyLogging:
|
|||
# do nothing if alerting is not switched on (unless it's a soft_budget alert with team-specific emails)
|
||||
return
|
||||
|
||||
if self.alerting is not None and "slack" in self.alerting:
|
||||
if self.alerting is not None and ("slack" in self.alerting or "ms_teams" in self.alerting):
|
||||
if self.slack_alerting_instance is not None:
|
||||
await self.slack_alerting_instance.budget_alerts(
|
||||
type=type,
|
||||
|
|
@ -2301,17 +2301,17 @@ class ProxyLogging:
|
|||
and isinstance(request_data["metadata"]["alerting_metadata"], dict)
|
||||
):
|
||||
alerting_metadata = request_data["metadata"]["alerting_metadata"]
|
||||
if "slack" in self.alerting or "ms_teams" in self.alerting:
|
||||
await self.slack_alerting_instance.send_alert(
|
||||
message=message,
|
||||
level=level,
|
||||
alert_type=alert_type,
|
||||
user_info=None,
|
||||
alerting_metadata=alerting_metadata,
|
||||
**extra_kwargs,
|
||||
)
|
||||
for client in self.alerting:
|
||||
if client == "slack":
|
||||
await self.slack_alerting_instance.send_alert(
|
||||
message=message,
|
||||
level=level,
|
||||
alert_type=alert_type,
|
||||
user_info=None,
|
||||
alerting_metadata=alerting_metadata,
|
||||
**extra_kwargs,
|
||||
)
|
||||
elif client == "sentry":
|
||||
if client == "sentry":
|
||||
if litellm.utils.sentry_sdk_instance is not None:
|
||||
litellm.utils.sentry_sdk_instance.capture_message(formatted_message)
|
||||
else:
|
||||
|
|
|
|||
122
tests/test_litellm/integrations/SlackAlerting/test_ms_teams.py
Normal file
122
tests/test_litellm/integrations/SlackAlerting/test_ms_teams.py
Normal file
|
|
@ -0,0 +1,122 @@
|
|||
import json
|
||||
from typing import Final
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
from litellm.integrations.SlackAlerting.batching_handler import send_to_webhook
|
||||
from litellm.integrations.SlackAlerting.ms_teams import (
|
||||
MS_TEAMS_ALERTING_DESTINATION,
|
||||
MS_TEAMS_WEBHOOK_URL_ENV,
|
||||
build_ms_teams_payload,
|
||||
get_ms_teams_webhook_url,
|
||||
)
|
||||
from litellm.integrations.SlackAlerting.slack_alerting import SlackAlerting
|
||||
from litellm.proxy._types import AlertType
|
||||
|
||||
|
||||
def test_build_ms_teams_payload_wraps_text_in_adaptive_card():
|
||||
payload: Final = build_ms_teams_payload("hello alert")
|
||||
assert payload["type"] == "message"
|
||||
attachment: Final = payload["attachments"][0]
|
||||
assert attachment["contentType"] == "application/vnd.microsoft.card.adaptive"
|
||||
card: Final = attachment["content"]
|
||||
assert card["type"] == "AdaptiveCard"
|
||||
assert card["body"] == ({"type": "TextBlock", "text": "hello alert", "wrap": True},)
|
||||
|
||||
|
||||
def test_get_ms_teams_webhook_url_reads_env(monkeypatch):
|
||||
monkeypatch.setenv(MS_TEAMS_WEBHOOK_URL_ENV, "https://teams.example/webhook")
|
||||
assert get_ms_teams_webhook_url() == "https://teams.example/webhook"
|
||||
monkeypatch.delenv(MS_TEAMS_WEBHOOK_URL_ENV)
|
||||
assert get_ms_teams_webhook_url() is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_send_alert_enqueues_ms_teams_item(monkeypatch):
|
||||
monkeypatch.setenv(MS_TEAMS_WEBHOOK_URL_ENV, "https://teams.example/webhook")
|
||||
slack_alerting: Final = SlackAlerting(alerting=["ms_teams"])
|
||||
await slack_alerting.send_alert(
|
||||
message="proxy is down",
|
||||
level="High",
|
||||
alert_type=AlertType.db_exceptions,
|
||||
alerting_metadata={},
|
||||
)
|
||||
assert len(slack_alerting.log_queue) == 1
|
||||
item: Final = slack_alerting.log_queue[0]
|
||||
assert item["url"] == "https://teams.example/webhook"
|
||||
assert item["format"] == MS_TEAMS_ALERTING_DESTINATION
|
||||
assert item["alert_type"] == AlertType.db_exceptions
|
||||
assert "proxy is down" in item["payload"]["text"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_send_alert_ms_teams_missing_webhook_drops_alert(monkeypatch):
|
||||
monkeypatch.delenv(MS_TEAMS_WEBHOOK_URL_ENV, raising=False)
|
||||
slack_alerting: Final = SlackAlerting(alerting=["ms_teams"])
|
||||
await slack_alerting.send_alert(
|
||||
message="proxy is down",
|
||||
level="High",
|
||||
alert_type=AlertType.db_exceptions,
|
||||
alerting_metadata={},
|
||||
)
|
||||
assert len(slack_alerting.log_queue) == 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_send_alert_slack_and_ms_teams_enqueue_both(monkeypatch):
|
||||
monkeypatch.setenv(MS_TEAMS_WEBHOOK_URL_ENV, "https://teams.example/webhook")
|
||||
monkeypatch.setenv("SLACK_WEBHOOK_URL", "https://hooks.slack.com/services/test")
|
||||
slack_alerting: Final = SlackAlerting(alerting=["slack", "ms_teams"])
|
||||
await slack_alerting.send_alert(
|
||||
message="proxy is down",
|
||||
level="High",
|
||||
alert_type=AlertType.db_exceptions,
|
||||
alerting_metadata={},
|
||||
)
|
||||
urls: Final = sorted(item["url"] for item in slack_alerting.log_queue)
|
||||
assert urls == ["https://hooks.slack.com/services/test", "https://teams.example/webhook"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_send_to_webhook_posts_adaptive_card_for_ms_teams_items():
|
||||
slack_alerting: Final = SlackAlerting(alerting=["ms_teams"])
|
||||
mock_response: Final = MagicMock()
|
||||
mock_response.status_code = 200
|
||||
slack_alerting.async_http_handler = MagicMock()
|
||||
slack_alerting.async_http_handler.post = AsyncMock(return_value=mock_response)
|
||||
|
||||
item: Final = {
|
||||
"url": "https://teams.example/webhook",
|
||||
"headers": {"Content-type": "application/json"},
|
||||
"payload": {"text": "alert body"},
|
||||
"alert_type": AlertType.db_exceptions,
|
||||
"format": MS_TEAMS_ALERTING_DESTINATION,
|
||||
}
|
||||
await send_to_webhook(slackAlertingInstance=slack_alerting, item=item, count=1)
|
||||
|
||||
call_kwargs: Final = slack_alerting.async_http_handler.post.call_args.kwargs
|
||||
assert call_kwargs["url"] == "https://teams.example/webhook"
|
||||
sent_body: Final = json.loads(call_kwargs["data"])
|
||||
assert sent_body["type"] == "message"
|
||||
assert sent_body["attachments"][0]["content"]["body"][0]["text"] == "alert body"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_send_to_webhook_keeps_slack_payload_shape():
|
||||
slack_alerting: Final = SlackAlerting(alerting=["slack"])
|
||||
mock_response: Final = MagicMock()
|
||||
mock_response.status_code = 200
|
||||
slack_alerting.async_http_handler = MagicMock()
|
||||
slack_alerting.async_http_handler.post = AsyncMock(return_value=mock_response)
|
||||
|
||||
item: Final = {
|
||||
"url": "https://hooks.slack.com/services/test",
|
||||
"headers": {"Content-type": "application/json"},
|
||||
"payload": {"text": "alert body"},
|
||||
"alert_type": AlertType.db_exceptions,
|
||||
}
|
||||
await send_to_webhook(slackAlertingInstance=slack_alerting, item=item, count=1)
|
||||
|
||||
call_kwargs: Final = slack_alerting.async_http_handler.post.call_args.kwargs
|
||||
assert json.loads(call_kwargs["data"]) == {"text": "alert body"}
|
||||
|
|
@ -1,3 +1,4 @@
|
|||
import json
|
||||
import time
|
||||
from datetime import datetime, timedelta
|
||||
from types import SimpleNamespace
|
||||
|
|
@ -15,7 +16,7 @@ import litellm
|
|||
import litellm.proxy.health_endpoints._health_endpoints as _health_endpoints_module
|
||||
from litellm.litellm_core_utils.health_check_helpers import TEST_IMAGE_BASE64
|
||||
|
||||
from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth
|
||||
from litellm.proxy._types import LitellmUserRoles, ProxyException, UserAPIKeyAuth
|
||||
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
|
||||
from litellm.proxy.health_endpoints._health_endpoints import (
|
||||
_db_health_readiness_check,
|
||||
|
|
@ -144,9 +145,7 @@ async def test_db_health_transport_error_never_raises(transport_error):
|
|||
result = await _db_health_readiness_check()
|
||||
|
||||
assert result["status"] == "disconnected"
|
||||
mock_prisma.attempt_db_reconnect.assert_called_once_with(
|
||||
reason="health_readiness_check"
|
||||
)
|
||||
mock_prisma.attempt_db_reconnect.assert_called_once_with(reason="health_readiness_check")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
|
@ -176,9 +175,7 @@ async def test_db_health_transport_error_reconnect_succeeds(transport_error):
|
|||
result = await _db_health_readiness_check()
|
||||
|
||||
assert result["status"] == "connected"
|
||||
mock_prisma.attempt_db_reconnect.assert_called_once_with(
|
||||
reason="health_readiness_check"
|
||||
)
|
||||
mock_prisma.attempt_db_reconnect.assert_called_once_with(reason="health_readiness_check")
|
||||
assert mock_prisma.health_check.call_count == 2
|
||||
|
||||
|
||||
|
|
@ -198,9 +195,7 @@ async def test_db_health_transport_error_reconnect_fails(transport_error):
|
|||
"""
|
||||
mock_prisma = MagicMock()
|
||||
mock_prisma.health_check = AsyncMock(side_effect=transport_error)
|
||||
mock_prisma.attempt_db_reconnect = AsyncMock(
|
||||
side_effect=RuntimeError("reconnect failed")
|
||||
)
|
||||
mock_prisma.attempt_db_reconnect = AsyncMock(side_effect=RuntimeError("reconnect failed"))
|
||||
|
||||
_health_endpoints_module.db_health_cache = {
|
||||
"status": "connected",
|
||||
|
|
@ -252,9 +247,7 @@ async def test_health_services_endpoint_sqs(status, error_message):
|
|||
"""
|
||||
with patch("litellm.integrations.sqs.SQSLogger") as MockSQSLogger:
|
||||
mock_instance = MagicMock()
|
||||
mock_instance.async_health_check = AsyncMock(
|
||||
return_value={"status": status, "error_message": error_message}
|
||||
)
|
||||
mock_instance.async_health_check = AsyncMock(return_value={"status": status, "error_message": error_message})
|
||||
MockSQSLogger.return_value = mock_instance
|
||||
|
||||
result = await health_services_endpoint(service="sqs")
|
||||
|
|
@ -451,14 +444,9 @@ async def test_test_model_connection_loads_config_from_router():
|
|||
# Verify that config params were loaded and merged
|
||||
# Note: request params override config params, so model from request is used
|
||||
assert model_params.get("api_key") == "resolved-api-key-from-env"
|
||||
assert (
|
||||
model_params.get("api_base")
|
||||
== "https://resolved-endpoint.openai.azure.com/"
|
||||
)
|
||||
assert model_params.get("api_base") == "https://resolved-endpoint.openai.azure.com/"
|
||||
assert model_params.get("api_version") == "2024-10-21"
|
||||
assert (
|
||||
model_params.get("model") == "gpt-4o"
|
||||
) # Request param overrides config param
|
||||
assert model_params.get("model") == "gpt-4o" # Request param overrides config param
|
||||
|
||||
# Verify result
|
||||
assert result["status"] == "success"
|
||||
|
|
@ -594,9 +582,7 @@ async def test_test_model_connection_uses_model_info_id_to_disambiguate_duplicat
|
|||
assert ahealth_check_call_args is not None
|
||||
model_params = ahealth_check_call_args.kwargs.get("model_params", {})
|
||||
|
||||
assert model_params.get("api_base") == (
|
||||
"https://deployment-B-base.invalid/v1"
|
||||
), (
|
||||
assert model_params.get("api_base") == ("https://deployment-B-base.invalid/v1"), (
|
||||
"Expected /health/test_connection to probe deployment B's "
|
||||
"api_base when model_info.id='deployment-B-id' was provided. "
|
||||
f"Got: {model_params.get('api_base')!r}. This means the "
|
||||
|
|
@ -771,14 +757,10 @@ async def test_test_model_connection_uses_loaded_deployment_team_id():
|
|||
"can_user_make_model_call",
|
||||
wraps=ModelManagementAuthChecks.can_user_make_model_call,
|
||||
) as spy_auth_check,
|
||||
patch(
|
||||
"litellm.proxy.management_endpoints.model_management_endpoints.TeamRepository"
|
||||
) as MockTeamRepo,
|
||||
patch("litellm.proxy.management_endpoints.model_management_endpoints.TeamRepository") as MockTeamRepo,
|
||||
):
|
||||
mock_team_repo_instance = MagicMock()
|
||||
mock_team_repo_instance.table.find_unique = AsyncMock(
|
||||
side_effect=fake_find_unique
|
||||
)
|
||||
mock_team_repo_instance.table.find_unique = AsyncMock(side_effect=fake_find_unique)
|
||||
MockTeamRepo.return_value = mock_team_repo_instance
|
||||
|
||||
with pytest.raises(HTTPException) as exc_info:
|
||||
|
|
@ -873,14 +855,10 @@ async def test_test_model_connection_uses_loaded_deployment_team_id_via_model_na
|
|||
"can_user_make_model_call",
|
||||
wraps=ModelManagementAuthChecks.can_user_make_model_call,
|
||||
) as spy_auth_check,
|
||||
patch(
|
||||
"litellm.proxy.management_endpoints.model_management_endpoints.TeamRepository"
|
||||
) as MockTeamRepo,
|
||||
patch("litellm.proxy.management_endpoints.model_management_endpoints.TeamRepository") as MockTeamRepo,
|
||||
):
|
||||
mock_team_repo_instance = MagicMock()
|
||||
mock_team_repo_instance.table.find_unique = AsyncMock(
|
||||
side_effect=fake_find_unique
|
||||
)
|
||||
mock_team_repo_instance.table.find_unique = AsyncMock(side_effect=fake_find_unique)
|
||||
MockTeamRepo.return_value = mock_team_repo_instance
|
||||
|
||||
with pytest.raises(HTTPException) as exc_info:
|
||||
|
|
@ -920,9 +898,7 @@ async def test_test_model_connection_authorizes_on_params_after_health_check_par
|
|||
from litellm.types.router import Deployment
|
||||
|
||||
marker = "sentinel-from-health-check-params"
|
||||
mock_can_user_make_model_call = AsyncMock(
|
||||
side_effect=HTTPException(status_code=403, detail="denied")
|
||||
)
|
||||
mock_can_user_make_model_call = AsyncMock(side_effect=HTTPException(status_code=403, detail="denied"))
|
||||
|
||||
with (
|
||||
patch( # test-quality-ok: proxy module global, no injection seam
|
||||
|
|
@ -1005,9 +981,7 @@ async def test_test_model_connection_authorized_team_admin_passes_real_auth():
|
|||
return SimpleNamespace(
|
||||
model_dump=lambda: LiteLLM_TeamTable(
|
||||
team_id=owner_team_id,
|
||||
members_with_roles=[
|
||||
{"user_id": owner_admin_user_id, "role": "admin"}
|
||||
],
|
||||
members_with_roles=[{"user_id": owner_admin_user_id, "role": "admin"}],
|
||||
).model_dump()
|
||||
)
|
||||
return None
|
||||
|
|
@ -1023,9 +997,7 @@ async def test_test_model_connection_authorized_team_admin_passes_real_auth():
|
|||
"can_user_make_model_call",
|
||||
wraps=ModelManagementAuthChecks.can_user_make_model_call,
|
||||
) as spy_auth_check,
|
||||
patch(
|
||||
"litellm.proxy.management_endpoints.model_management_endpoints.TeamRepository"
|
||||
) as MockTeamRepo,
|
||||
patch("litellm.proxy.management_endpoints.model_management_endpoints.TeamRepository") as MockTeamRepo,
|
||||
patch(
|
||||
"litellm.proxy.health_endpoints._health_endpoints.litellm.ahealth_check",
|
||||
AsyncMock(return_value=health_result),
|
||||
|
|
@ -1036,9 +1008,7 @@ async def test_test_model_connection_authorized_team_admin_passes_real_auth():
|
|||
),
|
||||
):
|
||||
mock_team_repo_instance = MagicMock()
|
||||
mock_team_repo_instance.table.find_unique = AsyncMock(
|
||||
side_effect=fake_find_unique
|
||||
)
|
||||
mock_team_repo_instance.table.find_unique = AsyncMock(side_effect=fake_find_unique)
|
||||
MockTeamRepo.return_value = mock_team_repo_instance
|
||||
|
||||
result = await health_test_model_connection(
|
||||
|
|
@ -1065,9 +1035,7 @@ async def test_test_model_connection_authorized_team_admin_passes_real_auth():
|
|||
async def test_health_services_endpoint_galileo(status, error_message):
|
||||
with patch("litellm.integrations.galileo.GalileoObserve") as MockGalileoObserve:
|
||||
mock_instance = MagicMock()
|
||||
mock_instance.async_health_check = AsyncMock(
|
||||
return_value={"status": status, "error_message": error_message}
|
||||
)
|
||||
mock_instance.async_health_check = AsyncMock(return_value={"status": status, "error_message": error_message})
|
||||
MockGalileoObserve.return_value = mock_instance
|
||||
|
||||
result = await health_services_endpoint(service="galileo")
|
||||
|
|
@ -1140,13 +1108,9 @@ async def test_health_services_endpoint_newrelic_blocks_non_admin(role):
|
|||
user_role=role,
|
||||
)
|
||||
|
||||
with patch(
|
||||
"litellm.integrations.newrelic.newrelic.NewRelicLogger"
|
||||
) as MockNewRelicLogger:
|
||||
with patch("litellm.integrations.newrelic.newrelic.NewRelicLogger") as MockNewRelicLogger:
|
||||
mock_instance = MagicMock()
|
||||
mock_instance.async_health_check = AsyncMock(
|
||||
return_value={"status": "healthy", "error_message": ""}
|
||||
)
|
||||
mock_instance.async_health_check = AsyncMock(return_value={"status": "healthy", "error_message": ""})
|
||||
MockNewRelicLogger.return_value = mock_instance
|
||||
|
||||
with pytest.raises(ProxyException) as exc_info:
|
||||
|
|
@ -1175,13 +1139,9 @@ async def test_health_services_endpoint_newrelic_allows_proxy_admin(admin_role):
|
|||
user_role=admin_role,
|
||||
)
|
||||
|
||||
with patch(
|
||||
"litellm.integrations.newrelic.newrelic.NewRelicLogger"
|
||||
) as MockNewRelicLogger:
|
||||
with patch("litellm.integrations.newrelic.newrelic.NewRelicLogger") as MockNewRelicLogger:
|
||||
mock_instance = MagicMock()
|
||||
mock_instance.async_health_check = AsyncMock(
|
||||
return_value={"status": "healthy", "error_message": ""}
|
||||
)
|
||||
mock_instance.async_health_check = AsyncMock(return_value={"status": "healthy", "error_message": ""})
|
||||
MockNewRelicLogger.return_value = mock_instance
|
||||
|
||||
result = await health_services_endpoint(
|
||||
|
|
@ -1232,20 +1192,14 @@ def test_health_liveliness_endpoint(proxy_client):
|
|||
duration_ms = (end_time - start_time) * 1000
|
||||
|
||||
# Assert response status
|
||||
assert (
|
||||
response.status_code == 200
|
||||
), f"Expected 200 OK, got {response.status_code}: {response.text}"
|
||||
assert response.status_code == 200, f"Expected 200 OK, got {response.status_code}: {response.text}"
|
||||
|
||||
# Assert response content (FastAPI JSON-encodes the string)
|
||||
assert (
|
||||
response.json() == "I'm alive!"
|
||||
), f"Expected 'I'm alive!' message, got: {response.json()}"
|
||||
assert response.json() == "I'm alive!", f"Expected 'I'm alive!' message, got: {response.json()}"
|
||||
|
||||
# Verify response is fast (should be < 100ms for a simple endpoint)
|
||||
# This is critical for orchestration systems that poll frequently
|
||||
assert (
|
||||
duration_ms < 100
|
||||
), f"Health check took {duration_ms:.2f}ms, expected < 100ms for a simple endpoint"
|
||||
assert duration_ms < 100, f"Health check took {duration_ms:.2f}ms, expected < 100ms for a simple endpoint"
|
||||
|
||||
# Log the duration for visibility (useful for CI/CD monitoring)
|
||||
print(f"\n/health/liveliness response time: {duration_ms:.2f}ms")
|
||||
|
|
@ -1265,19 +1219,13 @@ def test_health_liveness_endpoint(proxy_client):
|
|||
duration_ms = (end_time - start_time) * 1000
|
||||
|
||||
# Assert response status
|
||||
assert (
|
||||
response.status_code == 200
|
||||
), f"Expected 200 OK, got {response.status_code}: {response.text}"
|
||||
assert response.status_code == 200, f"Expected 200 OK, got {response.status_code}: {response.text}"
|
||||
|
||||
# Assert response content (FastAPI JSON-encodes the string)
|
||||
assert (
|
||||
response.json() == "I'm alive!"
|
||||
), f"Expected 'I'm alive!' message, got: {response.json()}"
|
||||
assert response.json() == "I'm alive!", f"Expected 'I'm alive!' message, got: {response.json()}"
|
||||
|
||||
# Verify response is fast (should be < 100ms for a simple endpoint)
|
||||
assert (
|
||||
duration_ms < 100
|
||||
), f"Health check took {duration_ms:.2f}ms, expected < 100ms for a simple endpoint"
|
||||
assert duration_ms < 100, f"Health check took {duration_ms:.2f}ms, expected < 100ms for a simple endpoint"
|
||||
|
||||
# Log the duration for visibility (useful for CI/CD monitoring)
|
||||
print(f"\n/health/liveness response time: {duration_ms:.2f}ms")
|
||||
|
|
@ -1298,15 +1246,11 @@ def test_health_readiness(proxy_client):
|
|||
duration_ms = (end_time - start_time) * 1000
|
||||
|
||||
# Assert response status
|
||||
assert (
|
||||
response.status_code == 200
|
||||
), f"Expected 200 OK, got {response.status_code}: {response.text}"
|
||||
assert response.status_code == 200, f"Expected 200 OK, got {response.status_code}: {response.text}"
|
||||
|
||||
# Verify response is fast (readiness may include DB check if available, so < 500ms is reasonable)
|
||||
# This is critical for orchestration systems (Kubernetes) that poll frequently
|
||||
assert (
|
||||
duration_ms < 500
|
||||
), f"Health check took {duration_ms:.2f}ms, expected < 500ms for readiness endpoint"
|
||||
assert duration_ms < 500, f"Health check took {duration_ms:.2f}ms, expected < 500ms for readiness endpoint"
|
||||
|
||||
# Assert response contains only low-detail public probe fields. `db` is
|
||||
# included so unauthenticated probes can distinguish "DB unreachable"
|
||||
|
|
@ -1325,9 +1269,7 @@ def test_health_readiness_details_returns_diagnostic_fields(monkeypatch):
|
|||
"""
|
||||
app = FastAPI()
|
||||
app.include_router(_health_endpoints_module.router)
|
||||
app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth(
|
||||
user_role=LitellmUserRoles.PROXY_ADMIN
|
||||
)
|
||||
app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN)
|
||||
client = TestClient(app)
|
||||
|
||||
monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", None)
|
||||
|
|
@ -1477,9 +1419,7 @@ def test_get_callback_identifier_custom_logger_registry_and_fallback():
|
|||
|
||||
unregistered = UnregisteredCallback()
|
||||
# Mock registry to return empty list (not registered)
|
||||
with patch.object(
|
||||
CustomLoggerRegistry, "get_all_callback_strs_from_class_type", return_value=[]
|
||||
):
|
||||
with patch.object(CustomLoggerRegistry, "get_all_callback_strs_from_class_type", return_value=[]):
|
||||
result = get_callback_identifier(unregistered)
|
||||
# Should fall back to callback_name() which returns __class__.__name__
|
||||
assert result == "UnregisteredCallback"
|
||||
|
|
@ -1568,13 +1508,9 @@ async def test_health_endpoint_filters_model_list_by_user_access():
|
|||
|
||||
await health_endpoint(response=Response(), user_api_key_dict=user_api_key_dict)
|
||||
|
||||
assert (
|
||||
"model_list" in captured
|
||||
), "health_endpoint did not call _perform_health_check_and_save"
|
||||
assert "model_list" in captured, "health_endpoint did not call _perform_health_check_and_save"
|
||||
returned_names = {m["model_name"] for m in captured["model_list"]}
|
||||
assert returned_names == {
|
||||
"model-a"
|
||||
}, f"health_endpoint did not scope model_list to caller access: {returned_names}"
|
||||
assert returned_names == {"model-a"}, f"health_endpoint did not scope model_list to caller access: {returned_names}"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
|
@ -1704,9 +1640,7 @@ async def test_health_endpoint_resolves_all_team_models_to_team_allowlist():
|
|||
await health_endpoint(response=Response(), user_api_key_dict=user_api_key_dict)
|
||||
|
||||
returned_names = {m["model_name"] for m in captured["model_list"]}
|
||||
assert returned_names == {
|
||||
"model-b"
|
||||
}, f"all-team-models key should health-check the team's models: {returned_names}"
|
||||
assert returned_names == {"model-b"}, f"all-team-models key should health-check the team's models: {returned_names}"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
|
@ -1788,15 +1722,13 @@ async def test_health_endpoint_filters_background_cache_by_user_access():
|
|||
# vacuously when the cache filter drops everything because cached
|
||||
# entries lack the model_id key — both entries carry model_id above.)
|
||||
assert len(cached_results["healthy_endpoints"]) == 2
|
||||
assert all(
|
||||
ep.get("model_id") for ep in cached_results["healthy_endpoints"]
|
||||
), "test fixture invariant: every cached entry must carry a model_id"
|
||||
assert all(ep.get("model_id") for ep in cached_results["healthy_endpoints"]), (
|
||||
"test fixture invariant: every cached entry must carry a model_id"
|
||||
)
|
||||
|
||||
# The non-admin caller must not see api_base on the returned cache entries.
|
||||
returned = result.get("healthy_endpoints", [])
|
||||
assert (
|
||||
len(returned) == 1
|
||||
), f"expected exactly one cached entry after scoping, got {len(returned)}"
|
||||
assert len(returned) == 1, f"expected exactly one cached entry after scoping, got {len(returned)}"
|
||||
assert returned[0]["model_id"] == "id-a"
|
||||
assert "api_base" not in returned[0]
|
||||
assert result["healthy_count"] == 1
|
||||
|
|
@ -1887,13 +1819,12 @@ async def test_health_endpoint_admin_sees_routing_fields_non_admin_does_not():
|
|||
non_admin_eps = non_admin_result.get("healthy_endpoints", [])
|
||||
|
||||
assert len(admin_eps) == 1
|
||||
assert (
|
||||
admin_eps[0]["api_base"]
|
||||
== "https://us-central1-aiplatform.googleapis.com/v1/projects/p"
|
||||
), "admin must see the full api_base so they can identify the region"
|
||||
assert (
|
||||
admin_eps[0]["api_version"] == "2024-10-21"
|
||||
), "admin must see api_version so they can distinguish provider deployments"
|
||||
assert admin_eps[0]["api_base"] == "https://us-central1-aiplatform.googleapis.com/v1/projects/p", (
|
||||
"admin must see the full api_base so they can identify the region"
|
||||
)
|
||||
assert admin_eps[0]["api_version"] == "2024-10-21", (
|
||||
"admin must see api_version so they can distinguish provider deployments"
|
||||
)
|
||||
|
||||
assert len(non_admin_eps) == 1
|
||||
assert "api_base" not in non_admin_eps[0]
|
||||
|
|
@ -1910,10 +1841,7 @@ async def test_health_endpoint_admin_sees_routing_fields_non_admin_does_not():
|
|||
# Stripping must produce a copy — the shared cache must still carry the
|
||||
# routing fields so the next admin caller can read them.
|
||||
cached_first = cached_results["healthy_endpoints"][0]
|
||||
assert (
|
||||
cached_first["api_base"]
|
||||
== "https://us-central1-aiplatform.googleapis.com/v1/projects/p"
|
||||
)
|
||||
assert cached_first["api_base"] == "https://us-central1-aiplatform.googleapis.com/v1/projects/p"
|
||||
assert cached_first["api_version"] == "2024-10-21"
|
||||
|
||||
|
||||
|
|
@ -2058,9 +1986,7 @@ async def test_health_endpoint_blocks_cross_scope_model_id_under_background_cach
|
|||
|
||||
leaked_ids = {ep.get("model_id") for ep in result.get("healthy_endpoints", [])}
|
||||
leaked_ids |= {ep.get("model_id") for ep in result.get("unhealthy_endpoints", [])}
|
||||
assert (
|
||||
"id-b" not in leaked_ids
|
||||
), "background cache leaked an out-of-scope deployment to a scoped caller"
|
||||
assert "id-b" not in leaked_ids, "background cache leaked an out-of-scope deployment to a scoped caller"
|
||||
assert result["healthy_count"] == 0
|
||||
assert response.status_code == 503
|
||||
|
||||
|
|
@ -2287,9 +2213,7 @@ async def test_health_endpoint_no_model_param_returns_200_even_when_zero_healthy
|
|||
async def fake_perform(**kwargs):
|
||||
return {
|
||||
"healthy_endpoints": [],
|
||||
"unhealthy_endpoints": [
|
||||
{"model": "openai/gpt-4o", "model_id": "id-a", "error": "boom"}
|
||||
],
|
||||
"unhealthy_endpoints": [{"model": "openai/gpt-4o", "model_id": "id-a", "error": "boom"}],
|
||||
"healthy_count": 0,
|
||||
"unhealthy_count": 1,
|
||||
}
|
||||
|
|
@ -2747,6 +2671,70 @@ class TestNoRedisWarning:
|
|||
assert details["show_no_redis_warning"] is False
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_health_services_endpoint_ms_teams_posts_adaptive_card():
|
||||
mock_response = MagicMock()
|
||||
mock_response.status_code = 200
|
||||
mock_post = AsyncMock(return_value=mock_response)
|
||||
mock_proxy_logging = MagicMock()
|
||||
mock_proxy_logging.slack_alerting_instance.async_http_handler.post = mock_post
|
||||
|
||||
with (
|
||||
patch( # test-quality-ok: endpoint reads proxy_server module globals, same pattern as sibling tests
|
||||
"litellm.proxy.proxy_server.general_settings",
|
||||
{"alerting": ["ms_teams"]},
|
||||
),
|
||||
patch( # test-quality-ok: endpoint reads proxy_server module globals, same pattern as sibling tests
|
||||
"litellm.proxy.proxy_server.proxy_logging_obj",
|
||||
mock_proxy_logging,
|
||||
),
|
||||
patch.dict("os.environ", {"MS_TEAMS_WEBHOOK_URL": "https://teams.example/webhook"}),
|
||||
):
|
||||
result = await health_services_endpoint(service="ms_teams")
|
||||
|
||||
assert result["status"] == "success"
|
||||
call_kwargs = mock_post.call_args.kwargs
|
||||
assert call_kwargs["url"] == "https://teams.example/webhook"
|
||||
sent_body = json.loads(call_kwargs["data"])
|
||||
assert sent_body["type"] == "message"
|
||||
assert sent_body["attachments"][0]["contentType"] == "application/vnd.microsoft.card.adaptive"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_health_services_endpoint_ms_teams_surfaces_delivery_failure():
|
||||
mock_response = MagicMock()
|
||||
mock_response.status_code = 400
|
||||
mock_response.text = "Invalid webhook"
|
||||
mock_proxy_logging = MagicMock()
|
||||
mock_proxy_logging.slack_alerting_instance.async_http_handler.post = AsyncMock(return_value=mock_response)
|
||||
|
||||
with (
|
||||
patch( # test-quality-ok: endpoint reads proxy_server module globals, same pattern as sibling tests
|
||||
"litellm.proxy.proxy_server.general_settings",
|
||||
{"alerting": ["ms_teams"]},
|
||||
),
|
||||
patch( # test-quality-ok: endpoint reads proxy_server module globals, same pattern as sibling tests
|
||||
"litellm.proxy.proxy_server.proxy_logging_obj",
|
||||
mock_proxy_logging,
|
||||
),
|
||||
patch.dict("os.environ", {"MS_TEAMS_WEBHOOK_URL": "https://teams.example/webhook"}),
|
||||
):
|
||||
with pytest.raises(ProxyException) as exc_info:
|
||||
await health_services_endpoint(service="ms_teams")
|
||||
|
||||
assert "status 400" in str(exc_info.value.message)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_health_services_endpoint_ms_teams_requires_alerting_config():
|
||||
with patch( # test-quality-ok: endpoint reads proxy_server module globals, same pattern as sibling tests
|
||||
"litellm.proxy.proxy_server.general_settings",
|
||||
{"alerting": ["slack"]},
|
||||
):
|
||||
with pytest.raises(ProxyException):
|
||||
await health_services_endpoint(service="ms_teams")
|
||||
|
||||
|
||||
def test_test_model_connection_accepts_image_edit_mode(monkeypatch):
|
||||
"""
|
||||
Regression: /health/test_connection rejected mode=image_edit with a 422
|
||||
|
|
|
|||
161
ui/litellm-dashboard/src/components/MSTeamsSettings.tsx
Normal file
161
ui/litellm-dashboard/src/components/MSTeamsSettings.tsx
Normal file
|
|
@ -0,0 +1,161 @@
|
|||
import React, { useState } from "react";
|
||||
import { Button } from "@/components/ui/button";
|
||||
import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card";
|
||||
import { InputGroup, InputGroupAddon, InputGroupButton, InputGroupInput } from "@/components/ui/input-group";
|
||||
import { Eye, EyeOff } from "lucide-react";
|
||||
import { toast } from "@/lib/toast";
|
||||
import { getCallbacksCall, serviceHealthCheck, setCallbacksCall } from "./networking";
|
||||
|
||||
interface AlertingDestination {
|
||||
name: string;
|
||||
variables?: Record<string, string | null>;
|
||||
}
|
||||
|
||||
interface MSTeamsSettingsProps {
|
||||
accessToken: string | null;
|
||||
userID: string | null;
|
||||
userRole: string | null;
|
||||
alerts: AlertingDestination[];
|
||||
}
|
||||
|
||||
const FIELD_HELP: Record<string, React.ReactNode> = {
|
||||
MS_TEAMS_WEBHOOK_URL: (
|
||||
<>
|
||||
Incoming webhook URL for your Teams channel (Workflows or incoming webhook connector)
|
||||
<span className="text-destructive"> Required * </span>
|
||||
</>
|
||||
),
|
||||
};
|
||||
|
||||
const SENSITIVE_FIELD_PATTERN = /(PASSWORD|SECRET|KEY|TOKEN|URL)/i;
|
||||
|
||||
const MSTeamsSettings: React.FC<MSTeamsSettingsProps> = ({ accessToken, userID, userRole, alerts }) => {
|
||||
const [visibleFields, setVisibleFields] = useState<Record<string, boolean>>({});
|
||||
|
||||
const toggleFieldVisibility = (key: string) => {
|
||||
setVisibleFields((prev) => ({
|
||||
...prev,
|
||||
[key]: !prev[key],
|
||||
}));
|
||||
};
|
||||
|
||||
const handleSaveMSTeamsSettings = async () => {
|
||||
if (!accessToken || !userID || !userRole) {
|
||||
return;
|
||||
}
|
||||
|
||||
// Only send fields the admin actually edited. Values rendered from the
|
||||
// server are masked or sourced from the process environment, so
|
||||
// re-submitting an untouched field would persist a mask or copy
|
||||
// env-managed config into the database.
|
||||
const updatedVariables: Record<string, string> = Object.fromEntries(
|
||||
alerts
|
||||
.filter((alert) => alert.name === "ms_teams")
|
||||
.flatMap((alert) =>
|
||||
Object.entries(alert.variables ?? {}).flatMap(([key, value]) => {
|
||||
const inputElement = document.querySelector(`input[name="${key}"]`) as HTMLInputElement;
|
||||
if (!inputElement || !inputElement.value) {
|
||||
return [];
|
||||
}
|
||||
if (inputElement.value === (value == null ? "" : String(value))) {
|
||||
return [];
|
||||
}
|
||||
return [[key, inputElement.value] as const];
|
||||
}),
|
||||
),
|
||||
);
|
||||
|
||||
try {
|
||||
// Re-read the persisted destinations at save time so that a Teams save
|
||||
// never restores destinations another form disabled after page load.
|
||||
const currentConfig = await getCallbacksCall(accessToken, userID, userRole);
|
||||
const currentDestinations: string[] = currentConfig.active_alerting_destinations ?? [];
|
||||
const payload = {
|
||||
general_settings: {
|
||||
alerting: Array.from(new Set([...currentDestinations, "ms_teams"])),
|
||||
},
|
||||
environment_variables: updatedVariables,
|
||||
};
|
||||
await setCallbacksCall(accessToken, payload);
|
||||
toast.success("MS Teams settings updated successfully");
|
||||
} catch (error) {
|
||||
toast.fromError(error);
|
||||
}
|
||||
};
|
||||
|
||||
return (
|
||||
<Card>
|
||||
<CardHeader>
|
||||
<CardTitle className="text-base">Microsoft Teams Alerting Settings</CardTitle>
|
||||
<p className="text-sm">
|
||||
Send LiteLLM alerts to a Microsoft Teams channel via an incoming webhook. Create one from{" "}
|
||||
<a
|
||||
href="https://learn.microsoft.com/en-us/microsoftteams/platform/webhooks-and-connectors/how-to/add-incoming-webhook"
|
||||
target="_blank"
|
||||
rel="noreferrer"
|
||||
className="text-primary underline underline-offset-4"
|
||||
>
|
||||
Microsoft Docs: incoming webhooks
|
||||
</a>
|
||||
</p>
|
||||
</CardHeader>
|
||||
|
||||
<CardContent>
|
||||
{alerts
|
||||
.filter((alert) => alert.name === "ms_teams")
|
||||
.map((alert, index) => (
|
||||
<div key={index} className="grid grid-cols-1 gap-4 sm:grid-cols-2">
|
||||
{Object.entries(alert.variables ?? {}).map(([key, value]) => {
|
||||
const isSensitive = SENSITIVE_FIELD_PATTERN.test(key);
|
||||
const isVisible = visibleFields[key] || false;
|
||||
return (
|
||||
<div key={key} className="space-y-1">
|
||||
<p className="text-sm">{key}</p>
|
||||
<InputGroup className="max-w-100">
|
||||
<InputGroupInput
|
||||
name={key}
|
||||
defaultValue={value as string}
|
||||
type={isSensitive && !isVisible ? "password" : "text"}
|
||||
/>
|
||||
{isSensitive && (
|
||||
<InputGroupAddon align="inline-end">
|
||||
<InputGroupButton
|
||||
size="icon-xs"
|
||||
onClick={() => toggleFieldVisibility(key)}
|
||||
aria-label={isVisible ? "Hide credential" : "Show credential"}
|
||||
>
|
||||
{isVisible ? <EyeOff /> : <Eye />}
|
||||
</InputGroupButton>
|
||||
</InputGroupAddon>
|
||||
)}
|
||||
</InputGroup>
|
||||
<div className="text-xs text-muted-foreground italic">{FIELD_HELP[key]}</div>
|
||||
</div>
|
||||
);
|
||||
})}
|
||||
</div>
|
||||
))}
|
||||
|
||||
<div className="mt-6 flex gap-2">
|
||||
<Button onClick={() => handleSaveMSTeamsSettings()}>Save Changes</Button>
|
||||
<Button
|
||||
variant="secondary"
|
||||
onClick={async () => {
|
||||
if (!accessToken) return;
|
||||
try {
|
||||
await serviceHealthCheck(accessToken, "ms_teams");
|
||||
toast.success("MS Teams test alert triggered. Check your Teams channel.");
|
||||
} catch (error) {
|
||||
toast.fromError(error);
|
||||
}
|
||||
}}
|
||||
>
|
||||
Test MS Teams Alerts
|
||||
</Button>
|
||||
</div>
|
||||
</CardContent>
|
||||
</Card>
|
||||
);
|
||||
};
|
||||
|
||||
export default MSTeamsSettings;
|
||||
|
|
@ -18,6 +18,7 @@ import { Switch } from "@/components/ui/switch";
|
|||
import { Table, TableBody, TableCell, TableHead, TableHeader, TableRow } from "@/components/ui/table";
|
||||
import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs";
|
||||
import EmailSettings from "./email_settings";
|
||||
import MSTeamsSettings from "./MSTeamsSettings";
|
||||
import { Logo } from "@/components/molecules/logo/Logo";
|
||||
import { toast } from "@/lib/toast";
|
||||
|
||||
|
|
@ -490,6 +491,7 @@ const Settings: React.FC<SettingsPageProps> = ({ accessToken, userRole, userID,
|
|||
<TabsTrigger value="alerting-types">Alerting Types</TabsTrigger>
|
||||
<TabsTrigger value="alerting-settings">Alerting Settings</TabsTrigger>
|
||||
<TabsTrigger value="email-alerts">Email Alerts</TabsTrigger>
|
||||
<TabsTrigger value="ms-teams-alerts">MS Teams Alerts</TabsTrigger>
|
||||
</TabsList>
|
||||
<TabsContent value="logging-callbacks" keepMounted>
|
||||
<LoggingCallbacksTable
|
||||
|
|
@ -607,6 +609,9 @@ const Settings: React.FC<SettingsPageProps> = ({ accessToken, userRole, userID,
|
|||
<TabsContent value="email-alerts" keepMounted>
|
||||
<EmailSettings accessToken={accessToken} premiumUser={premiumUser} alerts={alerts} />
|
||||
</TabsContent>
|
||||
<TabsContent value="ms-teams-alerts" keepMounted>
|
||||
<MSTeamsSettings accessToken={accessToken} userID={userID} userRole={userRole} alerts={alerts} />
|
||||
</TabsContent>
|
||||
</Tabs>
|
||||
</div>
|
||||
|
||||
|
|
|
|||
2
ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
2
ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
|
|
@ -47157,7 +47157,7 @@ export interface operations {
|
|||
parameters: {
|
||||
query: {
|
||||
/** @description Specify the service being hit. */
|
||||
service: ("slack_budget_alerts" | "langfuse" | "langfuse_otel" | "slack" | "openmeter" | "webhook" | "email" | "braintrust" | "datadog" | "datadog_llm_observability" | "generic_api" | "arize" | "galileo" | "newrelic" | "sqs") | string;
|
||||
service: ("slack_budget_alerts" | "langfuse" | "langfuse_otel" | "slack" | "ms_teams" | "openmeter" | "webhook" | "email" | "braintrust" | "datadog" | "datadog_llm_observability" | "generic_api" | "arize" | "galileo" | "newrelic" | "sqs") | string;
|
||||
};
|
||||
header?: never;
|
||||
path?: never;
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue