mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-09 03:18:44 +00:00
Merge pull request #5302 from BerriAI/litellm_dont_cooldown_provider_deployments
[Fix Router] - Don't cooldown Default Provider deployment
This commit is contained in:
Ishaan Jaff
GitHub
commit
7080e3919d
4 changed files with 333 additions and 139 deletions
|
|
@ -62,6 +62,8 @@ from litellm.router_utils.cooldown_callbacks import router_cooldown_handler
|
|||
from litellm.router_utils.fallback_event_handlers import (
|
||||
log_failure_fallback_event,
|
||||
log_success_fallback_event,
|
||||
run_async_fallback,
|
||||
run_sync_fallback,
|
||||
)
|
||||
from litellm.router_utils.handle_error import send_llm_exception_alert
|
||||
from litellm.scheduler import FlowItem, Scheduler
|
||||
|
|
@ -316,6 +318,7 @@ class Router:
|
|||
self.default_deployment = None # use this to track the users default deployment, when they want to use model = *
|
||||
self.default_max_parallel_requests = default_max_parallel_requests
|
||||
self.provider_default_deployments: Dict[str, List] = {}
|
||||
self.provider_default_deployment_ids: List[str] = []
|
||||
|
||||
if model_list is not None:
|
||||
model_list = copy.deepcopy(model_list)
|
||||
|
|
@ -2383,34 +2386,16 @@ class Router:
|
|||
if fallback_model_group is None:
|
||||
raise original_exception
|
||||
|
||||
for mg in fallback_model_group:
|
||||
"""
|
||||
Iterate through the model groups and try calling that deployment
|
||||
"""
|
||||
try:
|
||||
kwargs["model"] = mg
|
||||
kwargs.setdefault("metadata", {}).update(
|
||||
{"model_group": mg}
|
||||
) # update model_group used, if fallbacks are done
|
||||
response = await self.async_function_with_retries(
|
||||
*args, **kwargs
|
||||
)
|
||||
verbose_router_logger.info(
|
||||
"Successful fallback b/w models."
|
||||
)
|
||||
# callback for successfull_fallback_event():
|
||||
await log_success_fallback_event(
|
||||
original_model_group=original_model_group,
|
||||
kwargs=kwargs,
|
||||
)
|
||||
response = await run_async_fallback(
|
||||
*args,
|
||||
litellm_router=self,
|
||||
fallback_model_group=fallback_model_group,
|
||||
original_model_group=original_model_group,
|
||||
original_exception=original_exception,
|
||||
**kwargs,
|
||||
)
|
||||
return response
|
||||
|
||||
return response
|
||||
except Exception as e:
|
||||
await log_failure_fallback_event(
|
||||
original_model_group=original_model_group,
|
||||
kwargs=kwargs,
|
||||
)
|
||||
pass
|
||||
else:
|
||||
error_message = "model={}. context_window_fallbacks={}. fallbacks={}.\n\nSet 'context_window_fallback' - https://docs.litellm.ai/docs/routing#fallbacks".format(
|
||||
model_group, context_window_fallbacks, fallbacks
|
||||
|
|
@ -2436,33 +2421,15 @@ class Router:
|
|||
if fallback_model_group is None:
|
||||
raise original_exception
|
||||
|
||||
for mg in fallback_model_group:
|
||||
"""
|
||||
Iterate through the model groups and try calling that deployment
|
||||
"""
|
||||
try:
|
||||
kwargs["model"] = mg
|
||||
kwargs.setdefault("metadata", {}).update(
|
||||
{"model_group": mg}
|
||||
) # update model_group used, if fallbacks are done
|
||||
response = await self.async_function_with_retries(
|
||||
*args, **kwargs
|
||||
)
|
||||
verbose_router_logger.info(
|
||||
"Successful fallback b/w models."
|
||||
)
|
||||
# callback for successfull_fallback_event():
|
||||
await log_success_fallback_event(
|
||||
original_model_group=original_model_group,
|
||||
kwargs=kwargs,
|
||||
)
|
||||
return response
|
||||
except Exception as e:
|
||||
await log_failure_fallback_event(
|
||||
original_model_group=original_model_group,
|
||||
kwargs=kwargs,
|
||||
)
|
||||
pass
|
||||
response = await run_async_fallback(
|
||||
*args,
|
||||
litellm_router=self,
|
||||
fallback_model_group=fallback_model_group,
|
||||
original_model_group=original_model_group,
|
||||
original_exception=original_exception,
|
||||
**kwargs,
|
||||
)
|
||||
return response
|
||||
else:
|
||||
error_message = "model={}. content_policy_fallback={}. fallbacks={}.\n\nSet 'content_policy_fallback' - https://docs.litellm.ai/docs/routing#fallbacks".format(
|
||||
model_group, content_policy_fallbacks, fallbacks
|
||||
|
|
@ -2502,39 +2469,16 @@ class Router:
|
|||
if hasattr(original_exception, "message"):
|
||||
original_exception.message += f"No fallback model group found for original model_group={model_group}. Fallbacks={fallbacks}"
|
||||
raise original_exception
|
||||
for mg in fallback_model_group:
|
||||
"""
|
||||
Iterate through the model groups and try calling that deployment
|
||||
"""
|
||||
try:
|
||||
## LOGGING
|
||||
kwargs = self.log_retry(kwargs=kwargs, e=original_exception)
|
||||
verbose_router_logger.info(
|
||||
f"Falling back to model_group = {mg}"
|
||||
)
|
||||
kwargs["model"] = mg
|
||||
kwargs.setdefault("metadata", {}).update(
|
||||
{"model_group": mg}
|
||||
) # update model_group used, if fallbacks are done
|
||||
response = await self.async_function_with_fallbacks(
|
||||
*args, **kwargs
|
||||
)
|
||||
verbose_router_logger.info(
|
||||
"Successful fallback b/w models."
|
||||
)
|
||||
# callback for successfull_fallback_event():
|
||||
await log_success_fallback_event(
|
||||
original_model_group=original_model_group,
|
||||
kwargs=kwargs,
|
||||
)
|
||||
|
||||
return response
|
||||
except Exception as e:
|
||||
await log_failure_fallback_event(
|
||||
original_model_group=original_model_group,
|
||||
kwargs=kwargs,
|
||||
)
|
||||
raise e
|
||||
response = await run_async_fallback(
|
||||
*args,
|
||||
litellm_router=self,
|
||||
fallback_model_group=fallback_model_group,
|
||||
original_model_group=original_model_group,
|
||||
original_exception=original_exception,
|
||||
**kwargs,
|
||||
)
|
||||
return response
|
||||
except Exception as new_exception:
|
||||
verbose_router_logger.error(
|
||||
"litellm.router.py::async_function_with_fallbacks() - Error occurred while trying to do fallbacks - {}\n{}\n\nDebug Information:\nCooldown Deployments={}".format(
|
||||
|
|
@ -2708,6 +2652,8 @@ class Router:
|
|||
):
|
||||
raise error
|
||||
|
||||
if isinstance(error, litellm.NotFoundError):
|
||||
raise error
|
||||
# Error we should only retry if there are other deployments
|
||||
if isinstance(error, openai.RateLimitError):
|
||||
if (
|
||||
|
|
@ -2725,6 +2671,11 @@ class Router:
|
|||
if _num_healthy_deployments <= 0: # if no healthy deployments
|
||||
raise error # then raise error
|
||||
|
||||
# Do not retry if there are no healthy deployments
|
||||
# just raise the error
|
||||
if _num_healthy_deployments <= 0: # if no healthy deployments
|
||||
raise error
|
||||
|
||||
return True
|
||||
|
||||
def function_with_fallbacks(self, *args, **kwargs):
|
||||
|
|
@ -2751,6 +2702,7 @@ class Router:
|
|||
return response
|
||||
except Exception as e:
|
||||
original_exception = e
|
||||
original_model_group = kwargs.get("model")
|
||||
verbose_router_logger.debug(f"An exception occurs {original_exception}")
|
||||
try:
|
||||
verbose_router_logger.debug(
|
||||
|
|
@ -2772,21 +2724,14 @@ class Router:
|
|||
if fallback_model_group is None:
|
||||
raise original_exception
|
||||
|
||||
for mg in fallback_model_group:
|
||||
"""
|
||||
Iterate through the model groups and try calling that deployment
|
||||
"""
|
||||
try:
|
||||
## LOGGING
|
||||
kwargs = self.log_retry(kwargs=kwargs, e=original_exception)
|
||||
kwargs["model"] = mg
|
||||
kwargs.setdefault("metadata", {}).update(
|
||||
{"model_group": mg}
|
||||
) # update model_group used, if fallbacks are done
|
||||
response = self.function_with_fallbacks(*args, **kwargs)
|
||||
return response
|
||||
except Exception as e:
|
||||
pass
|
||||
return run_sync_fallback(
|
||||
*args,
|
||||
litellm_router=self,
|
||||
fallback_model_group=fallback_model_group,
|
||||
original_model_group=original_model_group,
|
||||
original_exception=original_exception,
|
||||
**kwargs,
|
||||
)
|
||||
elif (
|
||||
isinstance(e, litellm.ContentPolicyViolationError)
|
||||
and content_policy_fallbacks is not None
|
||||
|
|
@ -2803,21 +2748,14 @@ class Router:
|
|||
if fallback_model_group is None:
|
||||
raise original_exception
|
||||
|
||||
for mg in fallback_model_group:
|
||||
"""
|
||||
Iterate through the model groups and try calling that deployment
|
||||
"""
|
||||
try:
|
||||
## LOGGING
|
||||
kwargs = self.log_retry(kwargs=kwargs, e=original_exception)
|
||||
kwargs["model"] = mg
|
||||
kwargs.setdefault("metadata", {}).update(
|
||||
{"model_group": mg}
|
||||
) # update model_group used, if fallbacks are done
|
||||
response = self.function_with_fallbacks(*args, **kwargs)
|
||||
return response
|
||||
except Exception as e:
|
||||
pass
|
||||
return run_sync_fallback(
|
||||
*args,
|
||||
litellm_router=self,
|
||||
fallback_model_group=fallback_model_group,
|
||||
original_model_group=original_model_group,
|
||||
original_exception=original_exception,
|
||||
**kwargs,
|
||||
)
|
||||
elif fallbacks is not None:
|
||||
verbose_router_logger.debug(f"inside model fallbacks: {fallbacks}")
|
||||
fallback_model_group = None
|
||||
|
|
@ -2841,21 +2779,14 @@ class Router:
|
|||
if fallback_model_group is None:
|
||||
raise original_exception
|
||||
|
||||
for mg in fallback_model_group:
|
||||
"""
|
||||
Iterate through the model groups and try calling that deployment
|
||||
"""
|
||||
try:
|
||||
## LOGGING
|
||||
kwargs = self.log_retry(kwargs=kwargs, e=original_exception)
|
||||
kwargs["model"] = mg
|
||||
kwargs.setdefault("metadata", {}).update(
|
||||
{"model_group": mg}
|
||||
) # update model_group used, if fallbacks are done
|
||||
response = self.function_with_fallbacks(*args, **kwargs)
|
||||
return response
|
||||
except Exception as e:
|
||||
raise e
|
||||
return run_sync_fallback(
|
||||
*args,
|
||||
litellm_router=self,
|
||||
fallback_model_group=fallback_model_group,
|
||||
original_model_group=original_model_group,
|
||||
original_exception=original_exception,
|
||||
**kwargs,
|
||||
)
|
||||
except Exception as e:
|
||||
raise e
|
||||
raise original_exception
|
||||
|
|
@ -3248,6 +3179,9 @@ class Router:
|
|||
if self._is_cooldown_required(exception_status=exception_status) == False:
|
||||
return
|
||||
|
||||
if deployment in self.provider_default_deployment_ids:
|
||||
return
|
||||
|
||||
_allowed_fails = self.get_allowed_fails_from_policy(
|
||||
exception=original_exception,
|
||||
)
|
||||
|
|
@ -3655,6 +3589,9 @@ class Router:
|
|||
deployment.to_json(exclude_none=True)
|
||||
]
|
||||
|
||||
if deployment.model_info.id:
|
||||
self.provider_default_deployment_ids.append(deployment.model_info.id)
|
||||
|
||||
# Azure GPT-Vision Enhancements, users can pass os.environ/
|
||||
data_sources = deployment.litellm_params.get("dataSources", []) or []
|
||||
|
||||
|
|
|
|||
|
|
@ -1,9 +1,87 @@
|
|||
from typing import TYPE_CHECKING, Any
|
||||
from typing import TYPE_CHECKING, Any, Dict, List, Tuple
|
||||
|
||||
import litellm
|
||||
from litellm._logging import verbose_router_logger
|
||||
from litellm.integrations.custom_logger import CustomLogger
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from litellm.router import Router as _Router
|
||||
|
||||
LitellmRouter = _Router
|
||||
else:
|
||||
LitellmRouter = Any
|
||||
|
||||
|
||||
async def run_async_fallback(
|
||||
litellm_router: LitellmRouter,
|
||||
*args: Tuple[Any],
|
||||
fallback_model_group: List[str],
|
||||
original_model_group: str,
|
||||
original_exception: Exception,
|
||||
**kwargs,
|
||||
) -> Any:
|
||||
"""
|
||||
Iterate through the model groups and try calling that deployment.
|
||||
"""
|
||||
error_from_fallbacks = original_exception
|
||||
for mg in fallback_model_group:
|
||||
if mg == original_model_group:
|
||||
continue
|
||||
try:
|
||||
# LOGGING
|
||||
kwargs = litellm_router.log_retry(kwargs=kwargs, e=original_exception)
|
||||
verbose_router_logger.info(f"Falling back to model_group = {mg}")
|
||||
kwargs["model"] = mg
|
||||
kwargs.setdefault("metadata", {}).update(
|
||||
{"model_group": mg}
|
||||
) # update model_group used, if fallbacks are done
|
||||
response = await litellm_router.async_function_with_fallbacks(
|
||||
*args, **kwargs
|
||||
)
|
||||
verbose_router_logger.info("Successful fallback b/w models.")
|
||||
# callback for successfull_fallback_event():
|
||||
await log_success_fallback_event(
|
||||
original_model_group=original_model_group, kwargs=kwargs
|
||||
)
|
||||
return response
|
||||
except Exception as e:
|
||||
error_from_fallbacks = e
|
||||
await log_failure_fallback_event(
|
||||
original_model_group=original_model_group, kwargs=kwargs
|
||||
)
|
||||
raise error_from_fallbacks
|
||||
|
||||
|
||||
def run_sync_fallback(
|
||||
litellm_router: LitellmRouter,
|
||||
*args: Tuple[Any],
|
||||
fallback_model_group: List[str],
|
||||
original_model_group: str,
|
||||
original_exception: Exception,
|
||||
**kwargs,
|
||||
) -> Any:
|
||||
"""
|
||||
Iterate through the model groups and try calling that deployment.
|
||||
"""
|
||||
error_from_fallbacks = original_exception
|
||||
for mg in fallback_model_group:
|
||||
if mg == original_model_group:
|
||||
continue
|
||||
try:
|
||||
# LOGGING
|
||||
kwargs = litellm_router.log_retry(kwargs=kwargs, e=original_exception)
|
||||
verbose_router_logger.info(f"Falling back to model_group = {mg}")
|
||||
kwargs["model"] = mg
|
||||
kwargs.setdefault("metadata", {}).update(
|
||||
{"model_group": mg}
|
||||
) # update model_group used, if fallbacks are done
|
||||
response = litellm_router.function_with_fallbacks(*args, **kwargs)
|
||||
verbose_router_logger.info("Successful fallback b/w models.")
|
||||
return response
|
||||
except Exception as e:
|
||||
error_from_fallbacks = e
|
||||
raise error_from_fallbacks
|
||||
|
||||
|
||||
async def log_success_fallback_event(original_model_group: str, kwargs: dict):
|
||||
for _callback in litellm.callbacks:
|
||||
|
|
|
|||
|
|
@ -1185,3 +1185,82 @@ async def test_router_content_policy_fallbacks(
|
|||
)
|
||||
|
||||
assert response.model == "my-fake-model"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("sync_mode", [False, True])
|
||||
@pytest.mark.asyncio
|
||||
async def test_using_default_fallback(sync_mode):
|
||||
litellm.set_verbose = True
|
||||
|
||||
import logging
|
||||
|
||||
from litellm._logging import verbose_logger, verbose_router_logger
|
||||
|
||||
verbose_logger.setLevel(logging.DEBUG)
|
||||
verbose_router_logger.setLevel(logging.DEBUG)
|
||||
litellm.default_fallbacks = ["very-bad-model"]
|
||||
router = Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "openai/*",
|
||||
"litellm_params": {
|
||||
"model": "openai/*",
|
||||
"api_key": os.getenv("OPENAI_API_KEY"),
|
||||
},
|
||||
},
|
||||
],
|
||||
)
|
||||
try:
|
||||
if sync_mode:
|
||||
response = router.completion(
|
||||
model="openai/foo",
|
||||
messages=[{"role": "user", "content": "Hey, how's it going?"}],
|
||||
)
|
||||
else:
|
||||
response = await router.acompletion(
|
||||
model="openai/foo",
|
||||
messages=[{"role": "user", "content": "Hey, how's it going?"}],
|
||||
)
|
||||
print("got response=", response)
|
||||
pytest.fail(f"Expected call to fail we passed model=openai/foo")
|
||||
except Exception as e:
|
||||
print("got exception = ", e)
|
||||
assert "No healthy deployment available, passed model=very-bad-model" in str(e)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("sync_mode", [False])
|
||||
@pytest.mark.asyncio
|
||||
async def test_using_default_working_fallback(sync_mode):
|
||||
litellm.set_verbose = True
|
||||
|
||||
import logging
|
||||
|
||||
from litellm._logging import verbose_logger, verbose_router_logger
|
||||
|
||||
verbose_logger.setLevel(logging.DEBUG)
|
||||
verbose_router_logger.setLevel(logging.DEBUG)
|
||||
litellm.default_fallbacks = ["openai/gpt-3.5-turbo"]
|
||||
router = Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "openai/*",
|
||||
"litellm_params": {
|
||||
"model": "openai/*",
|
||||
"api_key": os.getenv("OPENAI_API_KEY"),
|
||||
},
|
||||
},
|
||||
],
|
||||
)
|
||||
|
||||
if sync_mode:
|
||||
response = router.completion(
|
||||
model="openai/foo",
|
||||
messages=[{"role": "user", "content": "Hey, how's it going?"}],
|
||||
)
|
||||
else:
|
||||
response = await router.acompletion(
|
||||
model="openai/foo",
|
||||
messages=[{"role": "user", "content": "Hey, how's it going?"}],
|
||||
)
|
||||
print("got response=", response)
|
||||
assert response is not None
|
||||
|
|
|
|||
|
|
@ -1,18 +1,24 @@
|
|||
#### What this tests ####
|
||||
# This tests calling router with fallback models
|
||||
|
||||
import sys, os, time
|
||||
import traceback, asyncio
|
||||
import asyncio
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
import traceback
|
||||
|
||||
import pytest
|
||||
|
||||
sys.path.insert(
|
||||
0, os.path.abspath("../..")
|
||||
) # Adds the parent directory to the system path
|
||||
|
||||
import httpx
|
||||
import openai
|
||||
|
||||
import litellm
|
||||
from litellm import Router
|
||||
from litellm.integrations.custom_logger import CustomLogger
|
||||
import openai, httpx
|
||||
|
||||
|
||||
class MyCustomHandler(CustomLogger):
|
||||
|
|
@ -127,7 +133,7 @@ async def test_router_retries_errors(sync_mode, error_type):
|
|||
["AuthenticationErrorRetries", "ContentPolicyViolationErrorRetries"], #
|
||||
)
|
||||
async def test_router_retry_policy(error_type):
|
||||
from litellm.router import RetryPolicy, AllowedFailsPolicy
|
||||
from litellm.router import AllowedFailsPolicy, RetryPolicy
|
||||
|
||||
retry_policy = RetryPolicy(
|
||||
ContentPolicyViolationErrorRetries=3, AuthenticationErrorRetries=0
|
||||
|
|
@ -386,7 +392,7 @@ def test_retry_rate_limit_error_with_healthy_deployments():
|
|||
|
||||
def test_do_retry_rate_limit_error_with_no_fallbacks_and_no_healthy_deployments():
|
||||
"""
|
||||
Test 2. It SHOULD Retry, when healthy_deployments is [] and fallbacks is None
|
||||
Test 2. It SHOULD NOT Retry, when healthy_deployments is [] and fallbacks is None
|
||||
"""
|
||||
healthy_deployments = []
|
||||
|
||||
|
|
@ -409,9 +415,10 @@ def test_do_retry_rate_limit_error_with_no_fallbacks_and_no_healthy_deployments(
|
|||
response = router.should_retry_this_error(
|
||||
error=rate_limit_error, healthy_deployments=healthy_deployments
|
||||
)
|
||||
assert response == True
|
||||
pytest.fail("Should have raised an error")
|
||||
except Exception as e:
|
||||
pytest.fail("Should not have failed this error - {}".format(str(e)))
|
||||
print("got an exception", e)
|
||||
pass
|
||||
|
||||
|
||||
def test_raise_context_window_exceeded_error():
|
||||
|
|
@ -582,3 +589,96 @@ def test_timeout_for_rate_limit_error_with_no_healthy_deployments():
|
|||
)
|
||||
|
||||
assert _timeout > 0.0
|
||||
|
||||
|
||||
def test_no_retry_for_not_found_error_404():
|
||||
healthy_deployments = []
|
||||
|
||||
router = Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "gpt-3.5-turbo",
|
||||
"litellm_params": {
|
||||
"model": "azure/chatgpt-v-2",
|
||||
"api_key": os.getenv("AZURE_API_KEY"),
|
||||
"api_version": os.getenv("AZURE_API_VERSION"),
|
||||
"api_base": os.getenv("AZURE_API_BASE"),
|
||||
},
|
||||
}
|
||||
]
|
||||
)
|
||||
|
||||
# Act & Assert
|
||||
error = litellm.NotFoundError(
|
||||
message="404 model not found",
|
||||
model="gpt-12",
|
||||
llm_provider="azure",
|
||||
)
|
||||
try:
|
||||
response = router.should_retry_this_error(
|
||||
error=error, healthy_deployments=healthy_deployments
|
||||
)
|
||||
pytest.fail(
|
||||
"Should have raised an exception 404 NotFoundError should never be retried, it's typically model_not_found error"
|
||||
)
|
||||
except Exception as e:
|
||||
print("got exception", e)
|
||||
|
||||
|
||||
internal_server_error = litellm.InternalServerError(
|
||||
message="internal server error",
|
||||
model="gpt-12",
|
||||
llm_provider="azure",
|
||||
)
|
||||
|
||||
rate_limit_error = litellm.RateLimitError(
|
||||
message="rate limit error",
|
||||
model="gpt-12",
|
||||
llm_provider="azure",
|
||||
)
|
||||
|
||||
service_unavailable_error = litellm.ServiceUnavailableError(
|
||||
message="service unavailable error",
|
||||
model="gpt-12",
|
||||
llm_provider="azure",
|
||||
)
|
||||
|
||||
timeout_error = litellm.Timeout(
|
||||
message="timeout error",
|
||||
model="gpt-12",
|
||||
llm_provider="azure",
|
||||
)
|
||||
|
||||
|
||||
def test_no_retry_when_no_healthy_deployments():
|
||||
healthy_deployments = []
|
||||
|
||||
router = Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "gpt-3.5-turbo",
|
||||
"litellm_params": {
|
||||
"model": "azure/chatgpt-v-2",
|
||||
"api_key": os.getenv("AZURE_API_KEY"),
|
||||
"api_version": os.getenv("AZURE_API_VERSION"),
|
||||
"api_base": os.getenv("AZURE_API_BASE"),
|
||||
},
|
||||
}
|
||||
]
|
||||
)
|
||||
|
||||
for error in [
|
||||
internal_server_error,
|
||||
rate_limit_error,
|
||||
service_unavailable_error,
|
||||
timeout_error,
|
||||
]:
|
||||
try:
|
||||
response = router.should_retry_this_error(
|
||||
error=error, healthy_deployments=healthy_deployments
|
||||
)
|
||||
pytest.fail(
|
||||
"Should have raised an exception, there's no point retrying an error when there are 0 healthy deployments"
|
||||
)
|
||||
except Exception as e:
|
||||
print("got exception", e)
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue