diff --git a/litellm/router.py b/litellm/router.py
index 2d7bd517b28..e261c1743db 100644
--- a/litellm/router.py
+++ b/litellm/router.py
@@ -62,6 +62,8 @@ from litellm.router_utils.cooldown_callbacks import router_cooldown_handler
 from litellm.router_utils.fallback_event_handlers import (
     log_failure_fallback_event,
     log_success_fallback_event,
+    run_async_fallback,
+    run_sync_fallback,
 )
 from litellm.router_utils.handle_error import send_llm_exception_alert
 from litellm.scheduler import FlowItem, Scheduler
@@ -316,6 +318,7 @@ class Router:
         self.default_deployment = None  # use this to track the users default deployment, when they want to use model = *
         self.default_max_parallel_requests = default_max_parallel_requests
         self.provider_default_deployments: Dict[str, List] = {}
+        self.provider_default_deployment_ids: List[str] = []
 
         if model_list is not None:
             model_list = copy.deepcopy(model_list)
@@ -2383,34 +2386,16 @@ class Router:
                         if fallback_model_group is None:
                             raise original_exception
 
-                        for mg in fallback_model_group:
-                            """
-                            Iterate through the model groups and try calling that deployment
-                            """
-                            try:
-                                kwargs["model"] = mg
-                                kwargs.setdefault("metadata", {}).update(
-                                    {"model_group": mg}
-                                )  # update model_group used, if fallbacks are done
-                                response = await self.async_function_with_retries(
-                                    *args, **kwargs
-                                )
-                                verbose_router_logger.info(
-                                    "Successful fallback b/w models."
-                                )
-                                # callback for successfull_fallback_event():
-                                await log_success_fallback_event(
-                                    original_model_group=original_model_group,
-                                    kwargs=kwargs,
-                                )
+                        response = await run_async_fallback(
+                            *args,
+                            litellm_router=self,
+                            fallback_model_group=fallback_model_group,
+                            original_model_group=original_model_group,
+                            original_exception=original_exception,
+                            **kwargs,
+                        )
+                        return response
 
-                                return response
-                            except Exception as e:
-                                await log_failure_fallback_event(
-                                    original_model_group=original_model_group,
-                                    kwargs=kwargs,
-                                )
-                                pass
                     else:
                         error_message = "model={}. context_window_fallbacks={}. fallbacks={}.\n\nSet 'context_window_fallback' - https://docs.litellm.ai/docs/routing#fallbacks".format(
                             model_group, context_window_fallbacks, fallbacks
@@ -2436,33 +2421,15 @@ class Router:
                         if fallback_model_group is None:
                             raise original_exception
 
-                        for mg in fallback_model_group:
-                            """
-                            Iterate through the model groups and try calling that deployment
-                            """
-                            try:
-                                kwargs["model"] = mg
-                                kwargs.setdefault("metadata", {}).update(
-                                    {"model_group": mg}
-                                )  # update model_group used, if fallbacks are done
-                                response = await self.async_function_with_retries(
-                                    *args, **kwargs
-                                )
-                                verbose_router_logger.info(
-                                    "Successful fallback b/w models."
-                                )
-                                # callback for successfull_fallback_event():
-                                await log_success_fallback_event(
-                                    original_model_group=original_model_group,
-                                    kwargs=kwargs,
-                                )
-                                return response
-                            except Exception as e:
-                                await log_failure_fallback_event(
-                                    original_model_group=original_model_group,
-                                    kwargs=kwargs,
-                                )
-                                pass
+                        response = await run_async_fallback(
+                            *args,
+                            litellm_router=self,
+                            fallback_model_group=fallback_model_group,
+                            original_model_group=original_model_group,
+                            original_exception=original_exception,
+                            **kwargs,
+                        )
+                        return response
                     else:
                         error_message = "model={}. content_policy_fallback={}. fallbacks={}.\n\nSet 'content_policy_fallback' - https://docs.litellm.ai/docs/routing#fallbacks".format(
                             model_group, content_policy_fallbacks, fallbacks
@@ -2502,39 +2469,16 @@ class Router:
                         if hasattr(original_exception, "message"):
                             original_exception.message += f"No fallback model group found for original model_group={model_group}. Fallbacks={fallbacks}"
                         raise original_exception
-                    for mg in fallback_model_group:
-                        """
-                        Iterate through the model groups and try calling that deployment
-                        """
-                        try:
-                            ## LOGGING
-                            kwargs = self.log_retry(kwargs=kwargs, e=original_exception)
-                            verbose_router_logger.info(
-                                f"Falling back to model_group = {mg}"
-                            )
-                            kwargs["model"] = mg
-                            kwargs.setdefault("metadata", {}).update(
-                                {"model_group": mg}
-                            )  # update model_group used, if fallbacks are done
-                            response = await self.async_function_with_fallbacks(
-                                *args, **kwargs
-                            )
-                            verbose_router_logger.info(
-                                "Successful fallback b/w models."
-                            )
-                            # callback for successfull_fallback_event():
-                            await log_success_fallback_event(
-                                original_model_group=original_model_group,
-                                kwargs=kwargs,
-                            )
 
-                            return response
-                        except Exception as e:
-                            await log_failure_fallback_event(
-                                original_model_group=original_model_group,
-                                kwargs=kwargs,
-                            )
-                            raise e
+                    response = await run_async_fallback(
+                        *args,
+                        litellm_router=self,
+                        fallback_model_group=fallback_model_group,
+                        original_model_group=original_model_group,
+                        original_exception=original_exception,
+                        **kwargs,
+                    )
+                    return response
             except Exception as new_exception:
                 verbose_router_logger.error(
                     "litellm.router.py::async_function_with_fallbacks() - Error occurred while trying to do fallbacks - {}\n{}\n\nDebug Information:\nCooldown Deployments={}".format(
@@ -2708,6 +2652,8 @@ class Router:
         ):
             raise error
 
+        if isinstance(error, litellm.NotFoundError):
+            raise error
         # Error we should only retry if there are other deployments
         if isinstance(error, openai.RateLimitError):
             if (
@@ -2725,6 +2671,11 @@ class Router:
             if _num_healthy_deployments <= 0:  # if no healthy deployments
                 raise error  # then raise error
 
+        # Do not retry if there are no healthy deployments
+        # just raise the error
+        if _num_healthy_deployments <= 0:  # if no healthy deployments
+            raise error
+
         return True
 
     def function_with_fallbacks(self, *args, **kwargs):
@@ -2751,6 +2702,7 @@ class Router:
             return response
         except Exception as e:
             original_exception = e
+            original_model_group = kwargs.get("model")
             verbose_router_logger.debug(f"An exception occurs {original_exception}")
             try:
                 verbose_router_logger.debug(
@@ -2772,21 +2724,14 @@ class Router:
                     if fallback_model_group is None:
                         raise original_exception
 
-                    for mg in fallback_model_group:
-                        """
-                        Iterate through the model groups and try calling that deployment
-                        """
-                        try:
-                            ## LOGGING
-                            kwargs = self.log_retry(kwargs=kwargs, e=original_exception)
-                            kwargs["model"] = mg
-                            kwargs.setdefault("metadata", {}).update(
-                                {"model_group": mg}
-                            )  # update model_group used, if fallbacks are done
-                            response = self.function_with_fallbacks(*args, **kwargs)
-                            return response
-                        except Exception as e:
-                            pass
+                    return run_sync_fallback(
+                        *args,
+                        litellm_router=self,
+                        fallback_model_group=fallback_model_group,
+                        original_model_group=original_model_group,
+                        original_exception=original_exception,
+                        **kwargs,
+                    )
                 elif (
                     isinstance(e, litellm.ContentPolicyViolationError)
                     and content_policy_fallbacks is not None
@@ -2803,21 +2748,14 @@ class Router:
                     if fallback_model_group is None:
                         raise original_exception
 
-                    for mg in fallback_model_group:
-                        """
-                        Iterate through the model groups and try calling that deployment
-                        """
-                        try:
-                            ## LOGGING
-                            kwargs = self.log_retry(kwargs=kwargs, e=original_exception)
-                            kwargs["model"] = mg
-                            kwargs.setdefault("metadata", {}).update(
-                                {"model_group": mg}
-                            )  # update model_group used, if fallbacks are done
-                            response = self.function_with_fallbacks(*args, **kwargs)
-                            return response
-                        except Exception as e:
-                            pass
+                    return run_sync_fallback(
+                        *args,
+                        litellm_router=self,
+                        fallback_model_group=fallback_model_group,
+                        original_model_group=original_model_group,
+                        original_exception=original_exception,
+                        **kwargs,
+                    )
                 elif fallbacks is not None:
                     verbose_router_logger.debug(f"inside model fallbacks: {fallbacks}")
                     fallback_model_group = None
@@ -2841,21 +2779,14 @@ class Router:
                     if fallback_model_group is None:
                         raise original_exception
 
-                    for mg in fallback_model_group:
-                        """
-                        Iterate through the model groups and try calling that deployment
-                        """
-                        try:
-                            ## LOGGING
-                            kwargs = self.log_retry(kwargs=kwargs, e=original_exception)
-                            kwargs["model"] = mg
-                            kwargs.setdefault("metadata", {}).update(
-                                {"model_group": mg}
-                            )  # update model_group used, if fallbacks are done
-                            response = self.function_with_fallbacks(*args, **kwargs)
-                            return response
-                        except Exception as e:
-                            raise e
+                    return run_sync_fallback(
+                        *args,
+                        litellm_router=self,
+                        fallback_model_group=fallback_model_group,
+                        original_model_group=original_model_group,
+                        original_exception=original_exception,
+                        **kwargs,
+                    )
             except Exception as e:
                 raise e
             raise original_exception
@@ -3248,6 +3179,9 @@ class Router:
         if self._is_cooldown_required(exception_status=exception_status) == False:
             return
 
+        if deployment in self.provider_default_deployment_ids:
+            return
+
         _allowed_fails = self.get_allowed_fails_from_policy(
             exception=original_exception,
         )
@@ -3655,6 +3589,9 @@ class Router:
                     deployment.to_json(exclude_none=True)
                 ]
 
+            if deployment.model_info.id:
+                self.provider_default_deployment_ids.append(deployment.model_info.id)
+
         # Azure GPT-Vision Enhancements, users can pass os.environ/
         data_sources = deployment.litellm_params.get("dataSources", []) or []
 
diff --git a/litellm/router_utils/fallback_event_handlers.py b/litellm/router_utils/fallback_event_handlers.py
index 98d9cd92de9..fc1ce648d4e 100644
--- a/litellm/router_utils/fallback_event_handlers.py
+++ b/litellm/router_utils/fallback_event_handlers.py
@@ -1,9 +1,87 @@
-from typing import TYPE_CHECKING, Any
+from typing import TYPE_CHECKING, Any, Dict, List, Tuple
 
 import litellm
 from litellm._logging import verbose_router_logger
 from litellm.integrations.custom_logger import CustomLogger
 
+if TYPE_CHECKING:
+    from litellm.router import Router as _Router
+
+    LitellmRouter = _Router
+else:
+    LitellmRouter = Any
+
+
+async def run_async_fallback(
+    litellm_router: LitellmRouter,
+    *args: Tuple[Any],
+    fallback_model_group: List[str],
+    original_model_group: str,
+    original_exception: Exception,
+    **kwargs,
+) -> Any:
+    """
+    Iterate through the model groups and try calling that deployment.
+    """
+    error_from_fallbacks = original_exception
+    for mg in fallback_model_group:
+        if mg == original_model_group:
+            continue
+        try:
+            # LOGGING
+            kwargs = litellm_router.log_retry(kwargs=kwargs, e=original_exception)
+            verbose_router_logger.info(f"Falling back to model_group = {mg}")
+            kwargs["model"] = mg
+            kwargs.setdefault("metadata", {}).update(
+                {"model_group": mg}
+            )  # update model_group used, if fallbacks are done
+            response = await litellm_router.async_function_with_fallbacks(
+                *args, **kwargs
+            )
+            verbose_router_logger.info("Successful fallback b/w models.")
+            # callback for successfull_fallback_event():
+            await log_success_fallback_event(
+                original_model_group=original_model_group, kwargs=kwargs
+            )
+            return response
+        except Exception as e:
+            error_from_fallbacks = e
+            await log_failure_fallback_event(
+                original_model_group=original_model_group, kwargs=kwargs
+            )
+    raise error_from_fallbacks
+
+
+def run_sync_fallback(
+    litellm_router: LitellmRouter,
+    *args: Tuple[Any],
+    fallback_model_group: List[str],
+    original_model_group: str,
+    original_exception: Exception,
+    **kwargs,
+) -> Any:
+    """
+    Iterate through the model groups and try calling that deployment.
+    """
+    error_from_fallbacks = original_exception
+    for mg in fallback_model_group:
+        if mg == original_model_group:
+            continue
+        try:
+            # LOGGING
+            kwargs = litellm_router.log_retry(kwargs=kwargs, e=original_exception)
+            verbose_router_logger.info(f"Falling back to model_group = {mg}")
+            kwargs["model"] = mg
+            kwargs.setdefault("metadata", {}).update(
+                {"model_group": mg}
+            )  # update model_group used, if fallbacks are done
+            response = litellm_router.function_with_fallbacks(*args, **kwargs)
+            verbose_router_logger.info("Successful fallback b/w models.")
+            return response
+        except Exception as e:
+            error_from_fallbacks = e
+    raise error_from_fallbacks
+
 
 async def log_success_fallback_event(original_model_group: str, kwargs: dict):
     for _callback in litellm.callbacks:
diff --git a/litellm/tests/test_router_fallbacks.py b/litellm/tests/test_router_fallbacks.py
index 2c552a64bfd..a9a225be7c5 100644
--- a/litellm/tests/test_router_fallbacks.py
+++ b/litellm/tests/test_router_fallbacks.py
@@ -1185,3 +1185,82 @@ async def test_router_content_policy_fallbacks(
         )
 
     assert response.model == "my-fake-model"
+
+
+@pytest.mark.parametrize("sync_mode", [False, True])
+@pytest.mark.asyncio
+async def test_using_default_fallback(sync_mode):
+    litellm.set_verbose = True
+
+    import logging
+
+    from litellm._logging import verbose_logger, verbose_router_logger
+
+    verbose_logger.setLevel(logging.DEBUG)
+    verbose_router_logger.setLevel(logging.DEBUG)
+    litellm.default_fallbacks = ["very-bad-model"]
+    router = Router(
+        model_list=[
+            {
+                "model_name": "openai/*",
+                "litellm_params": {
+                    "model": "openai/*",
+                    "api_key": os.getenv("OPENAI_API_KEY"),
+                },
+            },
+        ],
+    )
+    try:
+        if sync_mode:
+            response = router.completion(
+                model="openai/foo",
+                messages=[{"role": "user", "content": "Hey, how's it going?"}],
+            )
+        else:
+            response = await router.acompletion(
+                model="openai/foo",
+                messages=[{"role": "user", "content": "Hey, how's it going?"}],
+            )
+        print("got response=", response)
+        pytest.fail(f"Expected call to fail we passed model=openai/foo")
+    except Exception as e:
+        print("got exception = ", e)
+        assert "No healthy deployment available, passed model=very-bad-model" in str(e)
+
+
+@pytest.mark.parametrize("sync_mode", [False])
+@pytest.mark.asyncio
+async def test_using_default_working_fallback(sync_mode):
+    litellm.set_verbose = True
+
+    import logging
+
+    from litellm._logging import verbose_logger, verbose_router_logger
+
+    verbose_logger.setLevel(logging.DEBUG)
+    verbose_router_logger.setLevel(logging.DEBUG)
+    litellm.default_fallbacks = ["openai/gpt-3.5-turbo"]
+    router = Router(
+        model_list=[
+            {
+                "model_name": "openai/*",
+                "litellm_params": {
+                    "model": "openai/*",
+                    "api_key": os.getenv("OPENAI_API_KEY"),
+                },
+            },
+        ],
+    )
+
+    if sync_mode:
+        response = router.completion(
+            model="openai/foo",
+            messages=[{"role": "user", "content": "Hey, how's it going?"}],
+        )
+    else:
+        response = await router.acompletion(
+            model="openai/foo",
+            messages=[{"role": "user", "content": "Hey, how's it going?"}],
+        )
+    print("got response=", response)
+    assert response is not None
diff --git a/litellm/tests/test_router_retries.py b/litellm/tests/test_router_retries.py
index db2a1c6465c..f0503cd3f18 100644
--- a/litellm/tests/test_router_retries.py
+++ b/litellm/tests/test_router_retries.py
@@ -1,18 +1,24 @@
 #### What this tests ####
 #    This tests calling router with fallback models
 
-import sys, os, time
-import traceback, asyncio
+import asyncio
+import os
+import sys
+import time
+import traceback
+
 import pytest
 
 sys.path.insert(
     0, os.path.abspath("../..")
 )  # Adds the parent directory to the system path
 
+import httpx
+import openai
+
 import litellm
 from litellm import Router
 from litellm.integrations.custom_logger import CustomLogger
-import openai, httpx
 
 
 class MyCustomHandler(CustomLogger):
@@ -127,7 +133,7 @@ async def test_router_retries_errors(sync_mode, error_type):
     ["AuthenticationErrorRetries", "ContentPolicyViolationErrorRetries"],  #
 )
 async def test_router_retry_policy(error_type):
-    from litellm.router import RetryPolicy, AllowedFailsPolicy
+    from litellm.router import AllowedFailsPolicy, RetryPolicy
 
     retry_policy = RetryPolicy(
         ContentPolicyViolationErrorRetries=3, AuthenticationErrorRetries=0
@@ -386,7 +392,7 @@ def test_retry_rate_limit_error_with_healthy_deployments():
 
 def test_do_retry_rate_limit_error_with_no_fallbacks_and_no_healthy_deployments():
     """
-    Test 2. It SHOULD Retry, when healthy_deployments is [] and fallbacks is None
+    Test 2. It SHOULD NOT Retry, when healthy_deployments is [] and fallbacks is None
     """
     healthy_deployments = []
 
@@ -409,9 +415,10 @@ def test_do_retry_rate_limit_error_with_no_fallbacks_and_no_healthy_deployments(
         response = router.should_retry_this_error(
             error=rate_limit_error, healthy_deployments=healthy_deployments
         )
-        assert response == True
+        pytest.fail("Should have raised an error")
     except Exception as e:
-        pytest.fail("Should not have failed this error - {}".format(str(e)))
+        print("got an exception", e)
+        pass
 
 
 def test_raise_context_window_exceeded_error():
@@ -582,3 +589,96 @@ def test_timeout_for_rate_limit_error_with_no_healthy_deployments():
     )
 
     assert _timeout > 0.0
+
+
+def test_no_retry_for_not_found_error_404():
+    healthy_deployments = []
+
+    router = Router(
+        model_list=[
+            {
+                "model_name": "gpt-3.5-turbo",
+                "litellm_params": {
+                    "model": "azure/chatgpt-v-2",
+                    "api_key": os.getenv("AZURE_API_KEY"),
+                    "api_version": os.getenv("AZURE_API_VERSION"),
+                    "api_base": os.getenv("AZURE_API_BASE"),
+                },
+            }
+        ]
+    )
+
+    # Act & Assert
+    error = litellm.NotFoundError(
+        message="404 model not found",
+        model="gpt-12",
+        llm_provider="azure",
+    )
+    try:
+        response = router.should_retry_this_error(
+            error=error, healthy_deployments=healthy_deployments
+        )
+        pytest.fail(
+            "Should have raised an exception 404 NotFoundError should never be retried, it's typically model_not_found error"
+        )
+    except Exception as e:
+        print("got exception", e)
+
+
+internal_server_error = litellm.InternalServerError(
+    message="internal server error",
+    model="gpt-12",
+    llm_provider="azure",
+)
+
+rate_limit_error = litellm.RateLimitError(
+    message="rate limit error",
+    model="gpt-12",
+    llm_provider="azure",
+)
+
+service_unavailable_error = litellm.ServiceUnavailableError(
+    message="service unavailable error",
+    model="gpt-12",
+    llm_provider="azure",
+)
+
+timeout_error = litellm.Timeout(
+    message="timeout error",
+    model="gpt-12",
+    llm_provider="azure",
+)
+
+
+def test_no_retry_when_no_healthy_deployments():
+    healthy_deployments = []
+
+    router = Router(
+        model_list=[
+            {
+                "model_name": "gpt-3.5-turbo",
+                "litellm_params": {
+                    "model": "azure/chatgpt-v-2",
+                    "api_key": os.getenv("AZURE_API_KEY"),
+                    "api_version": os.getenv("AZURE_API_VERSION"),
+                    "api_base": os.getenv("AZURE_API_BASE"),
+                },
+            }
+        ]
+    )
+
+    for error in [
+        internal_server_error,
+        rate_limit_error,
+        service_unavailable_error,
+        timeout_error,
+    ]:
+        try:
+            response = router.should_retry_this_error(
+                error=error, healthy_deployments=healthy_deployments
+            )
+            pytest.fail(
+                "Should have raised an exception,  there's no point retrying an error when there are 0 healthy deployments"
+            )
+        except Exception as e:
+            print("got exception", e)