feat(router): discover token limits for hosted OpenAI-compatible models

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

litellm/llms/openai_like/model_info.py

@@ -0,0 +1,90 @@
+import hashlib
+import json
+from collections.abc import Mapping
+from types import MappingProxyType
+from typing import Annotated, Final, TypeAlias
+
+import httpx
+from pydantic import BaseModel, BeforeValidator, ConfigDict
+
+from litellm._logging import verbose_logger
+from litellm.caching.in_memory_cache import InMemoryCache
+from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
+from litellm.utils import _add_path_to_api_base  # pyright: ignore[reportPrivateUsage]  # shared provider URL helper
+
+MODEL_INFO_REFRESH_SECONDS: Final = 300
+_EMPTY_LIMITS: Final[Mapping[str, int]] = MappingProxyType({})
+
+
+def _positive_limit(value: object) -> int | None:
+    return value if isinstance(value, int) and not isinstance(value, bool) and value > 0 else None
+
+
+_TokenLimit: TypeAlias = Annotated[int | None, BeforeValidator(_positive_limit)]
+
+
+class _ModelCard(BaseModel):
+    model_config = ConfigDict(frozen=True)
+
+    id: str
+    max_model_len: _TokenLimit = None
+    context_length: _TokenLimit = None
+    max_input_tokens: _TokenLimit = None
+    max_output_tokens: _TokenLimit = None
+
+    def token_limits(self) -> Mapping[str, int]:
+        context: Final = self.max_model_len or self.context_length
+        input_limit: Final = self.max_input_tokens or context
+        output_limit: Final = self.max_output_tokens or context
+        return MappingProxyType(
+            {
+                key: value
+                for key, value in (
+                    ("max_tokens", context),
+                    ("max_input_tokens", min(input_limit, context) if input_limit and context else input_limit),
+                    ("max_output_tokens", min(output_limit, context) if output_limit and context else output_limit),
+                )
+                if value is not None
+            }
+        )
+
+
+class _ModelList(BaseModel):
+    model_config = ConfigDict(frozen=True)
+
+    data: tuple[_ModelCard, ...] = ()
+
+
+async def get_openai_compatible_model_info(
+    *,
+    model: str,
+    api_base: str,
+    headers: Mapping[str, str],
+    client: AsyncHTTPHandler,
+    cache: InMemoryCache,
+) -> Mapping[str, int]:
+    url: Final = _add_path_to_api_base(api_base, "/v1/models")
+    cache_key: Final = (
+        "upstream_model_info:" + hashlib.sha256(json.dumps((url, sorted(headers.items()))).encode()).hexdigest()
+    )
+    cached: Final[object] = cache.get_cache(cache_key)
+    if isinstance(cached, _ModelList):
+        return next((card.token_limits() for card in cached.data if card.id == model), _EMPTY_LIMITS)
+
+    try:
+        response: Final = await client.get(
+            url=url,
+            headers=dict(headers),  # mutable-ok: AsyncHTTPHandler requires a concrete dict
+            timeout=httpx.Timeout(5.0),
+            follow_redirects=False,
+            max_response_bytes=2 * 1024 * 1024,
+        )
+        response.raise_for_status()
+        models: Final = _ModelList.model_validate_json(response.content)
+    except Exception:  # noqa: BLE001  # optional upstream metadata must not interrupt proxy refresh
+        verbose_logger.debug("Could not discover upstream model token limits")
+        cache.set_cache(cache_key, _ModelList(), ttl=60)
+        return _EMPTY_LIMITS
+
+    cache.set_cache(cache_key, models, ttl=MODEL_INFO_REFRESH_SECONDS)
+    return next((card.token_limits() for card in models.data if card.id == model), _EMPTY_LIMITS)

litellm/proxy/proxy_server.py

@@ -305,6 +305,7 @@ from litellm.litellm_core_utils.sensitive_data_masker import (
     mask_sensitive_keys,
 )
 from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
+from litellm.llms.openai_like.model_info import MODEL_INFO_REFRESH_SECONDS
 from litellm.llms.vertex_ai.vertex_llm_base import VertexBase
 from litellm.proxy._lazy_features import attach_lazy_features, reserve_lazy_slot
 from litellm.proxy._types import *
@@ -1373,9 +1374,27 @@ async def proxy_startup_event(app: FastAPI) -> AsyncGenerator[None, None]:
     ## Initialize shared aiohttp session for connection reuse
     shared_aiohttp_session = await _initialize_shared_aiohttp_session()
 
+    model_info_scheduler: Final = scheduler if scheduler is not None else AsyncIOScheduler()
+    model_info_scheduler.add_job(
+        ProxyStartupEvent.refresh_model_info,
+        "interval",
+        seconds=MODEL_INFO_REFRESH_SECONDS,
+        id="refresh_model_info",
+        next_run_time=datetime.now(timezone.utc),
+        max_instances=1,
+        replace_existing=True,
+    )
+    if not model_info_scheduler.running:
+        model_info_scheduler.start()
+
     # End of startup event
     yield
 
+    if model_info_scheduler.running:
+        model_info_scheduler.remove_job("refresh_model_info")
+        if model_info_scheduler is not scheduler:
+            model_info_scheduler.shutdown(wait=False)
+
     # Shutdown event - drain in-flight requests before tearing down dependencies
     # so SIGTERM (rolling update, scale-down, liveness kill) doesn't drop them.
     GracefulShutdownManager.start_shutdown()
@@ -9293,6 +9312,12 @@ def get_litellm_model_info(model: dict = {}):
     model_info: Final = model.get("model_info", {})
     model_to_lookup = model.get("litellm_params", {}).get("model", None)
     try:
+        if llm_router is not None and model_info.get("id") is not None:
+            deployment_info: Final = llm_router.get_deployment_model_info(
+                model_id=model_info["id"], model_name=model_to_lookup
+            )
+            if deployment_info is not None:
+                return deployment_info
         if "azure" in model_to_lookup or model_info.get("base_model"):
             model_to_lookup = model_info.get("base_model", None)
         litellm_model_info: Final = litellm.get_model_info(model_to_lookup)
@@ -9325,6 +9350,11 @@ def giveup(e):
 
 
 class ProxyStartupEvent:
+    @staticmethod
+    async def refresh_model_info() -> None:
+        if llm_router is not None:
+            await llm_router.arefresh_model_info()
+
     @staticmethod
     def _warn_budget_without_db(max_budget: float | None, prisma_client: PrismaClient | None) -> None:
         if prisma_client is not None or not max_budget or max_budget <= 0:
@@ -13597,7 +13627,7 @@ def _enrich_model_info_with_litellm_data(
             except Exception:
                 litellm_model_info = {}
     for k, v in litellm_model_info.items():
-        if k not in model_info:
+        if model_info.get(k) is None:
             model_info[k] = v
     model["model_info"] = model_info
     # don't return the api key / vertex credentials

litellm/router.py

@@ -109,7 +109,9 @@ from litellm.llms.base_llm.vector_store.transformation import (
     RouterVectorStoreEmbeddingExecutor,
     vector_store_request_metadata,
 )
+from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, get_async_httpx_client
 from litellm.llms.openai_like.json_loader import JSONProviderRegistry
+from litellm.llms.openai_like.model_info import get_openai_compatible_model_info
 from litellm.router_strategy.budget_limiter import RouterBudgetLimiting
 from litellm.router_strategy.least_busy import LeastBusyLoggingHandler
 from litellm.router_strategy.lowest_cost import LowestCostLoggingHandler
@@ -10316,11 +10318,67 @@ class Router:
             return None
         return Deployment(**first_usable) if isinstance(first_usable, dict) else first_usable
 
+    async def arefresh_model_info(self, *, client: AsyncHTTPHandler | None = None) -> None:
+        """Refresh token limits advertised by configured OpenAI-compatible deployments."""
+        for raw_deployment in tuple(self.model_list):
+            try:
+                deployment: Final = Deployment.model_validate(raw_deployment)
+                params: Final = LiteLLM_Params.model_validate(
+                    MappingProxyType(
+                        {
+                            **deployment.litellm_params.model_dump(exclude_none=True),
+                            **(
+                                self.get_deployment_credentials_with_provider(deployment.model_info.id or "")
+                                or MappingProxyType({})
+                            ),
+                        }
+                    )
+                )
+                model, provider, dynamic_api_key, api_base = litellm.get_llm_provider(
+                    model=params.model, litellm_params=params
+                )
+                if provider not in ("hosted_vllm", "openai", "text-completion-openai", "openai_like"):
+                    continue
+                if api_base is None or "*" in model or params.get("use_clientside_credentials"):
+                    continue
+                api_key: Final = params.api_key or dynamic_api_key
+                headers: Final = TypeAdapter(Mapping[str, str]).validate_python(
+                    params.get("extra_headers") or params.get("headers") or MappingProxyType({})
+                )
+                auth_headers: Final = (
+                    MappingProxyType({"authorization": f"Bearer {api_key}"}) if api_key else MappingProxyType({})
+                )
+                limits: Final = await get_openai_compatible_model_info(
+                    model=model,
+                    api_base=api_base,
+                    headers=MappingProxyType(
+                        {
+                            **auth_headers,
+                            **MappingProxyType({key.lower(): value for key, value in headers.items()}),
+                        }
+                    ),
+                    client=client or get_async_httpx_client(llm_provider=LlmProviders.OPENAI),
+                    cache=self.cache.in_memory_cache,
+                )
+                model_id: Final = deployment.model_info.id
+                if not limits or model_id is None or self.get_model_info(model_id) is not raw_deployment:
+                    continue
+                litellm.register_model(
+                    model_cost={  # mutable-ok: register_model requires a concrete dict at its public boundary
+                        model_id: MappingProxyType({**limits, **self._deployment_model_cost_payload(deployment)}),
+                    },
+                    persist_across_reloads=False,
+                    warning_display_name=params.model,
+                )
+                self._invalidate_model_group_info_cache()
+            except Exception:  # noqa: BLE001  # one invalid deployment must not prevent refreshing the others
+                verbose_router_logger.debug("Could not refresh deployment model info")
+
     def get_model_listing_info(self, model_name: str) -> DeploymentModelListingInfo | None:
         """
         Return what the concrete deployments behind model_name contribute to its
         /v1/models entry: the cost-map keys for their underlying models, plus the widest
-        token limits explicitly configured in their model_info. Resolved via O(1) index
+        configured or discovered token limits. Resolved via O(1) index
         lookup.
 
         Returns None for wildcard-expanded or unknown names, where the listed name is the
@@ -10340,7 +10398,24 @@ class Router:
             return None
 
         deployments: Final = tuple(self.model_list[index] for index in indices)
-        model_infos: Final = tuple(deployment.get("model_info") or MappingProxyType({}) for deployment in deployments)
+        model_infos: Final = tuple(
+            MappingProxyType(
+                {
+                    **(
+                        litellm.model_cost.get((deployment.get("model_info") or MappingProxyType({})).get("id"))
+                        or MappingProxyType({})
+                    ),
+                    **MappingProxyType(
+                        {
+                            k: v
+                            for k, v in (deployment.get("model_info") or MappingProxyType({})).items()
+                            if v is not None
+                        }
+                    ),
+                }
+            )
+            for deployment in deployments
+        )
         params: Final = tuple(deployment.get("litellm_params") or MappingProxyType({}) for deployment in deployments)
         # base_model resolution mirrors get_router_model_info: unset or blank means the
         # deployment's own model name is the cost-map key.
@@ -10372,8 +10447,8 @@ class Router:
 
     def get_configured_token_limits(self, model_name: str) -> "tuple[int | None, int | None]":
         """
-        Return (max_input_tokens, max_output_tokens) explicitly configured in a concrete
-        deployment's model_info for model_name, via O(1) index lookup.
+        Return (max_input_tokens, max_output_tokens) configured or discovered for a concrete
+        deployment of model_name, via O(1) index lookup.
 
         Returns (None, None) for wildcard-expanded or unknown names, and treats a
         malformed configured value as absent rather than failing the caller.
@@ -10386,7 +10461,12 @@ class Router:
         if deployment is None:
             return (None, None)
 
-        model_info: Final = deployment.model_info
+        model_info: Final = MappingProxyType(
+            {
+                **(litellm.model_cost.get(deployment.model_info.id) or MappingProxyType({})),
+                **deployment.model_info.model_dump(exclude_none=True),
+            }
+        )
         return (
             coerce_token_limit(model_info.get("max_input_tokens")),
             coerce_token_limit(model_info.get("max_output_tokens")),
@@ -10651,11 +10731,13 @@ class Router:
 
         # get_model_info() hands back an lru_cache'd dict, so merge into a copy; unset
         # values are skipped or Deployment's None pricing defaults would erase the map's
-        merged_model_info: Final = copy.deepcopy(model_info)
-        if user_model_info:
-            for key, value in user_model_info.items():
-                if value is not None:
-                    merged_model_info[key] = value
+        merged_model_info: Final[ModelMapInfo] = {
+            **copy.deepcopy(model_info),
+            **copy.deepcopy(litellm.model_cost.get((deployment.get("model_info") or {}).get("id")) or {}),
+            **MappingProxyType(
+                {key: value for key, value in (user_model_info or MappingProxyType({})).items() if value is not None}
+            ),
+        }
 
         return merged_model_info
 

tests/test_litellm/llms/openai_like/test_model_info.py

@@ -0,0 +1,126 @@
+from collections.abc import Mapping
+from typing import Final
+from unittest.mock import Mock
+
+import httpx
+import pytest
+
+from litellm.caching.in_memory_cache import InMemoryCache
+from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
+from litellm.llms.openai_like.model_info import (
+    MODEL_INFO_REFRESH_SECONDS,
+    get_openai_compatible_model_info,
+)
+
+
+@pytest.mark.parametrize(
+    ("card", "expected"),
+    (
+        ({"max_model_len": 8192}, {"max_tokens": 8192, "max_input_tokens": 8192, "max_output_tokens": 8192}),
+        (
+            {"context_length": 4096, "max_output_tokens": 1024},
+            {"max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 1024},
+        ),
+        (
+            {"max_model_len": 4096, "max_input_tokens": 2048, "max_output_tokens": 8192},
+            {"max_tokens": 4096, "max_input_tokens": 2048, "max_output_tokens": 4096},
+        ),
+        ({"max_input_tokens": 2048}, {"max_input_tokens": 2048}),
+        ({"max_output_tokens": 1024}, {"max_output_tokens": 1024}),
+        ({"max_model_len": True, "max_output_tokens": -1}, {}),
+        ({"max_model_len": "8192", "max_input_tokens": 0, "max_output_tokens": 1.5}, {}),
+        ({}, {}),
+    ),
+)
+async def test_discovers_only_valid_advertised_limits(card: Mapping[str, object], expected: Mapping[str, int]) -> None:
+    def respond(request: httpx.Request) -> httpx.Response:
+        assert request.url.path == "/tenant/v1/models"
+        assert request.headers["authorization"] == "Bearer local-key"
+        return httpx.Response(200, json={"data": [{"id": "org/model", **card}]})
+
+    handler: Final = AsyncHTTPHandler()
+    await handler.client.aclose()
+    async with httpx.AsyncClient(transport=httpx.MockTransport(respond)) as client:
+        handler.client = client
+        cache: Final = InMemoryCache()
+        result: Final = await get_openai_compatible_model_info(
+            model="org/model",
+            api_base="https://backend.test/tenant/v1/",
+            headers={"Authorization": "Bearer local-key"},
+            client=handler,
+            cache=cache,
+        )
+        assert result == expected
+        assert (
+            await get_openai_compatible_model_info(
+                model="missing",
+                api_base="https://backend.test/tenant/v1/",
+                headers={"Authorization": "Bearer local-key"},
+                client=handler,
+                cache=cache,
+            )
+            == {}
+        )
+
+
+async def test_cache_is_scoped_to_endpoint_and_authentication_and_expires() -> None:
+    clock: Final = Mock(return_value=0)
+    responder: Final = Mock(
+        side_effect=(
+            httpx.Response(
+                200, json={"data": [{"id": "first", "max_model_len": 1024}, {"id": "second", "max_model_len": 2048}]}
+            ),
+            httpx.Response(200, json={"data": [{"id": "first", "max_model_len": 4096}]}),
+            httpx.Response(200, json={"data": [{"id": "first", "max_model_len": 8192}]}),
+            httpx.Response(200, json={"data": [{"id": "first", "max_model_len": 16384}]}),
+        )
+    )
+    handler: Final = AsyncHTTPHandler()
+    await handler.client.aclose()
+    async with httpx.AsyncClient(transport=httpx.MockTransport(responder)) as client:
+        handler.client = client
+        cache: Final = InMemoryCache(clock=clock)
+
+        async def lookup(model: str = "first", host: str = "one.test", key: str = "one") -> Mapping[str, int]:
+            return await get_openai_compatible_model_info(
+                model=model, api_base=f"https://{host}", headers={"Authorization": key}, client=handler, cache=cache
+            )
+
+        assert (await lookup())["max_input_tokens"] == 1024
+        assert (await lookup("second"))["max_input_tokens"] == 2048
+        assert responder.call_count == 1
+        assert (await lookup(key="two"))["max_input_tokens"] == 4096
+        assert (await lookup(host="two.test"))["max_input_tokens"] == 8192
+        clock.return_value = MODEL_INFO_REFRESH_SECONDS + 1
+        assert (await lookup())["max_input_tokens"] == 16384
+        assert responder.call_count == 4
+
+
+@pytest.mark.parametrize(
+    "response",
+    (
+        httpx.Response(404),
+        httpx.Response(401),
+        httpx.Response(302, headers={"location": "https://elsewhere.test"}),
+        httpx.Response(200, content=b"not json"),
+        httpx.Response(200, json={"data": None}),
+        httpx.ReadTimeout("backend unavailable"),
+    ),
+)
+async def test_unavailable_metadata_is_best_effort_and_negative_cached(
+    response: httpx.Response | Exception,
+) -> None:
+    responder: Final = Mock(side_effect=response if isinstance(response, Exception) else None, return_value=response)
+    handler: Final = AsyncHTTPHandler()
+    await handler.client.aclose()
+    async with httpx.AsyncClient(transport=httpx.MockTransport(responder), follow_redirects=True) as client:
+        handler.client = client
+        cache: Final = InMemoryCache()
+        for _ in range(2):
+            assert (
+                await get_openai_compatible_model_info(
+                    model="model", api_base="https://backend.test", headers={}, client=handler, cache=cache
+                )
+                == {}
+            )
+        assert responder.call_count == 1

tests/test_litellm/proxy/proxy_server/test_routes_model_info.py

@@ -9,14 +9,71 @@ Pins (PR2):
 
 from __future__ import annotations
 
+import copy
+from collections.abc import Callable
+from contextlib import AbstractContextManager
+from typing import Final
 from unittest.mock import MagicMock
 
+import httpx
 import pytest
+from fastapi.testclient import TestClient
 
+import litellm
+from litellm.caching.llm_caching_handler import LLMClientCache
+from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
 from litellm.proxy import proxy_server
+from litellm.utils import _invalidate_model_cost_lowercase_map
 
 from .conftest import normalize  # type: ignore[import-not-found]
 
+
+async def test_upstream_limits_reach_model_info_routes(
+    client: TestClient,
+    auth_as: Callable[[], AbstractContextManager[object]],
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    monkeypatch.setattr(litellm, "model_cost", copy.deepcopy(litellm.model_cost))
+    monkeypatch.setattr(litellm, "in_memory_llm_clients_cache", LLMClientCache())
+    router: Final = litellm.Router(
+        model_list=[
+            {
+                "model_name": "local",
+                "litellm_params": {
+                    "model": "hosted_vllm/org/local-model",
+                    "api_base": "https://backend.test/v1",
+                    "api_key": "local-key",
+                },
+                "model_info": {"id": "local-deployment", "max_output_tokens": 512, "max_input_tokens": None},
+            }
+        ]
+    )
+    monkeypatch.setattr(proxy_server, "llm_router", router)
+    monkeypatch.setattr(proxy_server, "llm_model_list", router.get_model_list())
+    monkeypatch.setattr(proxy_server, "user_model", None)
+
+    def respond(request: httpx.Request) -> httpx.Response:
+        assert request.url.path == "/v1/models"
+        return httpx.Response(200, json={"data": [{"id": "org/local-model", "max_model_len": 4096}]})
+
+    handler: Final = AsyncHTTPHandler()
+    await handler.client.aclose()
+    async with httpx.AsyncClient(transport=httpx.MockTransport(respond)) as upstream:
+        handler.client = upstream
+        litellm.in_memory_llm_clients_cache.set_cache("async_httpx_clientopenai", handler)
+        await proxy_server.ProxyStartupEvent.refresh_model_info()
+        with auth_as():
+            for path in ("/v1/model/info", "/model/info"):
+                response: Final = client.get(path)
+                assert response.status_code == 200, response.text
+                info: Final = response.json()["data"][0]["model_info"]
+                assert (info["max_input_tokens"], info["max_output_tokens"]) == (4096, 512)
+            group_response: Final = client.get("/model_group/info")
+            assert group_response.status_code == 200, group_response.text
+            assert group_response.json()["data"][0]["max_input_tokens"] == 4096
+    _invalidate_model_cost_lowercase_map()
+
+
 # ---------------------------------------------------------------------------
 # GET /v2/model/info
 # ---------------------------------------------------------------------------

tests/test_litellm/test_router_model_cost_isolation.py

@@ -11,14 +11,16 @@ import copy
 import logging
 import os
 import re
-from unittest.mock import patch
+from typing import Final
+from unittest.mock import Mock, patch
 
+import httpx
 import pytest
 
-
 import litellm
 from litellm import Router
 from litellm.litellm_core_utils.ptu_pricing import ptu_config_error
+from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
 from litellm.types.router import Deployment, LiteLLM_Params, ModelInfo
 from litellm.utils import (
     _invalidate_model_cost_lowercase_map,
@@ -60,6 +62,118 @@ def _restore_model_cost_entries(original_entries):
     _invalidate_model_cost_lowercase_map()
 
 
+@pytest.mark.parametrize("provider", ("hosted_vllm", "openai", "openai_like", "text-completion-openai"))
+async def test_discovered_limits_are_isolated_overridable_and_refreshable(
+    provider: str, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    monkeypatch.setattr(litellm, "model_cost", copy.deepcopy(litellm.model_cost))
+    upstream_limit: Final = iter((8192, 4096, 16384, 2048))
+
+    def respond(request: httpx.Request) -> httpx.Response:
+        assert request.url.path == "/v1/models"
+        assert request.headers["authorization"] == "Bearer local-key"
+        return httpx.Response(200, json={"data": [{"id": "org/local-model", "max_model_len": next(upstream_limit)}]})
+
+    router: Final = Router(
+        model_list=[
+            {
+                "model_name": "local",
+                "litellm_params": {
+                    "model": f"{provider}/org/local-model",
+                    "api_base": f"https://{host}.test/v1",
+                    "api_key": "local-key",
+                },
+                "model_info": {"id": host, **overrides},
+            }
+            for host, overrides in (("one", {}), ("two", {"max_output_tokens": 512}))
+        ],
+        enable_pre_call_checks=True,
+    )
+    handler: Final = AsyncHTTPHandler()
+    await handler.client.aclose()
+    async with httpx.AsyncClient(transport=httpx.MockTransport(respond)) as client:
+        handler.client = client
+        await router.arefresh_model_info(client=handler)
+        first: Final = router.get_router_model_info(id="one", deployment=None, received_model_name="local")
+        second: Final = router.get_router_model_info(id="two", deployment=None, received_model_name="local")
+        assert (first["max_input_tokens"], first["max_output_tokens"]) == (8192, 8192)
+        assert (second["max_input_tokens"], second["max_output_tokens"]) == (4096, 512)
+        group: Final = router.get_model_group_info("local")
+        assert group is not None
+        assert group.max_input_tokens == 8192
+        listing: Final = router.get_model_listing_info("local")
+        assert listing is not None
+        assert listing.max_input_tokens == 8192
+        assert router.get_configured_token_limits("local") == (8192, 8192)
+        assert router._deployment_max_input_tokens("local", router.model_list[1]) == 4096
+        allowed: Final = router._pre_call_checks(
+            model="local", healthy_deployments=router.model_list, input="prompt", input_token_count=5000
+        )
+        assert [deployment["model_info"]["id"] for deployment in allowed] == ["one"]
+        assert router.model_list[0]["model_info"].get("max_input_tokens") is None
+        assert litellm.model_cost[f"{provider}/org/local-model"].get("max_input_tokens") is None
+        router.cache.in_memory_cache.flush_cache()
+        await router.arefresh_model_info(client=handler)
+        refreshed: Final = router.get_model_group_info("local")
+        assert refreshed is not None
+        assert refreshed.max_input_tokens == 16384
+        assert (
+            router.get_router_model_info(id="two", deployment=None, received_model_name="local")["max_output_tokens"]
+            == 512
+        )
+    _invalidate_model_cost_lowercase_map()
+
+
+async def test_discovery_preserves_input_overrides_and_survives_outages(monkeypatch: pytest.MonkeyPatch) -> None:
+    monkeypatch.setattr(litellm, "model_cost", copy.deepcopy(litellm.model_cost))
+    responses: Final = iter((
+        httpx.Response(200, json={"data": [{"id": "local-model", "max_model_len": 4096}]}),
+        httpx.Response(503),
+    ))
+
+    def respond(request: httpx.Request) -> httpx.Response:
+        assert request.url.host == "backend.test"
+        assert request.headers["authorization"] == "Bearer local-key"
+        assert request.headers["x-tenant"] == "tenant"
+        return next(responses)
+
+    router: Final = Router(model_list=[
+        {
+            "model_name": "configured",
+            "litellm_params": {
+                "model": "hosted_vllm/local-model",
+                "api_base": "https://backend.test/v1",
+                "api_key": "unused-key",
+                "extra_headers": {"authorization": "Bearer local-key", "X-Tenant": "tenant"},
+            },
+            "model_info": {"id": "configured", "max_input_tokens": 1024},
+        },
+        {
+            "model_name": "byok",
+            "litellm_params": {
+                "model": "openai/local-model",
+                "api_base": "https://caller.test/v1",
+                "use_clientside_credentials": True,
+            },
+        },
+        {"model_name": "default-openai", "litellm_params": {"model": "openai/local-model", "api_key": "unused"}},
+    ])
+    handler: Final = AsyncHTTPHandler()
+    await handler.client.aclose()
+    responder: Final = Mock(side_effect=respond)
+    async with httpx.AsyncClient(transport=httpx.MockTransport(responder)) as client:
+        handler.client = client
+        await router.arefresh_model_info(client=handler)
+        assert router.get_configured_token_limits("configured") == (1024, 4096)
+        router.cache.in_memory_cache.flush_cache()
+        await router.arefresh_model_info(client=handler)
+        assert router.get_configured_token_limits("configured") == (1024, 4096)
+        assert router.get_configured_token_limits("byok") == (None, None)
+        assert next(responses, None) is None
+        assert responder.call_count == 2
+    _invalidate_model_cost_lowercase_map()
+
+
 def test_should_not_pollute_shared_key_with_zero_cost_pricing():
     """
     When deployment A has input_cost_per_token=0 and deployment B has no