mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
fix(proxy): answer model lists from cached listings first and share one listing per server
Some checks are pending
Some checks are pending
This commit is contained in:
parent
83d52c7646
commit
42ee801409
8 changed files with 194 additions and 21 deletions
|
|
@ -613,6 +613,7 @@ NON_LLM_CONNECTION_TIMEOUT: Final = get_env_int(
|
|||
) # timeout for adjacent services (e.g. jwt auth)
|
||||
PROVIDER_MODEL_DISCOVERY_TIMEOUT_CACHE_TTL_SECONDS: Final = 60
|
||||
PROVIDER_MODEL_DISCOVERY_MAX_THREADS: Final = 4
|
||||
PROVIDER_MODEL_DISCOVERY_LISTING_LOCK_STRIPES: Final = 64
|
||||
MAX_EXCEPTION_MESSAGE_LENGTH: Final = get_env_int("MAX_EXCEPTION_MESSAGE_LENGTH", 2000)
|
||||
MAX_STRING_LENGTH_PROMPT_IN_DB: Final = get_env_int("MAX_STRING_LENGTH_PROMPT_IN_DB", 2048)
|
||||
MAX_STRING_LENGTH_COMPACTION_SUMMARY_IN_DB: Final = 64 * 1024
|
||||
|
|
|
|||
|
|
@ -22,7 +22,7 @@ from litellm.router_utils.fallback_event_handlers import get_fallback_model_grou
|
|||
from litellm.router_utils.pattern_match_deployments import PatternMatchRouter
|
||||
from litellm.types.router import CredentialLiteLLMParams, LiteLLM_Params
|
||||
from litellm.types.utils import LlmProviders
|
||||
from litellm.utils import ProviderConfigManager, get_valid_models
|
||||
from litellm.utils import ProviderConfigManager, answer_from_cached_provider_listings, get_valid_models
|
||||
|
||||
LLM_PROVIDER_PREFIXES: Final = frozenset(llm_provider.value for llm_provider in LlmProviders)
|
||||
|
||||
|
|
@ -40,6 +40,11 @@ async def run_model_discovery(
|
|||
*args: _P.args,
|
||||
**kwargs: _P.kwargs, # kwargs-ok: forwards the discovery function's own arguments
|
||||
) -> _R:
|
||||
cached_answer, needs_provider_listing = answer_from_cached_provider_listings(
|
||||
functools.partial(discover, *args, **kwargs)
|
||||
)
|
||||
if not needs_provider_listing:
|
||||
return cached_answer
|
||||
context: Final = contextvars.copy_context()
|
||||
return await asyncio.get_running_loop().run_in_executor(
|
||||
_MODEL_DISCOVERY_EXECUTOR, lambda: context.run(discover, *args, **kwargs)
|
||||
|
|
|
|||
|
|
@ -509,6 +509,18 @@ _UNKNOWN_CREATION_TIME: Final = datetime.max.replace(tzinfo=timezone.utc)
|
|||
class _RouterDeployment(TypedDict):
|
||||
model_name: ReadOnly[NotRequired[str]]
|
||||
model_info: ReadOnly[Mapping[str, object]]
|
||||
litellm_params: ReadOnly[Mapping[str, object]]
|
||||
|
||||
|
||||
def _deployment_serves_provider(llm_router: "Router", deployment: _RouterDeployment, custom_llm_provider: str) -> bool:
|
||||
litellm_params: Final = deployment["litellm_params"]
|
||||
model: Final = litellm_params.get("model")
|
||||
deployment_provider: Final = litellm_params.get("custom_llm_provider")
|
||||
return (
|
||||
isinstance(model, str)
|
||||
and llm_router.credential_provider(deployment_provider if isinstance(deployment_provider, str) else None, model)
|
||||
== custom_llm_provider
|
||||
)
|
||||
|
||||
|
||||
def _team_deployment_rank(model_info: Mapping[str, object], deployment_id: str) -> tuple[datetime, str]:
|
||||
|
|
@ -632,6 +644,11 @@ def get_team_provider_credentials(
|
|||
|
||||
if not include_accessible_deployments:
|
||||
return None
|
||||
if not any(
|
||||
_deployment_serves_provider(llm_router, deployment, custom_llm_provider)
|
||||
for deployment in llm_router.model_list or []
|
||||
):
|
||||
return None
|
||||
|
||||
# 2. Fall back to deployments the caller is allowed to access. The key's
|
||||
# effective allowlist (sentinels and access groups already expanded by
|
||||
|
|
|
|||
|
|
@ -12379,18 +12379,20 @@ class Router:
|
|||
credentials.pop("litellm_credential_name", None)
|
||||
|
||||
credentials["model"] = deployment.litellm_params.model
|
||||
|
||||
# Add custom_llm_provider
|
||||
if deployment.litellm_params.custom_llm_provider:
|
||||
credentials["custom_llm_provider"] = deployment.litellm_params.custom_llm_provider
|
||||
elif "/" in deployment.litellm_params.model:
|
||||
# Extract provider from "provider/model" format
|
||||
credentials["custom_llm_provider"] = deployment.litellm_params.model.split("/")[0]
|
||||
else:
|
||||
credentials["custom_llm_provider"] = "openai" # default
|
||||
credentials["custom_llm_provider"] = self.credential_provider(
|
||||
deployment.litellm_params.custom_llm_provider, deployment.litellm_params.model
|
||||
)
|
||||
|
||||
return credentials
|
||||
|
||||
@staticmethod
|
||||
def credential_provider(custom_llm_provider: str | None, model: str) -> str:
|
||||
if custom_llm_provider:
|
||||
return custom_llm_provider
|
||||
if "/" in model:
|
||||
return model.split("/")[0]
|
||||
return "openai"
|
||||
|
||||
@overload
|
||||
def get_router_model_info(
|
||||
self,
|
||||
|
|
|
|||
|
|
@ -79,6 +79,7 @@ from litellm.constants import (
|
|||
MINIMUM_PROMPT_CACHE_TOKEN_COUNT_OVERRIDE,
|
||||
NON_INFERENCE_CALL_TYPES,
|
||||
OPENAI_EMBEDDING_PARAMS,
|
||||
PROVIDER_MODEL_DISCOVERY_LISTING_LOCK_STRIPES,
|
||||
PROVIDER_MODEL_DISCOVERY_TIMEOUT_CACHE_TTL_SECONDS,
|
||||
PROVIDERS_THAT_AUTHENTICATE_ON_PROVIDER_INFO,
|
||||
TOOL_CHOICE_OBJECT_TOKEN_COUNT,
|
||||
|
|
@ -8371,6 +8372,13 @@ class AvailableModelsCache(InMemoryCache):
|
|||
super().__init__(ttl_seconds, max_size, clock=clock)
|
||||
self._env_hash: str | None = None
|
||||
self._listing_lock = threading.Lock()
|
||||
self._single_flight_locks: Final = tuple(
|
||||
threading.RLock() for _ in range(PROVIDER_MODEL_DISCOVERY_LISTING_LOCK_STRIPES)
|
||||
)
|
||||
|
||||
def single_flight_lock(self, custom_llm_provider: str, litellm_params: LiteLLM_Params | None) -> threading.RLock:
|
||||
cache_key: Final = self._get_cache_key(custom_llm_provider, litellm_params)
|
||||
return self._single_flight_locks[hash(cache_key) % len(self._single_flight_locks)]
|
||||
|
||||
def _get_env_hash(self) -> str:
|
||||
"""Create a hash of relevant environment variables"""
|
||||
|
|
@ -8451,6 +8459,22 @@ class AvailableModelsCache(InMemoryCache):
|
|||
# Global cache instance
|
||||
_model_cache: Final = AvailableModelsCache()
|
||||
|
||||
_UNCACHED_PROVIDER_LISTING: Final[contextvars.ContextVar[threading.Event | None]] = contextvars.ContextVar(
|
||||
"uncached_provider_listing", default=None
|
||||
)
|
||||
_Answer: Final = TypeVar("_Answer")
|
||||
|
||||
|
||||
def answer_from_cached_provider_listings(compute: Callable[[], _Answer]) -> tuple[_Answer, bool]:
|
||||
uncached_listing: Final = threading.Event()
|
||||
answer: Final = contextvars.copy_context().run(_answer_marking_uncached_listings, uncached_listing, compute)
|
||||
return answer, uncached_listing.is_set()
|
||||
|
||||
|
||||
def _answer_marking_uncached_listings(uncached_listing: threading.Event, compute: Callable[[], _Answer]) -> _Answer:
|
||||
_UNCACHED_PROVIDER_LISTING.set(uncached_listing)
|
||||
return compute()
|
||||
|
||||
|
||||
def _infer_valid_provider_from_env_vars(
|
||||
custom_llm_provider: str | None = None,
|
||||
|
|
@ -8504,21 +8528,40 @@ def _get_valid_models_from_provider_api(
|
|||
|
||||
if cached_result is not None:
|
||||
return cached_result
|
||||
uncached_listing: Final = _UNCACHED_PROVIDER_LISTING.get()
|
||||
if uncached_listing is not None:
|
||||
uncached_listing.set()
|
||||
return []
|
||||
with model_cache.single_flight_lock(custom_llm_provider, litellm_params):
|
||||
return _list_provider_models_unless_cached(
|
||||
provider_config, custom_llm_provider, litellm_params, model_cache
|
||||
)
|
||||
except Exception as e:
|
||||
verbose_logger.warning("Error getting valid models: %s", e)
|
||||
return []
|
||||
|
||||
|
||||
def _list_provider_models_unless_cached(
|
||||
provider_config: BaseLLMModelInfo,
|
||||
custom_llm_provider: str,
|
||||
litellm_params: LiteLLM_Params | None,
|
||||
model_cache: AvailableModelsCache,
|
||||
) -> list[str]:
|
||||
cached_while_waiting: Final = model_cache.get_cached_model_info(custom_llm_provider, litellm_params)
|
||||
if cached_while_waiting is not None:
|
||||
return cached_while_waiting
|
||||
try:
|
||||
models: Final = provider_config.discover_models(
|
||||
litellm_params=_provider_listing_params(custom_llm_provider, litellm_params)
|
||||
)
|
||||
|
||||
model_cache.set_cached_model_info(custom_llm_provider, litellm_params, models)
|
||||
return models
|
||||
except httpx.TimeoutException as e:
|
||||
verbose_logger.warning("Timed out getting valid models: %s", e)
|
||||
verbose_logger.warning("Error getting valid models: %s", e)
|
||||
model_cache.cache_timed_out_listing_unless_listed(
|
||||
custom_llm_provider, litellm_params, ttl=PROVIDER_MODEL_DISCOVERY_TIMEOUT_CACHE_TTL_SECONDS
|
||||
)
|
||||
return model_cache.get_cached_model_info(custom_llm_provider, litellm_params) or []
|
||||
except Exception as e:
|
||||
verbose_logger.warning("Error getting valid models: %s", e)
|
||||
return []
|
||||
model_cache.set_cached_model_info(custom_llm_provider, litellm_params, models)
|
||||
return models
|
||||
|
||||
|
||||
def get_valid_models(
|
||||
|
|
|
|||
|
|
@ -588,6 +588,67 @@ async def test_a_hung_wildcard_listing_leaves_the_event_loop_free_for_other_requ
|
|||
assert "hosted_vllm/qwen2.5" in await listing
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@respx.mock
|
||||
async def test_a_model_list_that_needs_no_listing_answers_while_hung_listings_hold_every_discovery_thread(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
import asyncio
|
||||
|
||||
import litellm
|
||||
from litellm import Router
|
||||
from litellm.constants import PROVIDER_MODEL_DISCOVERY_MAX_THREADS
|
||||
from litellm.proxy._types import UserAPIKeyAuth
|
||||
from litellm.proxy.utils import get_available_models_for_user
|
||||
|
||||
monkeypatch.setattr(litellm, "check_provider_endpoint", True)
|
||||
listing_started: Final = threading.Event()
|
||||
hung_server_released: Final = threading.Event()
|
||||
|
||||
def hung_listing(request: httpx.Request) -> httpx.Response:
|
||||
listing_started.set()
|
||||
hung_server_released.wait(timeout=10)
|
||||
return httpx.Response(503)
|
||||
|
||||
respx.get("http://vllm-holding-every-thread.test:8000/v1/models").mock(side_effect=hung_listing)
|
||||
router: Final = Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "stuck/*",
|
||||
"litellm_params": {
|
||||
"model": "hosted_vllm/*",
|
||||
"api_base": "http://vllm-holding-every-thread.test:8000/v1",
|
||||
},
|
||||
},
|
||||
{
|
||||
"model_name": "qwen-open",
|
||||
"litellm_params": {"model": "hosted_vllm/qwen2.5", "api_base": "http://vllm-open.test:8000/v1"},
|
||||
},
|
||||
]
|
||||
)
|
||||
|
||||
def list_models(models: list[str]) -> asyncio.Task[list[str]]:
|
||||
return asyncio.create_task(
|
||||
get_available_models_for_user(
|
||||
user_api_key_dict=UserAPIKeyAuth(api_key="sk-test", models=models),
|
||||
llm_router=router,
|
||||
general_settings={},
|
||||
user_model=None,
|
||||
)
|
||||
)
|
||||
|
||||
try:
|
||||
admin_listings: Final = tuple(list_models([]) for _ in range(PROVIDER_MODEL_DISCOVERY_MAX_THREADS))
|
||||
assert await asyncio.to_thread(listing_started.wait, 3)
|
||||
await asyncio.sleep(0.1)
|
||||
|
||||
assert await asyncio.wait_for(list_models(["qwen-open"]), timeout=2) == ["qwen-open"]
|
||||
finally:
|
||||
hung_server_released.set()
|
||||
for admin_models in await asyncio.gather(*admin_listings):
|
||||
assert "qwen-open" in admin_models
|
||||
|
||||
|
||||
@respx.mock
|
||||
def test_a_hung_wildcard_listing_leaves_the_default_executor_free_for_completion_calls(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
|
|
|
|||
|
|
@ -802,10 +802,11 @@ async def test_a_hung_wildcard_listing_during_a_credential_lookup_leaves_the_eve
|
|||
"model": "hosted_vllm/*",
|
||||
"api_base": "http://vllm-credential-lookup.test:8000/v1",
|
||||
},
|
||||
}
|
||||
},
|
||||
{"model_name": "gpt-4o", "litellm_params": {"model": "openai/gpt-4o", "api_key": "openai-key"}},
|
||||
]
|
||||
),
|
||||
user_api_key_dict=UserAPIKeyAuth(api_key="sk-test"),
|
||||
user_api_key_dict=UserAPIKeyAuth(api_key="sk-test", models=["hosted_vllm/*"]),
|
||||
custom_llm_provider="openai",
|
||||
)
|
||||
)
|
||||
|
|
@ -834,14 +835,20 @@ async def test_a_hung_wildcard_listing_during_a_credential_lookup_leaves_the_eve
|
|||
("gpt-4o", "openai/gpt-4o"),
|
||||
("api_base", "http://vllm-route-match.test:8000/v1"),
|
||||
),
|
||||
(
|
||||
"anthropic",
|
||||
"http://vllm-beside-no-anthropic-deployment.test:8000/v1",
|
||||
("gpt-4o", "openai/gpt-4o"),
|
||||
("api_key", None),
|
||||
),
|
||||
],
|
||||
)
|
||||
async def test_a_credential_lookup_a_configured_model_answers_never_lists_the_wildcard_server(
|
||||
async def test_a_credential_lookup_never_lists_a_wildcard_server_whose_models_cannot_change_the_answer(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
custom_llm_provider: str,
|
||||
vllm_api_base: str,
|
||||
openai_deployment: tuple[str, str],
|
||||
applied_credential: tuple[str, str],
|
||||
applied_credential: tuple[str, str | None],
|
||||
) -> None:
|
||||
import litellm
|
||||
|
||||
|
|
|
|||
|
|
@ -10616,6 +10616,43 @@ def test_a_timed_out_provider_model_listing_is_retried_only_after_its_timeout_tt
|
|||
assert models_route.call_count == 2
|
||||
|
||||
|
||||
@respx.mock
|
||||
def test_concurrent_listings_of_one_server_share_a_single_request() -> None:
|
||||
import time
|
||||
|
||||
from litellm.llms.vllm.common_utils import HostedVLLMModelInfo
|
||||
from litellm.types.router import LiteLLM_Params
|
||||
from litellm.utils import AvailableModelsCache, _get_valid_models_from_provider_api
|
||||
|
||||
listing_started: Final = threading.Event()
|
||||
server_released: Final = threading.Event()
|
||||
|
||||
def slow_listing(request: httpx.Request) -> httpx.Response:
|
||||
listing_started.set()
|
||||
server_released.wait(timeout=5)
|
||||
return httpx.Response(200, json={"data": [{"id": "qwen2.5"}]})
|
||||
|
||||
models_route: Final = respx.get("http://vllm-slow.test:8000/v1/models").mock(side_effect=slow_listing)
|
||||
litellm_params: Final = LiteLLM_Params(model="hosted_vllm/*", api_base="http://vllm-slow.test:8000/v1")
|
||||
model_cache: Final = AvailableModelsCache(clock=Mock(return_value=1000.0))
|
||||
|
||||
def list_models() -> list[str]:
|
||||
return _get_valid_models_from_provider_api(
|
||||
HostedVLLMModelInfo(), "hosted_vllm", litellm_params, model_cache=model_cache
|
||||
)
|
||||
|
||||
with ThreadPoolExecutor(max_workers=4) as callers:
|
||||
listings: Final = tuple(callers.submit(list_models) for _ in range(4))
|
||||
try:
|
||||
assert listing_started.wait(timeout=3)
|
||||
time.sleep(0.3)
|
||||
finally:
|
||||
server_released.set()
|
||||
assert [listing.result(timeout=5) for listing in listings] == [["qwen2.5"]] * 4
|
||||
|
||||
assert models_route.call_count == 1
|
||||
|
||||
|
||||
@respx.mock
|
||||
def test_a_listing_that_times_out_keeps_the_models_a_concurrent_listing_cached() -> None:
|
||||
from litellm.llms.vllm.common_utils import HostedVLLMModelInfo
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue