mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
fix(proxy): run model discovery on its own thread pool and keep credential lookups off the event loop
This commit is contained in:
parent
42303949e1
commit
22e417363a
11 changed files with 190 additions and 47 deletions
|
|
@ -612,6 +612,7 @@ NON_LLM_CONNECTION_TIMEOUT: Final = get_env_int(
|
|||
"NON_LLM_CONNECTION_TIMEOUT", 15
|
||||
) # timeout for adjacent services (e.g. jwt auth)
|
||||
PROVIDER_MODEL_DISCOVERY_TIMEOUT_CACHE_TTL_SECONDS: Final = 60
|
||||
PROVIDER_MODEL_DISCOVERY_MAX_THREADS: Final = 4
|
||||
MAX_EXCEPTION_MESSAGE_LENGTH: Final = get_env_int("MAX_EXCEPTION_MESSAGE_LENGTH", 2000)
|
||||
MAX_STRING_LENGTH_PROMPT_IN_DB: Final = get_env_int("MAX_STRING_LENGTH_PROMPT_IN_DB", 2048)
|
||||
MAX_STRING_LENGTH_COMPACTION_SUMMARY_IN_DB: Final = 64 * 1024
|
||||
|
|
|
|||
|
|
@ -1,14 +1,18 @@
|
|||
# What is this?
|
||||
## Common checks for /v1/models and `/model/info`
|
||||
import asyncio
|
||||
import contextvars
|
||||
import copy
|
||||
import functools
|
||||
import itertools
|
||||
import re
|
||||
from collections.abc import Callable, Mapping, Sequence
|
||||
from typing import Final
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from typing import Final, ParamSpec, TypeVar
|
||||
|
||||
import litellm
|
||||
from litellm._logging import verbose_proxy_logger
|
||||
from litellm.constants import PROVIDER_MODEL_DISCOVERY_MAX_THREADS
|
||||
from litellm.litellm_core_utils.credential_accessor import CredentialAccessor
|
||||
from litellm.llms.azure.ptu_capacity import is_str_keyed_mapping
|
||||
from litellm.proxy._types import SpecialModelNames, UserAPIKeyAuth
|
||||
|
|
@ -22,6 +26,26 @@ from litellm.utils import ProviderConfigManager, get_valid_models
|
|||
|
||||
LLM_PROVIDER_PREFIXES: Final = frozenset(llm_provider.value for llm_provider in LlmProviders)
|
||||
|
||||
_P: Final = ParamSpec("_P")
|
||||
_R: Final = TypeVar("_R")
|
||||
|
||||
_MODEL_DISCOVERY_EXECUTOR: Final = ThreadPoolExecutor(
|
||||
max_workers=PROVIDER_MODEL_DISCOVERY_MAX_THREADS, thread_name_prefix="litellm-model-discovery"
|
||||
)
|
||||
|
||||
|
||||
async def run_model_discovery(
|
||||
discover: Callable[_P, _R],
|
||||
/,
|
||||
*args: _P.args,
|
||||
**kwargs: _P.kwargs, # kwargs-ok: forwards the discovery function's own arguments
|
||||
) -> _R:
|
||||
context: Final = contextvars.copy_context()
|
||||
return await asyncio.get_running_loop().run_in_executor(
|
||||
_MODEL_DISCOVERY_EXECUTOR, lambda: context.run(discover, *args, **kwargs)
|
||||
)
|
||||
|
||||
|
||||
_CREDENTIAL_LITELLM_PARAM_FIELDS = set(CredentialLiteLLMParams.model_fields)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -489,7 +489,7 @@ async def create_batch(
|
|||
verbose_proxy_logger.debug("Created batch using model: %s", model_param)
|
||||
else:
|
||||
# SCENARIO 3: Fallback to custom_llm_provider (uses env variables)
|
||||
apply_team_provider_credentials(
|
||||
await apply_team_provider_credentials(
|
||||
data=cast(dict, _create_batch_data), # cast-ok: TypedDict is a dict at runtime
|
||||
llm_router=llm_router,
|
||||
user_api_key_dict=user_api_key_dict,
|
||||
|
|
@ -772,7 +772,7 @@ async def retrieve_batch(
|
|||
or get_custom_llm_provider_from_request_query(request=request)
|
||||
)
|
||||
custom_llm_provider: Final = requested_provider or "openai"
|
||||
apply_team_provider_credentials(
|
||||
await apply_team_provider_credentials(
|
||||
data=data,
|
||||
llm_router=llm_router,
|
||||
user_api_key_dict=user_api_key_dict,
|
||||
|
|
@ -986,7 +986,7 @@ async def list_batches(
|
|||
or get_custom_llm_provider_from_request_query(request=request)
|
||||
or "openai"
|
||||
)
|
||||
apply_team_provider_credentials(
|
||||
await apply_team_provider_credentials(
|
||||
data=data,
|
||||
llm_router=llm_router,
|
||||
user_api_key_dict=user_api_key_dict,
|
||||
|
|
@ -1209,7 +1209,7 @@ async def cancel_batch(
|
|||
# Extract batch_id from data to avoid "multiple values for keyword argument" error
|
||||
# data was cast from CancelBatchRequest which already contains batch_id
|
||||
data.pop("batch_id", None)
|
||||
apply_team_provider_credentials(
|
||||
await apply_team_provider_credentials(
|
||||
data=data,
|
||||
llm_router=llm_router,
|
||||
user_api_key_dict=user_api_key_dict,
|
||||
|
|
|
|||
|
|
@ -527,7 +527,7 @@ class _PROXY_BatchRateLimiter(CustomLogger):
|
|||
return True
|
||||
return False
|
||||
|
||||
def _resolve_batch_input_file_fetch_params(
|
||||
async def _resolve_batch_input_file_fetch_params(
|
||||
self,
|
||||
file_id: str,
|
||||
custom_llm_provider: str,
|
||||
|
|
@ -541,6 +541,7 @@ class _PROXY_BatchRateLimiter(CustomLogger):
|
|||
without decoding them, ``afile_content`` is called with the encoded ID
|
||||
and the upstream provider returns 404.
|
||||
"""
|
||||
from litellm.proxy.auth.model_checks import run_model_discovery
|
||||
from litellm.proxy.openai_files_endpoints.common_utils import (
|
||||
decode_model_from_file_id,
|
||||
explicit_request_credentials,
|
||||
|
|
@ -591,7 +592,8 @@ class _PROXY_BatchRateLimiter(CustomLogger):
|
|||
return file_id, fetch_kwargs
|
||||
|
||||
team_credentials: Final = (
|
||||
get_team_provider_credentials(
|
||||
await run_model_discovery(
|
||||
get_team_provider_credentials,
|
||||
llm_router=llm_router,
|
||||
user_api_key_dict=user_api_key_dict,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
|
|
@ -948,7 +950,7 @@ class _PROXY_BatchRateLimiter(CustomLogger):
|
|||
user_api_key_dict=user_api_key_dict,
|
||||
)
|
||||
else:
|
||||
provider_file_id, fetch_kwargs = self._resolve_batch_input_file_fetch_params(
|
||||
provider_file_id, fetch_kwargs = await self._resolve_batch_input_file_fetch_params(
|
||||
file_id=file_id,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
data=data or {},
|
||||
|
|
|
|||
|
|
@ -530,7 +530,7 @@ def get_team_provider_credentials(
|
|||
custom_llm_provider: str,
|
||||
*,
|
||||
include_accessible_deployments: bool = True,
|
||||
) -> dict | None:
|
||||
) -> dict[str, object] | None:
|
||||
"""
|
||||
Resolve upstream credentials for a provider-scoped file operation
|
||||
(e.g. GET /v1/files), which doesn't pin a model.
|
||||
|
|
@ -704,7 +704,7 @@ def explicit_request_credentials(data: Mapping[str, object]) -> Mapping[str, str
|
|||
return MappingProxyType({"api_key": api_key, "api_base": api_base})
|
||||
|
||||
|
||||
def apply_team_provider_credentials(
|
||||
async def apply_team_provider_credentials(
|
||||
data: dict, # mutable-ok: credentials are merged into the request payload in place, same contract as prepare_data_with_credentials
|
||||
llm_router: Optional["Router"],
|
||||
user_api_key_dict: "UserAPIKeyAuth",
|
||||
|
|
@ -719,7 +719,10 @@ def apply_team_provider_credentials(
|
|||
``forward_llm_provider_auth_headers``) replaces the deployment's, and the
|
||||
request's ``api_base`` replaces the deployment's only alongside that key.
|
||||
"""
|
||||
credentials: Final = get_team_provider_credentials(
|
||||
from litellm.proxy.auth.model_checks import run_model_discovery
|
||||
|
||||
credentials: Final = await run_model_discovery(
|
||||
get_team_provider_credentials,
|
||||
llm_router=llm_router,
|
||||
user_api_key_dict=user_api_key_dict,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
|
|
|
|||
|
|
@ -524,7 +524,7 @@ async def route_create_file(
|
|||
_create_file_request=_create_file_request,
|
||||
)
|
||||
else:
|
||||
apply_team_provider_credentials(
|
||||
await apply_team_provider_credentials(
|
||||
data=cast(dict, _create_file_request), # cast-ok: TypedDict is a plain dict at runtime; merged in place
|
||||
llm_router=llm_router,
|
||||
user_api_key_dict=user_api_key_dict,
|
||||
|
|
@ -1131,7 +1131,7 @@ async def get_file_content(
|
|||
)
|
||||
|
||||
if not should_route:
|
||||
apply_team_provider_credentials(
|
||||
await apply_team_provider_credentials(
|
||||
data=data,
|
||||
llm_router=llm_router,
|
||||
user_api_key_dict=user_api_key_dict,
|
||||
|
|
@ -1397,7 +1397,7 @@ async def get_file(
|
|||
# Remove file_id from data to avoid "multiple values for keyword argument" error
|
||||
# data was initialized with {"file_id": file_id}
|
||||
data.pop("file_id", None)
|
||||
apply_team_provider_credentials(
|
||||
await apply_team_provider_credentials(
|
||||
data=data,
|
||||
llm_router=llm_router,
|
||||
user_api_key_dict=user_api_key_dict,
|
||||
|
|
@ -1616,7 +1616,7 @@ async def delete_file(
|
|||
)
|
||||
else:
|
||||
data.pop("file_id", None)
|
||||
apply_team_provider_credentials(
|
||||
await apply_team_provider_credentials(
|
||||
data=data,
|
||||
llm_router=llm_router,
|
||||
user_api_key_dict=user_api_key_dict,
|
||||
|
|
@ -2018,7 +2018,7 @@ async def list_files(
|
|||
)
|
||||
else:
|
||||
resolved_custom_llm_provider: Final = custom_llm_provider or "openai"
|
||||
apply_team_provider_credentials(
|
||||
await apply_team_provider_credentials(
|
||||
data=data,
|
||||
llm_router=llm_router,
|
||||
user_api_key_dict=user_api_key_dict,
|
||||
|
|
|
|||
|
|
@ -440,6 +440,7 @@ from litellm.proxy.auth.model_checks import (
|
|||
get_key_models,
|
||||
get_mcp_server_ids,
|
||||
get_team_models,
|
||||
run_model_discovery,
|
||||
)
|
||||
from litellm.proxy.auth.password_policy import validate_password_not_breached, validate_password_policy
|
||||
from litellm.proxy.auth.user_api_key_auth import ( # noqa: F401, RUF100 # legacy module exports
|
||||
|
|
@ -12418,7 +12419,7 @@ async def model_list(
|
|||
# Get complete model list including wildcard routes if requested
|
||||
from litellm.proxy.auth.model_checks import get_complete_model_list
|
||||
|
||||
all_models = await asyncio.to_thread(
|
||||
all_models = await run_model_discovery(
|
||||
get_complete_model_list,
|
||||
key_models=[],
|
||||
team_models=[],
|
||||
|
|
@ -17854,7 +17855,7 @@ async def model_info_v1(
|
|||
alias_models: Final = copy.deepcopy(llm_router.get_model_list_from_model_alias())
|
||||
all_models.extend(alias_models)
|
||||
|
||||
all_models = await asyncio.to_thread(expand_wildcard_deployments_for_model_info, all_models, llm_router)
|
||||
all_models = await run_model_discovery(expand_wildcard_deployments_for_model_info, all_models, llm_router)
|
||||
|
||||
allowed_model_names: Final = await _get_v1_model_info_allowed_model_names_with_access_groups(
|
||||
user_api_key_dict=user_api_key_dict,
|
||||
|
|
@ -18189,7 +18190,7 @@ async def model_group_info(
|
|||
LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY,
|
||||
)
|
||||
all_models_str: Final = (
|
||||
await asyncio.to_thread(
|
||||
await run_model_discovery(
|
||||
get_complete_model_list,
|
||||
key_models=(),
|
||||
team_models=(),
|
||||
|
|
|
|||
|
|
@ -9499,7 +9499,7 @@ async def _agent_access_group_visible_models(
|
|||
) -> frozenset[str] | None:
|
||||
"""Models an agent key may still list once its attached access groups cap it, ``None`` when
|
||||
nothing caps it, so ``/v1/models`` never advertises a model the same key would be denied on."""
|
||||
from litellm.proxy.auth.model_checks import get_complete_model_list, get_team_models
|
||||
from litellm.proxy.auth.model_checks import get_complete_model_list, get_team_models, run_model_discovery
|
||||
|
||||
if not user_api_key_dict.agent_id:
|
||||
return None
|
||||
|
|
@ -9519,7 +9519,7 @@ async def _agent_access_group_visible_models(
|
|||
if not granted:
|
||||
return frozenset()
|
||||
return frozenset(
|
||||
await asyncio.to_thread(
|
||||
await run_model_discovery(
|
||||
get_complete_model_list,
|
||||
key_models=granted,
|
||||
team_models=(),
|
||||
|
|
@ -9575,6 +9575,7 @@ async def get_available_models_for_user(
|
|||
get_complete_model_list,
|
||||
get_key_models,
|
||||
get_team_models,
|
||||
run_model_discovery,
|
||||
)
|
||||
|
||||
# Get proxy model list and access groups
|
||||
|
|
@ -9655,7 +9656,7 @@ async def get_available_models_for_user(
|
|||
)
|
||||
|
||||
# Get complete model list
|
||||
complete_models: Final = await asyncio.to_thread(
|
||||
complete_models: Final = await run_model_discovery(
|
||||
get_complete_model_list,
|
||||
key_models=granted_key_models,
|
||||
team_models=granted_team_models,
|
||||
|
|
|
|||
|
|
@ -588,6 +588,59 @@ async def test_a_hung_wildcard_listing_leaves_the_event_loop_free_for_other_requ
|
|||
assert "hosted_vllm/qwen2.5" in await listing
|
||||
|
||||
|
||||
@respx.mock
|
||||
def test_a_hung_wildcard_listing_leaves_the_default_executor_free_for_completion_calls(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
import asyncio
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
|
||||
import litellm
|
||||
from litellm import Router
|
||||
from litellm.proxy._types import UserAPIKeyAuth
|
||||
from litellm.proxy.utils import get_available_models_for_user
|
||||
|
||||
monkeypatch.setattr(litellm, "check_provider_endpoint", True)
|
||||
listing_started: Final = threading.Event()
|
||||
completion_served: Final = threading.Event()
|
||||
|
||||
def listing_that_answers_once_a_completion_is_served(request: httpx.Request) -> httpx.Response:
|
||||
listing_started.set()
|
||||
if completion_served.wait(timeout=3):
|
||||
return httpx.Response(200, json={"data": [{"id": "qwen2.5"}]})
|
||||
return httpx.Response(503)
|
||||
|
||||
respx.get("http://vllm-default-executor.test:8000/v1/models").mock(
|
||||
side_effect=listing_that_answers_once_a_completion_is_served
|
||||
)
|
||||
router: Final = Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "hosted_vllm/*",
|
||||
"litellm_params": {"model": "hosted_vllm/*", "api_base": "http://vllm-default-executor.test:8000/v1"},
|
||||
}
|
||||
]
|
||||
)
|
||||
|
||||
async def list_models_while_a_completion_runs_on_the_default_executor() -> list[str]:
|
||||
loop: Final = asyncio.get_running_loop()
|
||||
loop.set_default_executor(ThreadPoolExecutor(max_workers=1))
|
||||
listing: Final = asyncio.create_task(
|
||||
get_available_models_for_user(
|
||||
user_api_key_dict=UserAPIKeyAuth(api_key="sk-test", models=[]),
|
||||
llm_router=router,
|
||||
general_settings={},
|
||||
user_model=None,
|
||||
)
|
||||
)
|
||||
with ThreadPoolExecutor(max_workers=1) as waiter:
|
||||
assert await loop.run_in_executor(waiter, listing_started.wait, 3)
|
||||
await asyncio.wait_for(loop.run_in_executor(None, completion_served.set), timeout=1)
|
||||
return await listing
|
||||
|
||||
assert "hosted_vllm/qwen2.5" in asyncio.run(list_models_while_a_completion_runs_on_the_default_executor())
|
||||
|
||||
|
||||
@respx.mock
|
||||
def test_hosted_vllm_wildcard_listing_sends_the_deployment_extra_headers(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
import litellm
|
||||
|
|
|
|||
|
|
@ -1274,7 +1274,8 @@ def test_should_not_skip_and_reuses_descriptors_when_limits_present():
|
|||
assert returned is descriptors
|
||||
|
||||
|
||||
def test_resolve_fetch_params_uses_request_model_credentials():
|
||||
@pytest.mark.asyncio
|
||||
async def test_resolve_fetch_params_uses_request_model_credentials():
|
||||
rate_limiter = _make_rate_limiter()
|
||||
with (
|
||||
patch("litellm.proxy.proxy_server.llm_router", MagicMock()),
|
||||
|
|
@ -1288,7 +1289,7 @@ def test_resolve_fetch_params_uses_request_model_credentials():
|
|||
),
|
||||
):
|
||||
provider_file_id, fetch_kwargs = (
|
||||
rate_limiter._resolve_batch_input_file_fetch_params(
|
||||
await rate_limiter._resolve_batch_input_file_fetch_params(
|
||||
file_id="file-plain-openai",
|
||||
custom_llm_provider="openai",
|
||||
data={"model": "my-vllm-batch"},
|
||||
|
|
@ -1300,7 +1301,8 @@ def test_resolve_fetch_params_uses_request_model_credentials():
|
|||
assert fetch_kwargs["api_base"] == "http://vllm:8000/v1"
|
||||
|
||||
|
||||
def test_resolve_fetch_params_fails_open_on_credential_lookup_error():
|
||||
@pytest.mark.asyncio
|
||||
async def test_resolve_fetch_params_fails_open_on_credential_lookup_error():
|
||||
rate_limiter = _make_rate_limiter()
|
||||
with (
|
||||
patch("litellm.proxy.proxy_server.llm_router", MagicMock()),
|
||||
|
|
@ -1310,7 +1312,7 @@ def test_resolve_fetch_params_fails_open_on_credential_lookup_error():
|
|||
),
|
||||
):
|
||||
provider_file_id, fetch_kwargs = (
|
||||
rate_limiter._resolve_batch_input_file_fetch_params(
|
||||
await rate_limiter._resolve_batch_input_file_fetch_params(
|
||||
file_id="file-plain-openai",
|
||||
custom_llm_provider="openai",
|
||||
data={"model": "my-vllm-batch"},
|
||||
|
|
@ -1320,7 +1322,8 @@ def test_resolve_fetch_params_fails_open_on_credential_lookup_error():
|
|||
assert fetch_kwargs == {"custom_llm_provider": "openai"}
|
||||
|
||||
|
||||
def test_resolve_fetch_params_model_embedded_fails_open_on_credential_error():
|
||||
@pytest.mark.asyncio
|
||||
async def test_resolve_fetch_params_model_embedded_fails_open_on_credential_error():
|
||||
import base64
|
||||
|
||||
rate_limiter = _make_rate_limiter()
|
||||
|
|
@ -1342,7 +1345,7 @@ def test_resolve_fetch_params_model_embedded_fails_open_on_credential_error():
|
|||
),
|
||||
):
|
||||
provider_file_id, fetch_kwargs = (
|
||||
rate_limiter._resolve_batch_input_file_fetch_params(
|
||||
await rate_limiter._resolve_batch_input_file_fetch_params(
|
||||
file_id=encoded_file_id,
|
||||
custom_llm_provider="openai",
|
||||
data={},
|
||||
|
|
|
|||
|
|
@ -1,9 +1,13 @@
|
|||
import asyncio
|
||||
import threading
|
||||
from datetime import datetime, timezone
|
||||
from types import MappingProxyType
|
||||
from typing import Final
|
||||
from unittest.mock import AsyncMock, MagicMock
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
import respx
|
||||
from fastapi import HTTPException
|
||||
from litellm_enterprise.proxy.hooks.managed_files import _PROXY_LiteLLMManagedFiles
|
||||
|
||||
|
|
@ -764,26 +768,77 @@ def test_a_request_key_that_cannot_be_an_http_header_is_a_bad_request(api_key: s
|
|||
assert api_key not in str(refused.value.detail)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@respx.mock
|
||||
async def test_a_hung_wildcard_listing_during_a_credential_lookup_leaves_the_event_loop_free(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
import litellm
|
||||
|
||||
monkeypatch.setattr(litellm, "check_provider_endpoint", True)
|
||||
listing_started: Final = threading.Event()
|
||||
other_request_served: Final = threading.Event()
|
||||
listing_answered: Final = threading.Event()
|
||||
|
||||
def listing_that_answers_once_another_request_is_served(request: httpx.Request) -> httpx.Response:
|
||||
listing_started.set()
|
||||
if other_request_served.wait(timeout=3):
|
||||
listing_answered.set()
|
||||
return httpx.Response(200, json={"data": [{"id": "qwen2.5"}]})
|
||||
return httpx.Response(503)
|
||||
|
||||
respx.get("http://vllm-credential-lookup.test:8000/v1/models").mock(
|
||||
side_effect=listing_that_answers_once_another_request_is_served
|
||||
)
|
||||
data: Final[dict[str, object]] = {}
|
||||
lookup: Final = asyncio.create_task(
|
||||
apply_team_provider_credentials(
|
||||
data=data,
|
||||
llm_router=Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "hosted_vllm/*",
|
||||
"litellm_params": {
|
||||
"model": "hosted_vllm/*",
|
||||
"api_base": "http://vllm-credential-lookup.test:8000/v1",
|
||||
},
|
||||
}
|
||||
]
|
||||
),
|
||||
user_api_key_dict=UserAPIKeyAuth(api_key="sk-test"),
|
||||
custom_llm_provider="hosted_vllm",
|
||||
)
|
||||
)
|
||||
assert await asyncio.to_thread(listing_started.wait, 3)
|
||||
other_request_served.set()
|
||||
await lookup
|
||||
|
||||
assert listing_answered.is_set()
|
||||
assert data.get("api_base") == "http://vllm-credential-lookup.test:8000/v1"
|
||||
|
||||
|
||||
def _apply_team_a_openai_credentials(data: dict[str, object]) -> None:
|
||||
apply_team_provider_credentials(
|
||||
data=data,
|
||||
llm_router=Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "openai/*",
|
||||
"litellm_params": {
|
||||
"model": "openai/*",
|
||||
"api_key": "team-openai-key",
|
||||
"api_base": "https://team.example/v1",
|
||||
},
|
||||
"model_info": {"id": "team-a-openai", "team_id": "team-a"},
|
||||
}
|
||||
]
|
||||
),
|
||||
user_api_key_dict=UserAPIKeyAuth(
|
||||
api_key="test-key", user_id="test-user", team_id="team-a", team_models=["openai/*"], models=["gpt-4o"]
|
||||
),
|
||||
custom_llm_provider="openai",
|
||||
asyncio.run(
|
||||
apply_team_provider_credentials(
|
||||
data=data,
|
||||
llm_router=Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "openai/*",
|
||||
"litellm_params": {
|
||||
"model": "openai/*",
|
||||
"api_key": "team-openai-key",
|
||||
"api_base": "https://team.example/v1",
|
||||
},
|
||||
"model_info": {"id": "team-a-openai", "team_id": "team-a"},
|
||||
}
|
||||
]
|
||||
),
|
||||
user_api_key_dict=UserAPIKeyAuth(
|
||||
api_key="test-key", user_id="test-user", team_id="team-a", team_models=["openai/*"], models=["gpt-4o"]
|
||||
),
|
||||
custom_llm_provider="openai",
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue