support default deployments

This commit is contained in:
Ishaan Jaff 2024-09-09 14:23:17 -07:00
parent f1d0045ae6
commit c4052ee7d7

View file

@ -1,5 +1,9 @@
"""
Use this to route requests between free and paid tiers
Use this to route requests between Teams
- If tags in request is a subset of tags in deployment, return deployment
- if deployments are set with default tags, return all default deployment
- If no default_deployments are set, return all deployments
"""
from typing import TYPE_CHECKING, Any, Dict, List, Literal, Optional, TypedDict, Union
@ -68,9 +72,26 @@ async def get_deployments_for_tag(
request_tags,
)
new_healthy_deployments.append(deployment)
elif "default" in deployment_tags:
verbose_logger.debug(
"adding default deployment with tags: %s, request tags: %s",
deployment_tags,
request_tags,
)
new_healthy_deployments.append(deployment)
return new_healthy_deployments
# for Untagged requests use default deployments if set
_default_deployments_with_tags = []
for deployment in healthy_deployments:
if "default" in deployment.get("litellm_params", {}).get("tags", []):
_default_deployments_with_tags.append(deployment)
if len(_default_deployments_with_tags) > 0:
return _default_deployments_with_tags
# if no default deployment is found, return healthy_deployments
verbose_logger.debug(
"no tier found in metadata, returning healthy_deployments: %s",
healthy_deployments,