scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-06 08:16:43 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 78

Merge branch 'LangfuseUsageDetails' of github.com:fabriciojoc/litellm into LangfuseUsageDetails

Browse source
This commit is contained in:
Fabricio Ceschin 2025-05-21 21:24:07 -04:00
commit f4f89cbae1
26 changed files with 2165 additions and 1249 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
docs/my-website/docs/providers/voyage.md
View file

@ -25,6 +25,8 @@ All models listed here https://docs.voyageai.com/embeddings/#models-and-specific
| Model Name | Function Call |
|-------------------------|------------------------------------------------------------|
| voyage-3.5 | `embedding(model="voyage/voyage-3.5", input)` |
| voyage-3.5-lite | `embedding(model="voyage/voyage-3.5-lite", input)` |
| voyage-3-large | `embedding(model="voyage/voyage-3-large", input)` |
| voyage-3 | `embedding(model="voyage/voyage-3", input)` |
| voyage-3-lite | `embedding(model="voyage/voyage-3-lite", input)` |
@ -35,8 +37,8 @@ All models listed here https://docs.voyageai.com/embeddings/#models-and-specific
| voyage-multilingual-2 | `embedding(model="voyage/voyage-multilingual-2 ", input)` |
| voyage-large-2-instruct | `embedding(model="voyage/voyage-large-2-instruct", input)` |
| voyage-large-2 | `embedding(model="voyage/voyage-large-2", input)` |
| voyage-2 | `embedding(model="voyage/voyage-2", input)` |
| voyage-2 | `embedding(model="voyage/voyage-2", input)` |
| voyage-lite-02-instruct | `embedding(model="voyage/voyage-lite-02-instruct", input)` |
| voyage-01 | `embedding(model="voyage/voyage-01", input)` |
| voyage-lite-01 | `embedding(model="voyage/voyage-lite-01", input)` |
| voyage-01 | `embedding(model="voyage/voyage-01", input)` |
| voyage-lite-01 | `embedding(model="voyage/voyage-lite-01", input)` |
| voyage-lite-01-instruct | `embedding(model="voyage/voyage-lite-01-instruct", input)` |

106
litellm/__init__.py
View file

@ -132,22 +132,22 @@ prometheus_initialize_budget_metrics: Optional[bool] = False
require_auth_for_metrics_endpoint: Optional[bool] = False
argilla_batch_size: Optional[int] = None
datadog_use_v1: Optional[bool] = False # if you want to use v1 datadog logged payload
gcs_pub_sub_use_v1: Optional[bool] = (
False # if you want to use v1 gcs pubsub logged payload
)
generic_api_use_v1: Optional[bool] = (
False # if you want to use v1 generic api logged payload
)
gcs_pub_sub_use_v1: Optional[
bool
] = False # if you want to use v1 gcs pubsub logged payload
generic_api_use_v1: Optional[
bool
] = False # if you want to use v1 generic api logged payload
argilla_transformation_object: Optional[Dict[str, Any]] = None
_async_input_callback: List[Union[str, Callable, CustomLogger]] = (
[]
) # internal variable - async custom callbacks are routed here.
_async_success_callback: List[Union[str, Callable, CustomLogger]] = (
[]
) # internal variable - async custom callbacks are routed here.
_async_failure_callback: List[Union[str, Callable, CustomLogger]] = (
[]
) # internal variable - async custom callbacks are routed here.
_async_input_callback: List[
Union[str, Callable, CustomLogger]
] = [] # internal variable - async custom callbacks are routed here.
_async_success_callback: List[
Union[str, Callable, CustomLogger]
] = [] # internal variable - async custom callbacks are routed here.
_async_failure_callback: List[
Union[str, Callable, CustomLogger]
] = [] # internal variable - async custom callbacks are routed here.
pre_call_rules: List[Callable] = []
post_call_rules: List[Callable] = []
turn_off_message_logging: Optional[bool] = False
@ -155,18 +155,18 @@ log_raw_request_response: bool = False
redact_messages_in_exceptions: Optional[bool] = False
redact_user_api_key_info: Optional[bool] = False
filter_invalid_headers: Optional[bool] = False
add_user_information_to_llm_headers: Optional[bool] = (
None # adds user_id, team_id, token hash (params from StandardLoggingMetadata) to request headers
)
add_user_information_to_llm_headers: Optional[
bool
] = None # adds user_id, team_id, token hash (params from StandardLoggingMetadata) to request headers
store_audit_logs = False # Enterprise feature, allow users to see audit logs
### end of callbacks #############
email: Optional[str] = (
None # Not used anymore, will be removed in next MAJOR release - https://github.com/BerriAI/litellm/discussions/648
)
token: Optional[str] = (
None # Not used anymore, will be removed in next MAJOR release - https://github.com/BerriAI/litellm/discussions/648
)
email: Optional[
str
] = None # Not used anymore, will be removed in next MAJOR release - https://github.com/BerriAI/litellm/discussions/648
token: Optional[
str
] = None # Not used anymore, will be removed in next MAJOR release - https://github.com/BerriAI/litellm/discussions/648
telemetry = True
max_tokens: int = DEFAULT_MAX_TOKENS # OpenAI Defaults
drop_params = bool(os.getenv("LITELLM_DROP_PARAMS", False))
@ -247,24 +247,20 @@ enable_loadbalancing_on_batch_endpoints: Optional[bool] = None
enable_caching_on_provider_specific_optional_params: bool = (
False # feature-flag for caching on optional params - e.g. 'top_k'
)
caching: bool = (
False # Not used anymore, will be removed in next MAJOR release - https://github.com/BerriAI/litellm/discussions/648
)
caching_with_models: bool = (
False # # Not used anymore, will be removed in next MAJOR release - https://github.com/BerriAI/litellm/discussions/648
)
cache: Optional[Cache] = (
None # cache object <- use this - https://docs.litellm.ai/docs/caching
)
caching: bool = False # Not used anymore, will be removed in next MAJOR release - https://github.com/BerriAI/litellm/discussions/648
caching_with_models: bool = False # # Not used anymore, will be removed in next MAJOR release - https://github.com/BerriAI/litellm/discussions/648
cache: Optional[
Cache
] = None # cache object <- use this - https://docs.litellm.ai/docs/caching
default_in_memory_ttl: Optional[float] = None
default_redis_ttl: Optional[float] = None
default_redis_batch_cache_expiry: Optional[float] = None
model_alias_map: Dict[str, str] = {}
model_group_alias_map: Dict[str, str] = {}
max_budget: float = 0.0 # set the max budget across all providers
budget_duration: Optional[str] = (
None # proxy only - resets budget after fixed duration. You can set duration as seconds ("30s"), minutes ("30m"), hours ("30h"), days ("30d").
)
budget_duration: Optional[
str
] = None # proxy only - resets budget after fixed duration. You can set duration as seconds ("30s"), minutes ("30m"), hours ("30h"), days ("30d").
default_soft_budget: float = (
DEFAULT_SOFT_BUDGET # by default all litellm proxy keys have a soft budget of 50.0
)
@ -273,15 +269,11 @@ forward_traceparent_to_llm_provider: bool = False
_current_cost = 0.0 # private variable, used if max budget is set
error_logs: Dict = {}
add_function_to_prompt: bool = (
False # if function calling not supported by api, append function call details to system prompt
)
add_function_to_prompt: bool = False # if function calling not supported by api, append function call details to system prompt
client_session: Optional[httpx.Client] = None
aclient_session: Optional[httpx.AsyncClient] = None
model_fallbacks: Optional[List] = None # Deprecated for 'litellm.fallbacks'
model_cost_map_url: str = (
"https://raw.githubusercontent.com/BerriAI/litellm/main/model_prices_and_context_window.json"
)
model_cost_map_url: str = "https://raw.githubusercontent.com/BerriAI/litellm/main/model_prices_and_context_window.json"
suppress_debug_info = False
dynamodb_table_name: Optional[str] = None
s3_callback_params: Optional[Dict] = None
@ -304,9 +296,7 @@ disable_end_user_cost_tracking_prometheus_only: Optional[bool] = None
custom_prometheus_metadata_labels: List[str] = []
#### REQUEST PRIORITIZATION ####
priority_reservation: Optional[Dict[str, float]] = None
force_ipv4: bool = (
False # when True, litellm will force ipv4 for all LLM requests. Some users have seen httpx ConnectionError when using ipv6.
)
force_ipv4: bool = False # when True, litellm will force ipv4 for all LLM requests. Some users have seen httpx ConnectionError when using ipv6.
module_level_aclient = AsyncHTTPHandler(
timeout=request_timeout, client_alias="module level aclient"
)
@ -320,13 +310,13 @@ fallbacks: Optional[List] = None
context_window_fallbacks: Optional[List] = None
content_policy_fallbacks: Optional[List] = None
allowed_fails: int = 3
num_retries_per_request: Optional[int] = (
None # for the request overall (incl. fallbacks + model retries)
)
num_retries_per_request: Optional[
int
] = None # for the request overall (incl. fallbacks + model retries)
####### SECRET MANAGERS #####################
secret_manager_client: Optional[Any] = (
None # list of instantiated key management clients - e.g. azure kv, infisical, etc.
)
secret_manager_client: Optional[
Any
] = None # list of instantiated key management clients - e.g. azure kv, infisical, etc.
_google_kms_resource_name: Optional[str] = None
_key_management_system: Optional[KeyManagementSystem] = None
_key_management_settings: KeyManagementSettings = KeyManagementSettings()
@ -448,6 +438,7 @@ snowflake_models: List = []
llama_models: List = []
nscale_models: List = []
def is_bedrock_pricing_only_model(key: str) -> bool:
"""
Excludes keys with the pattern 'bedrock/<region>/<model>'. These are in the model_prices_and_context_window.json file for pricing purposes only.
@ -1091,6 +1082,7 @@ from .proxy.proxy_cli import run_server
from .router import Router
from .assistants.main import *
from .batches.main import *
from .images.main import *
from .batch_completion.main import * # type: ignore
from .rerank_api.main import *
from .llms.anthropic.experimental_pass_through.messages.handler import *
@ -1117,10 +1109,10 @@ from .types.llms.custom_llm import CustomLLMItem
from .types.utils import GenericStreamingChunk
custom_provider_map: List[CustomLLMItem] = []
_custom_providers: List[str] = (
[]
) # internal helper util, used to track names of custom providers
disable_hf_tokenizer_download: Optional[bool] = (
None # disable huggingface tokenizer download. Defaults to openai clk100
)
_custom_providers: List[
str
] = [] # internal helper util, used to track names of custom providers
disable_hf_tokenizer_download: Optional[
bool
] = None # disable huggingface tokenizer download. Defaults to openai clk100
global_disable_no_log_param: bool = False

2
litellm/constants.py
View file

@ -153,12 +153,14 @@ FIREWORKS_AI_80_B = int(os.getenv("FIREWORKS_AI_80_B", 80))
#### Logging callback constants ####
REDACTED_BY_LITELM_STRING = "REDACTED_BY_LITELM"
############### LLM Provider Constants ###############
### ANTHROPIC CONSTANTS ###
ANTHROPIC_WEB_SEARCH_TOOL_MAX_USES = {
"low": 1,
"medium": 5,
"high": 10,
}
DEFAULT_IMAGE_ENDPOINT_MODEL = "dall-e-2"
LITELLM_CHAT_PROVIDERS = [
"openai",

736
litellm/images/main.py Normal file
View file

@ -0,0 +1,736 @@
import asyncio
import contextvars
from functools import partial
from typing import Any, Coroutine, Dict, Literal, Optional, Union, cast
import httpx
import litellm
from litellm import Logging, client, exception_type, get_litellm_params
from litellm.constants import DEFAULT_IMAGE_ENDPOINT_MODEL
from litellm.constants import request_timeout as DEFAULT_REQUEST_TIMEOUT
from litellm.exceptions import LiteLLMUnknownProvider
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.litellm_core_utils.mock_functions import mock_image_generation
from litellm.llms.base_llm import BaseImageEditConfig, BaseImageGenerationConfig
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
from litellm.llms.custom_llm import CustomLLM
#################### Initialize provider clients ####################
from litellm.main import (
azure_chat_completions,
base_llm_aiohttp_handler,
base_llm_http_handler,
bedrock_image_generation,
openai_chat_completions,
openai_image_variations,
vertex_image_generation,
)
from litellm.secret_managers.main import get_secret_str
from litellm.types.images.main import ImageEditOptionalRequestParams
from litellm.types.llms.openai import ImageGenerationRequestQuality
from litellm.types.router import GenericLiteLLMParams
from litellm.types.utils import (
LITELLM_IMAGE_VARIATION_PROVIDERS,
FileTypes,
LlmProviders,
all_litellm_params,
)
from litellm.utils import (
ImageResponse,
ProviderConfigManager,
get_llm_provider,
get_optional_params_image_gen,
)
from .utils import ImageEditRequestUtils
##### Image Generation #######################
@client
async def aimage_generation(*args, **kwargs) -> ImageResponse:
"""
Asynchronously calls the `image_generation` function with the given arguments and keyword arguments.
Parameters:
- `args` (tuple): Positional arguments to be passed to the `image_generation` function.
- `kwargs` (dict): Keyword arguments to be passed to the `image_generation` function.
Returns:
- `response` (Any): The response returned by the `image_generation` function.
"""
loop = asyncio.get_event_loop()
model = args[0] if len(args) > 0 else kwargs["model"]
### PASS ARGS TO Image Generation ###
kwargs["aimg_generation"] = True
custom_llm_provider = None
try:
# Use a partial function to pass your keyword arguments
func = partial(image_generation, *args, **kwargs)
# Add the context to the function
ctx = contextvars.copy_context()
func_with_context = partial(ctx.run, func)
_, custom_llm_provider, _, _ = get_llm_provider(
model=model, api_base=kwargs.get("api_base", None)
)
# Await normally
init_response = await loop.run_in_executor(None, func_with_context)
if isinstance(init_response, dict) or isinstance(
init_response, ImageResponse
): ## CACHING SCENARIO
if isinstance(init_response, dict):
init_response = ImageResponse(**init_response)
response = init_response
elif asyncio.iscoroutine(init_response):
response = await init_response # type: ignore
else:
# Call the synchronous function using run_in_executor
response = await loop.run_in_executor(None, func_with_context)
return response
except Exception as e:
custom_llm_provider = custom_llm_provider or "openai"
raise exception_type(
model=model,
custom_llm_provider=custom_llm_provider,
original_exception=e,
completion_kwargs=args,
extra_kwargs=kwargs,
)
@client
def image_generation( # noqa: PLR0915
prompt: str,
model: Optional[str] = None,
n: Optional[int] = None,
quality: Optional[Union[str, ImageGenerationRequestQuality]] = None,
response_format: Optional[str] = None,
size: Optional[str] = None,
style: Optional[str] = None,
user: Optional[str] = None,
timeout=600, # default to 10 minutes
api_key: Optional[str] = None,
api_base: Optional[str] = None,
api_version: Optional[str] = None,
custom_llm_provider=None,
**kwargs,
) -> ImageResponse:
"""
Maps the https://api.openai.com/v1/images/generations endpoint.
Currently supports just Azure + OpenAI.
"""
try:
args = locals()
aimg_generation = kwargs.get("aimg_generation", False)
litellm_call_id = kwargs.get("litellm_call_id", None)
logger_fn = kwargs.get("logger_fn", None)
mock_response: Optional[str] = kwargs.get("mock_response", None) # type: ignore
proxy_server_request = kwargs.get("proxy_server_request", None)
azure_ad_token_provider = kwargs.get("azure_ad_token_provider", None)
model_info = kwargs.get("model_info", None)
metadata = kwargs.get("metadata", {})
litellm_logging_obj: LiteLLMLoggingObj = kwargs.get("litellm_logging_obj") # type: ignore
client = kwargs.get("client", None)
extra_headers = kwargs.get("extra_headers", None)
headers: dict = kwargs.get("headers", None) or {}
base_model = kwargs.get("base_model", None)
if extra_headers is not None:
headers.update(extra_headers)
model_response: ImageResponse = litellm.utils.ImageResponse()
dynamic_api_key: Optional[str] = None
if model is not None or custom_llm_provider is not None:
model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider(
model=model, # type: ignore
custom_llm_provider=custom_llm_provider,
api_base=api_base,
)
else:
model = "dall-e-2"
custom_llm_provider = "openai" # default to dall-e-2 on openai
model_response._hidden_params["model"] = model
openai_params = [
"user",
"request_timeout",
"api_base",
"api_version",
"api_key",
"deployment_id",
"organization",
"base_url",
"default_headers",
"timeout",
"max_retries",
"n",
"quality",
"size",
"style",
]
litellm_params = all_litellm_params
default_params = openai_params + litellm_params
non_default_params = {
k: v for k, v in kwargs.items() if k not in default_params
} # model-specific params - pass them straight to the model/provider
image_generation_config: Optional[BaseImageGenerationConfig] = None
if (
custom_llm_provider is not None
and custom_llm_provider in LlmProviders._member_map_.values()
):
image_generation_config = (
ProviderConfigManager.get_provider_image_generation_config(
model=base_model or model,
provider=LlmProviders(custom_llm_provider),
)
)
optional_params = get_optional_params_image_gen(
model=base_model or model,
n=n,
quality=quality,
response_format=response_format,
size=size,
style=style,
user=user,
custom_llm_provider=custom_llm_provider,
provider_config=image_generation_config,
**non_default_params,
)
litellm_params_dict = get_litellm_params(**kwargs)
logging: Logging = litellm_logging_obj
logging.update_environment_variables(
model=model,
user=user,
optional_params=optional_params,
litellm_params={
"timeout": timeout,
"azure": False,
"litellm_call_id": litellm_call_id,
"logger_fn": logger_fn,
"proxy_server_request": proxy_server_request,
"model_info": model_info,
"metadata": metadata,
"preset_cache_key": None,
"stream_response": {},
},
custom_llm_provider=custom_llm_provider,
)
if "custom_llm_provider" not in logging.model_call_details:
logging.model_call_details["custom_llm_provider"] = custom_llm_provider
if mock_response is not None:
return mock_image_generation(model=model, mock_response=mock_response)
if custom_llm_provider == "azure":
# azure configs
api_type = get_secret_str("AZURE_API_TYPE") or "azure"
api_base = api_base or litellm.api_base or get_secret_str("AZURE_API_BASE")
api_version = (
api_version
or litellm.api_version
or get_secret_str("AZURE_API_VERSION")
)
api_key = (
api_key
or litellm.api_key
or litellm.azure_key
or get_secret_str("AZURE_OPENAI_API_KEY")
or get_secret_str("AZURE_API_KEY")
)
azure_ad_token = optional_params.pop(
"azure_ad_token", None
) or get_secret_str("AZURE_AD_TOKEN")
default_headers = {
"Content-Type": "application/json;",
"api-key": api_key,
}
for k, v in default_headers.items():
if k not in headers:
headers[k] = v
model_response = azure_chat_completions.image_generation(
model=model,
prompt=prompt,
timeout=timeout,
api_key=api_key,
api_base=api_base,
azure_ad_token=azure_ad_token,
azure_ad_token_provider=azure_ad_token_provider,
logging_obj=litellm_logging_obj,
optional_params=optional_params,
model_response=model_response,
api_version=api_version,
aimg_generation=aimg_generation,
client=client,
headers=headers,
litellm_params=litellm_params_dict,
)
elif (
custom_llm_provider == "openai"
or custom_llm_provider in litellm.openai_compatible_providers
):
model_response = openai_chat_completions.image_generation(
model=model,
prompt=prompt,
timeout=timeout,
api_key=api_key or dynamic_api_key,
api_base=api_base,
logging_obj=litellm_logging_obj,
optional_params=optional_params,
model_response=model_response,
aimg_generation=aimg_generation,
client=client,
)
elif custom_llm_provider == "bedrock":
if model is None:
raise Exception("Model needs to be set for bedrock")
model_response = bedrock_image_generation.image_generation( # type: ignore
model=model,
prompt=prompt,
timeout=timeout,
logging_obj=litellm_logging_obj,
optional_params=optional_params,
model_response=model_response,
aimg_generation=aimg_generation,
client=client,
)
elif custom_llm_provider == "vertex_ai":
vertex_ai_project = (
optional_params.pop("vertex_project", None)
or optional_params.pop("vertex_ai_project", None)
or litellm.vertex_project
or get_secret_str("VERTEXAI_PROJECT")
)
vertex_ai_location = (
optional_params.pop("vertex_location", None)
or optional_params.pop("vertex_ai_location", None)
or litellm.vertex_location
or get_secret_str("VERTEXAI_LOCATION")
)
vertex_credentials = (
optional_params.pop("vertex_credentials", None)
or optional_params.pop("vertex_ai_credentials", None)
or get_secret_str("VERTEXAI_CREDENTIALS")
)
api_base = (
api_base
or litellm.api_base
or get_secret_str("VERTEXAI_API_BASE")
or get_secret_str("VERTEX_API_BASE")
)
model_response = vertex_image_generation.image_generation(
model=model,
prompt=prompt,
timeout=timeout,
logging_obj=litellm_logging_obj,
optional_params=optional_params,
model_response=model_response,
vertex_project=vertex_ai_project,
vertex_location=vertex_ai_location,
vertex_credentials=vertex_credentials,
aimg_generation=aimg_generation,
api_base=api_base,
client=client,
)
elif (
custom_llm_provider in litellm._custom_providers
): # Assume custom LLM provider
# Get the Custom Handler
custom_handler: Optional[CustomLLM] = None
for item in litellm.custom_provider_map:
if item["provider"] == custom_llm_provider:
custom_handler = item["custom_handler"]
if custom_handler is None:
raise LiteLLMUnknownProvider(
model=model, custom_llm_provider=custom_llm_provider
)
## ROUTE LLM CALL ##
if aimg_generation is True:
async_custom_client: Optional[AsyncHTTPHandler] = None
if client is not None and isinstance(client, AsyncHTTPHandler):
async_custom_client = client
## CALL FUNCTION
model_response = custom_handler.aimage_generation( # type: ignore
model=model,
prompt=prompt,
api_key=api_key,
api_base=api_base,
model_response=model_response,
optional_params=optional_params,
logging_obj=litellm_logging_obj,
timeout=timeout,
client=async_custom_client,
)
else:
custom_client: Optional[HTTPHandler] = None
if client is not None and isinstance(client, HTTPHandler):
custom_client = client
## CALL FUNCTION
model_response = custom_handler.image_generation(
model=model,
prompt=prompt,
api_key=api_key,
api_base=api_base,
model_response=model_response,
optional_params=optional_params,
logging_obj=litellm_logging_obj,
timeout=timeout,
client=custom_client,
)
return model_response
except Exception as e:
## Map to OpenAI Exception
raise exception_type(
model=model,
custom_llm_provider=custom_llm_provider,
original_exception=e,
completion_kwargs=locals(),
extra_kwargs=kwargs,
)
@client
async def aimage_variation(*args, **kwargs) -> ImageResponse:
"""
Asynchronously calls the `image_variation` function with the given arguments and keyword arguments.
Parameters:
- `args` (tuple): Positional arguments to be passed to the `image_variation` function.
- `kwargs` (dict): Keyword arguments to be passed to the `image_variation` function.
Returns:
- `response` (Any): The response returned by the `image_variation` function.
"""
loop = asyncio.get_event_loop()
model = kwargs.get("model", None)
custom_llm_provider = kwargs.get("custom_llm_provider", None)
### PASS ARGS TO Image Generation ###
kwargs["async_call"] = True
try:
# Use a partial function to pass your keyword arguments
func = partial(image_variation, *args, **kwargs)
# Add the context to the function
ctx = contextvars.copy_context()
func_with_context = partial(ctx.run, func)
if custom_llm_provider is None and model is not None:
_, custom_llm_provider, _, _ = get_llm_provider(
model=model, api_base=kwargs.get("api_base", None)
)
# Await normally
init_response = await loop.run_in_executor(None, func_with_context)
if isinstance(init_response, dict) or isinstance(
init_response, ImageResponse
): ## CACHING SCENARIO
if isinstance(init_response, dict):
init_response = ImageResponse(**init_response)
response = init_response
elif asyncio.iscoroutine(init_response):
response = await init_response # type: ignore
else:
# Call the synchronous function using run_in_executor
response = await loop.run_in_executor(None, func_with_context)
return response
except Exception as e:
custom_llm_provider = custom_llm_provider or "openai"
raise exception_type(
model=model,
custom_llm_provider=custom_llm_provider,
original_exception=e,
completion_kwargs=args,
extra_kwargs=kwargs,
)
@client
def image_variation(
image: FileTypes,
model: str = "dall-e-2", # set to dall-e-2 by default - like OpenAI.
n: int = 1,
response_format: Literal["url", "b64_json"] = "url",
size: Optional[str] = None,
user: Optional[str] = None,
**kwargs,
) -> ImageResponse:
# get non-default params
client = kwargs.get("client", None)
# get logging object
litellm_logging_obj = cast(LiteLLMLoggingObj, kwargs.get("litellm_logging_obj"))
# get the litellm params
litellm_params = get_litellm_params(**kwargs)
# get the custom llm provider
model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider(
model=model,
custom_llm_provider=litellm_params.get("custom_llm_provider", None),
api_base=litellm_params.get("api_base", None),
api_key=litellm_params.get("api_key", None),
)
# route to the correct provider w/ the params
try:
llm_provider = LlmProviders(custom_llm_provider)
image_variation_provider = LITELLM_IMAGE_VARIATION_PROVIDERS(llm_provider)
except ValueError:
raise ValueError(
f"Invalid image variation provider: {custom_llm_provider}. Supported providers are: {LITELLM_IMAGE_VARIATION_PROVIDERS}"
)
model_response = ImageResponse()
response: Optional[ImageResponse] = None
provider_config = ProviderConfigManager.get_provider_model_info(
model=model or "", # openai defaults to dall-e-2
provider=llm_provider,
)
if provider_config is None:
raise ValueError(
f"image variation provider has no known model info config - required for getting api keys, etc.: {custom_llm_provider}. Supported providers are: {LITELLM_IMAGE_VARIATION_PROVIDERS}"
)
api_key = provider_config.get_api_key(litellm_params.get("api_key", None))
api_base = provider_config.get_api_base(litellm_params.get("api_base", None))
if image_variation_provider == LITELLM_IMAGE_VARIATION_PROVIDERS.OPENAI:
if api_key is None:
raise ValueError("API key is required for OpenAI image variations")
if api_base is None:
raise ValueError("API base is required for OpenAI image variations")
response = openai_image_variations.image_variations(
model_response=model_response,
api_key=api_key,
api_base=api_base,
model=model,
image=image,
timeout=litellm_params.get("timeout", None),
custom_llm_provider=custom_llm_provider,
logging_obj=litellm_logging_obj,
optional_params={},
litellm_params=litellm_params,
)
elif image_variation_provider == LITELLM_IMAGE_VARIATION_PROVIDERS.TOPAZ:
if api_key is None:
raise ValueError("API key is required for Topaz image variations")
if api_base is None:
raise ValueError("API base is required for Topaz image variations")
response = base_llm_aiohttp_handler.image_variations(
model_response=model_response,
api_key=api_key,
api_base=api_base,
model=model,
image=image,
timeout=litellm_params.get("timeout", None) or DEFAULT_REQUEST_TIMEOUT,
custom_llm_provider=custom_llm_provider,
logging_obj=litellm_logging_obj,
optional_params={},
litellm_params=litellm_params,
client=client,
)
# return the response
if response is None:
raise ValueError(
f"Invalid image variation provider: {custom_llm_provider}. Supported providers are: {LITELLM_IMAGE_VARIATION_PROVIDERS}"
)
return response
@client
def image_edit(
image: FileTypes,
prompt: str,
model: Optional[str] = None,
mask: Optional[str] = None,
n: Optional[int] = None,
quality: Optional[Union[str, ImageGenerationRequestQuality]] = None,
response_format: Optional[str] = None,
size: Optional[str] = None,
user: Optional[str] = None,
# Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs.
# The extra values given here take precedence over values defined on the client or passed to this method.
extra_headers: Optional[Dict[str, Any]] = None,
extra_query: Optional[Dict[str, Any]] = None,
extra_body: Optional[Dict[str, Any]] = None,
timeout: Optional[Union[float, httpx.Timeout]] = None,
# LiteLLM specific params,
custom_llm_provider: Optional[str] = None,
**kwargs,
) -> Union[ImageResponse, Coroutine[Any, Any, ImageResponse]]:
"""
Maps the image edit functionality, similar to OpenAI's images/edits endpoint.
"""
local_vars = locals()
try:
litellm_logging_obj: LiteLLMLoggingObj = kwargs.get("litellm_logging_obj") # type: ignore
litellm_call_id: Optional[str] = kwargs.get("litellm_call_id", None)
_is_async = kwargs.pop("adelete_responses", False) is True
# get llm provider logic
litellm_params = GenericLiteLLMParams(**kwargs)
model, custom_llm_provider, _, _ = get_llm_provider(
model=model or DEFAULT_IMAGE_ENDPOINT_MODEL,
custom_llm_provider=custom_llm_provider,
)
# get provider config
image_edit_provider_config: Optional[
BaseImageEditConfig
] = ProviderConfigManager.get_provider_image_edit_config(
model=model,
provider=litellm.LlmProviders(custom_llm_provider),
)
if image_edit_provider_config is None:
raise ValueError(f"image edit is not supported for {custom_llm_provider}")
local_vars.update(kwargs)
# Get ImageEditOptionalRequestParams with only valid parameters
image_edit_optional_params: ImageEditOptionalRequestParams = (
ImageEditRequestUtils.get_requested_image_edit_optional_param(local_vars)
)
# Get optional parameters for the responses API
image_edit_request_params: Dict = (
ImageEditRequestUtils.get_optional_params_image_edit(
model=model,
image_edit_provider_config=image_edit_provider_config,
image_edit_optional_params=image_edit_optional_params,
)
)
# Pre Call logging
litellm_logging_obj.update_environment_variables(
model=model,
user=user,
optional_params=dict(image_edit_request_params),
litellm_params={
"litellm_call_id": litellm_call_id,
**image_edit_request_params,
},
custom_llm_provider=custom_llm_provider,
)
# Call the handler with _is_async flag instead of directly calling the async handler
return base_llm_http_handler.image_edit_handler(
model=model,
image=image,
prompt=prompt,
image_edit_provider_config=image_edit_provider_config,
image_edit_optional_request_params=image_edit_request_params,
custom_llm_provider=custom_llm_provider,
litellm_params=litellm_params,
logging_obj=litellm_logging_obj,
extra_headers=extra_headers,
extra_body=extra_body,
timeout=timeout or DEFAULT_REQUEST_TIMEOUT,
_is_async=_is_async,
client=kwargs.get("client"),
)
except Exception as e:
raise litellm.exception_type(
model=None,
custom_llm_provider=custom_llm_provider,
original_exception=e,
completion_kwargs=local_vars,
extra_kwargs=kwargs,
)
@client
async def aimage_edit(
image: FileTypes,
model: str,
prompt: str,
mask: Optional[str] = None,
n: Optional[int] = None,
quality: Optional[Union[str, ImageGenerationRequestQuality]] = None,
response_format: Optional[str] = None,
size: Optional[str] = None,
user: Optional[str] = None,
# Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs.
# The extra values given here take precedence over values defined on the client or passed to this method.
extra_headers: Optional[Dict[str, Any]] = None,
extra_query: Optional[Dict[str, Any]] = None,
extra_body: Optional[Dict[str, Any]] = None,
timeout: Optional[Union[float, httpx.Timeout]] = None,
# LiteLLM specific params,
custom_llm_provider: Optional[str] = None,
**kwargs,
) -> ImageResponse:
"""
Asynchronously calls the `image_edit` function with the given arguments and keyword arguments.
Parameters:
- `args` (tuple): Positional arguments to be passed to the `image_edit` function.
- `kwargs` (dict): Keyword arguments to be passed to the `image_edit` function.
Returns:
- `response` (Any): The response returned by the `image_edit` function.
"""
local_vars = locals()
try:
loop = asyncio.get_event_loop()
kwargs["async_call"] = True
# get custom llm provider so we can use this for mapping exceptions
if custom_llm_provider is None:
_, custom_llm_provider, _, _ = litellm.get_llm_provider(
model=model, api_base=local_vars.get("base_url", None)
)
func = partial(
image_edit,
image=image,
prompt=prompt,
mask=mask,
model=model,
n=n,
quality=quality,
response_format=response_format,
size=size,
user=user,
timeout=timeout,
custom_llm_provider=custom_llm_provider,
**kwargs,
)
ctx = contextvars.copy_context()
func_with_context = partial(ctx.run, func)
init_response = await loop.run_in_executor(None, func_with_context)
if asyncio.iscoroutine(init_response):
response = await init_response
else:
response = init_response
return response
except Exception as e:
raise litellm.exception_type(
model=None,
custom_llm_provider=custom_llm_provider,
original_exception=e,
completion_kwargs=local_vars,
extra_kwargs=kwargs,
)

71
litellm/images/utils.py Normal file
View file

@ -0,0 +1,71 @@
from typing import Any, Dict, cast, get_type_hints
import litellm
from litellm.llms.base_llm.image_edit.transformation import BaseImageEditConfig
from litellm.types.images.main import ImageEditOptionalRequestParams
class ImageEditRequestUtils:
@staticmethod
def get_optional_params_image_edit(
model: str,
image_edit_provider_config: BaseImageEditConfig,
image_edit_optional_params: ImageEditOptionalRequestParams,
) -> Dict:
"""
Get optional parameters for the image edit API.
Args:
params: Dictionary of all parameters
model: The model name
image_edit_provider_config: The provider configuration for image edit API
Returns:
A dictionary of supported parameters for the image edit API
"""
# Remove None values and internal parameters
# Get supported parameters for the model
supported_params = image_edit_provider_config.get_supported_openai_params(model)
# Check for unsupported parameters
unsupported_params = [
param
for param in image_edit_optional_params
if param not in supported_params
]
if unsupported_params:
raise litellm.UnsupportedParamsError(
model=model,
message=f"The following parameters are not supported for model {model}: {', '.join(unsupported_params)}",
)
# Map parameters to provider-specific format
mapped_params = image_edit_provider_config.map_openai_params(
image_edit_optional_params=image_edit_optional_params,
model=model,
drop_params=litellm.drop_params,
)
return mapped_params
@staticmethod
def get_requested_image_edit_optional_param(
params: Dict[str, Any],
) -> ImageEditOptionalRequestParams:
"""
Filter parameters to only include those defined in ImageEditOptionalRequestParams.
Args:
params: Dictionary of parameters to filter
Returns:
ImageEditOptionalRequestParams instance with only the valid parameters
"""
valid_keys = get_type_hints(ImageEditOptionalRequestParams).keys()
filtered_params = {
k: v for k, v in params.items() if k in valid_keys and v is not None
}
return cast(ImageEditOptionalRequestParams, filtered_params)

7
litellm/litellm_core_utils/streaming_handler.py
View file

@ -697,10 +697,11 @@ class CustomStreamWrapper:
If a valid ID is received in any chunk, use it for the response.
"""
if self.response_id is None and id and isinstance(id, str) and id.strip():
self.response_id = id
if id and isinstance(id, str) and id.strip():
self.response_id = id
elif self.response_id is None:
self.response_id = id
model_response._hidden_params["received_model_id"] = id
if self.response_id is not None and isinstance(self.response_id, str):
model_response.id = self.response_id

2
litellm/llms/base_llm/__init__.py
View file

@ -2,6 +2,7 @@ from .anthropic_messages.transformation import BaseAnthropicMessagesConfig
from .audio_transcription.transformation import BaseAudioTranscriptionConfig
from .chat.transformation import BaseConfig
from .embedding.transformation import BaseEmbeddingConfig
from .image_edit.transformation import BaseImageEditConfig
from .image_generation.transformation import BaseImageGenerationConfig
__all__ = [
@ -10,4 +11,5 @@ __all__ = [
"BaseAudioTranscriptionConfig",
"BaseAnthropicMessagesConfig",
"BaseEmbeddingConfig",
"BaseImageEditConfig",
]

120
litellm/llms/base_llm/image_edit/transformation.py Normal file
View file

@ -0,0 +1,120 @@
import types
from abc import ABC, abstractmethod
from typing import TYPE_CHECKING, Any, Dict, Optional, Tuple
import httpx
from httpx._types import RequestFiles
from litellm.types.images.main import ImageEditOptionalRequestParams
from litellm.types.responses.main import *
from litellm.types.router import GenericLiteLLMParams
from litellm.types.utils import FileTypes
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
from litellm.utils import ImageResponse as _ImageResponse
from ..chat.transformation import BaseLLMException as _BaseLLMException
LiteLLMLoggingObj = _LiteLLMLoggingObj
BaseLLMException = _BaseLLMException
ImageResponse = _ImageResponse
else:
LiteLLMLoggingObj = Any
BaseLLMException = Any
ImageResponse = Any
class BaseImageEditConfig(ABC):
def __init__(self):
pass
@classmethod
def get_config(cls):
return {
k: v
for k, v in cls.__dict__.items()
if not k.startswith("__")
and not k.startswith("_abc")
and not isinstance(
v,
(
types.FunctionType,
types.BuiltinFunctionType,
classmethod,
staticmethod,
),
)
and v is not None
}
@abstractmethod
def get_supported_openai_params(self, model: str) -> list:
pass
@abstractmethod
def map_openai_params(
self,
image_edit_optional_params: ImageEditOptionalRequestParams,
model: str,
drop_params: bool,
) -> Dict:
pass
@abstractmethod
def validate_environment(
self,
headers: dict,
model: str,
api_key: Optional[str] = None,
) -> dict:
return {}
@abstractmethod
def get_complete_url(
self,
api_base: Optional[str],
litellm_params: dict,
) -> str:
"""
OPTIONAL
Get the complete url for the request
Some providers need `model` in `api_base`
"""
if api_base is None:
raise ValueError("api_base is required")
return api_base
@abstractmethod
def transform_image_edit_request(
self,
model: str,
prompt: str,
image: FileTypes,
image_edit_optional_request_params: Dict,
litellm_params: GenericLiteLLMParams,
headers: dict,
) -> Tuple[Dict, RequestFiles]:
pass
@abstractmethod
def transform_image_edit_response(
self,
model: str,
raw_response: httpx.Response,
logging_obj: LiteLLMLoggingObj,
) -> ImageResponse:
pass
def get_error_class(
self, error_message: str, status_code: int, headers: Union[dict, httpx.Headers]
) -> BaseLLMException:
from ..chat.transformation import BaseLLMException
raise BaseLLMException(
status_code=status_code,
message=error_message,
headers=headers,
)

2
litellm/llms/custom_httpx/aiohttp_handler.py
View file

@ -102,7 +102,7 @@ class BaseLLMAIOHTTPHandler:
api_base: str,
headers: dict,
data: dict,
timeout: Union[float, httpx.Timeout],
timeout: Optional[Union[float, httpx.Timeout]],
litellm_params: dict,
stream: bool = False,
files: Optional[dict] = None,

13
litellm/llms/custom_httpx/http_handler.py
View file

@ -6,6 +6,7 @@ from typing import TYPE_CHECKING, Any, Callable, List, Mapping, Optional, Union
import httpx
from httpx import USE_CLIENT_DEFAULT, AsyncHTTPTransport, HTTPTransport
from httpx._types import RequestFiles
import litellm
from litellm.constants import _DEFAULT_TTL_FOR_HTTPX_CLIENTS
@ -199,6 +200,7 @@ class AsyncHTTPHandler:
timeout: Optional[Union[float, httpx.Timeout]] = None,
stream: bool = False,
logging_obj: Optional[LiteLLMLoggingObject] = None,
files: Optional[RequestFiles] = None,
):
start_time = time.time()
try:
@ -206,7 +208,14 @@ class AsyncHTTPHandler:
timeout = self.timeout
req = self.client.build_request(
"POST", url, data=data, json=json, params=params, headers=headers, timeout=timeout # type: ignore
"POST",
url,
data=data, # type: ignore
json=json,
params=params,
headers=headers,
timeout=timeout,
files=files,
)
response = await self.client.send(req, stream=stream)
response.raise_for_status()
@ -533,7 +542,7 @@ class HTTPHandler:
headers: Optional[dict] = None,
stream: bool = False,
timeout: Optional[Union[float, httpx.Timeout]] = None,
files: Optional[dict] = None,
files: Optional[Union[dict, RequestFiles]] = None,
content: Any = None,
logging_obj: Optional[LiteLLMLoggingObject] = None,
):

200
litellm/llms/custom_httpx/llm_http_handler.py
View file

@ -30,6 +30,7 @@ from litellm.llms.base_llm.base_model_iterator import MockResponseIterator
from litellm.llms.base_llm.chat.transformation import BaseConfig
from litellm.llms.base_llm.embedding.transformation import BaseEmbeddingConfig
from litellm.llms.base_llm.files.transformation import BaseFilesConfig
from litellm.llms.base_llm.image_edit.transformation import BaseImageEditConfig
from litellm.llms.base_llm.realtime.transformation import BaseRealtimeConfig
from litellm.llms.base_llm.rerank.transformation import BaseRerankConfig
from litellm.llms.base_llm.responses.transformation import BaseResponsesAPIConfig
@ -58,7 +59,12 @@ from litellm.types.rerank import OptionalRerankParams, RerankResponse
from litellm.types.responses.main import DeleteResponseResult
from litellm.types.router import GenericLiteLLMParams
from litellm.types.utils import EmbeddingResponse, FileTypes, TranscriptionResponse
from litellm.utils import CustomStreamWrapper, ModelResponse, ProviderConfigManager
from litellm.utils import (
CustomStreamWrapper,
ImageResponse,
ModelResponse,
ProviderConfigManager,
)
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
@ -2018,7 +2024,9 @@ class BaseLLMHTTPHandler:
def _handle_error(
self,
e: Exception,
provider_config: Union[BaseConfig, BaseRerankConfig, BaseResponsesAPIConfig],
provider_config: Union[
BaseConfig, BaseRerankConfig, BaseResponsesAPIConfig, BaseImageEditConfig
],
):
status_code = getattr(e, "status_code", 500)
error_headers = getattr(e, "headers", None)
@ -2098,3 +2106,191 @@ class BaseLLMHTTPHandler:
raise Exception(
f"Unexpected error while closing WebSocket: {close_error}"
)
def image_edit_handler(
self,
model: str,
image: Any,
prompt: str,
image_edit_provider_config: BaseImageEditConfig,
image_edit_optional_request_params: Dict,
custom_llm_provider: str,
litellm_params: GenericLiteLLMParams,
logging_obj: LiteLLMLoggingObj,
timeout: Union[float, httpx.Timeout],
extra_headers: Optional[Dict[str, Any]] = None,
extra_body: Optional[Dict[str, Any]] = None,
client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None,
_is_async: bool = False,
fake_stream: bool = False,
litellm_metadata: Optional[Dict[str, Any]] = None,
) -> Union[ImageResponse, Coroutine[Any, Any, ImageResponse],]:
"""
Handles image edit requests.
When _is_async=True, returns a coroutine instead of making the call directly.
"""
if _is_async:
# Return the async coroutine if called with _is_async=True
return self.async_image_edit_handler(
model=model,
image=image,
prompt=prompt,
image_edit_provider_config=image_edit_provider_config,
image_edit_optional_request_params=image_edit_optional_request_params,
custom_llm_provider=custom_llm_provider,
litellm_params=litellm_params,
logging_obj=logging_obj,
extra_headers=extra_headers,
extra_body=extra_body,
timeout=timeout,
client=client if isinstance(client, AsyncHTTPHandler) else None,
fake_stream=fake_stream,
litellm_metadata=litellm_metadata,
)
if client is None or not isinstance(client, HTTPHandler):
sync_httpx_client = _get_httpx_client(
params={"ssl_verify": litellm_params.get("ssl_verify", None)}
)
else:
sync_httpx_client = client
headers = image_edit_provider_config.validate_environment(
api_key=litellm_params.api_key,
headers=image_edit_optional_request_params.get("extra_headers", {}) or {},
model=model,
)
if extra_headers:
headers.update(extra_headers)
api_base = image_edit_provider_config.get_complete_url(
api_base=litellm_params.api_base,
litellm_params=dict(litellm_params),
)
data, files = image_edit_provider_config.transform_image_edit_request(
model=model,
image=image,
prompt=prompt,
image_edit_optional_request_params=image_edit_optional_request_params,
litellm_params=litellm_params,
headers=headers,
)
## LOGGING
logging_obj.pre_call(
input=prompt,
api_key="",
additional_args={
"complete_input_dict": data,
"api_base": api_base,
"headers": headers,
},
)
try:
response = sync_httpx_client.post(
url=api_base,
headers=headers,
data=data,
files=files,
timeout=timeout,
)
except Exception as e:
raise self._handle_error(
e=e,
provider_config=image_edit_provider_config,
)
return image_edit_provider_config.transform_image_edit_response(
model=model,
raw_response=response,
logging_obj=logging_obj,
)
async def async_image_edit_handler(
self,
model: str,
image: FileTypes,
prompt: str,
image_edit_provider_config: BaseImageEditConfig,
image_edit_optional_request_params: Dict,
custom_llm_provider: str,
litellm_params: GenericLiteLLMParams,
logging_obj: LiteLLMLoggingObj,
timeout: Union[float, httpx.Timeout],
extra_headers: Optional[Dict[str, Any]] = None,
extra_body: Optional[Dict[str, Any]] = None,
client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None,
fake_stream: bool = False,
litellm_metadata: Optional[Dict[str, Any]] = None,
) -> ImageResponse:
"""
Async version of the image edit handler.
Uses async HTTP client to make requests.
"""
if client is None or not isinstance(client, AsyncHTTPHandler):
async_httpx_client = get_async_httpx_client(
llm_provider=litellm.LlmProviders(custom_llm_provider),
params={"ssl_verify": litellm_params.get("ssl_verify", None)},
)
else:
async_httpx_client = client
headers = image_edit_provider_config.validate_environment(
api_key=litellm_params.api_key,
headers=image_edit_optional_request_params.get("extra_headers", {}) or {},
model=model,
)
if extra_headers:
headers.update(extra_headers)
api_base = image_edit_provider_config.get_complete_url(
api_base=litellm_params.api_base,
litellm_params=dict(litellm_params),
)
data, files = image_edit_provider_config.transform_image_edit_request(
model=model,
image=image,
prompt=prompt,
image_edit_optional_request_params=image_edit_optional_request_params,
litellm_params=litellm_params,
headers=headers,
)
## LOGGING
logging_obj.pre_call(
input=prompt,
api_key="",
additional_args={
"complete_input_dict": data,
"api_base": api_base,
"headers": headers,
},
)
try:
response = await async_httpx_client.post(
url=api_base,
headers=headers,
data=data,
files=files,
timeout=timeout,
)
except Exception as e:
raise self._handle_error(
e=e,
provider_config=image_edit_provider_config,
)
return image_edit_provider_config.transform_image_edit_response(
model=model,
raw_response=response,
logging_obj=logging_obj,
)

38
litellm/llms/ollama/common_utils.py
View file

@ -1,14 +1,11 @@
from typing import Union
from typing import Optional, Union
import httpx
from litellm import verbose_logger
# dynamic import to allow usage even if httpx is not installed in dev env
try:
import httpx
except ImportError:
httpx = None # type: ignore
from litellm.llms.base_llm.chat.transformation import BaseLLMException
class OllamaError(BaseLLMException):
def __init__(
self, status_code: int, message: str, headers: Union[dict, httpx.Headers]
@ -51,27 +48,30 @@ def _convert_image(image):
from litellm.llms.base_llm.base_utils import BaseLLMModelInfo
class OllamaModelInfo(BaseLLMModelInfo):
"""
Dynamic model listing for Ollama server.
Fetches /api/models and /api/tags, then for each tag also /api/models?tag=...
Returns the union of all model names.
"""
@staticmethod
def get_api_key(api_key=None) -> None:
return None # Ollama does not use an API key by default
@staticmethod
def get_api_base(api_base: str | None = None) -> str:
def get_api_base(api_base: Optional[str] = None) -> str:
from litellm.secret_managers.main import get_secret_str
# env var OLLAMA_API_BASE or default
return api_base or get_secret_str("OLLAMA_API_BASE") or "http://localhost:11434"
def get_models(self, api_key=None, api_base: str | None = None) -> list[str]:
def get_models(self, api_key=None, api_base: Optional[str] = None) -> list[str]:
"""
List all models available on the Ollama server via /api/tags endpoint.
"""
import httpx
base = self.get_api_base(api_base)
names: set[str] = set()
try:
@ -80,15 +80,19 @@ class OllamaModelInfo(BaseLLMModelInfo):
data = resp.json()
# Expecting a dict with a 'models' list
models_list = []
if isinstance(data, dict) and 'models' in data and isinstance(data['models'], list):
models_list = data['models']
if (
isinstance(data, dict)
and "models" in data
and isinstance(data["models"], list)
):
models_list = data["models"]
elif isinstance(data, list):
models_list = data
# Extract model names
for entry in models_list:
if not isinstance(entry, dict):
continue
nm = entry.get('name') or entry.get('model')
nm = entry.get("name") or entry.get("model")
if isinstance(nm, str):
names.add(nm)
except Exception as e:
@ -96,14 +100,18 @@ class OllamaModelInfo(BaseLLMModelInfo):
# If tags endpoint fails, fall back to static list
try:
from litellm import models_by_provider
static = models_by_provider.get("ollama", []) or []
return [f"ollama/{m}" for m in static]
except Exception as e1:
verbose_logger.warning(f"Error retrieving static ollama models as fallback: {e1}")
verbose_logger.warning(
f"Error retrieving static ollama models as fallback: {e1}"
)
return []
# assemble full model names
result = sorted(names)
return result
def validate_environment(
self,
headers: dict,

147
litellm/llms/openai/image_edit/transformation.py Normal file
View file

@ -0,0 +1,147 @@
from io import BufferedReader
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, cast
import httpx
from httpx._types import RequestFiles
import litellm
from litellm.llms.base_llm.image_edit.transformation import BaseImageEditConfig
from litellm.secret_managers.main import get_secret_str
from litellm.types.images.main import (
ImageEditOptionalRequestParams,
ImageEditRequestParams,
)
from litellm.types.llms.openai import FileTypes
from litellm.types.router import GenericLiteLLMParams
from litellm.utils import ImageResponse
from ..common_utils import OpenAIError
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
LiteLLMLoggingObj = _LiteLLMLoggingObj
else:
LiteLLMLoggingObj = Any
class OpenAIImageEditConfig(BaseImageEditConfig):
def get_supported_openai_params(self, model: str) -> list:
"""
All OpenAI Image Edits params are supported
"""
return [
"image",
"prompt",
"background",
"mask",
"model",
"n",
"quality",
"response_format",
"size",
"user",
"extra_headers",
"extra_query",
"extra_body",
"timeout",
]
def map_openai_params(
self,
image_edit_optional_params: ImageEditOptionalRequestParams,
model: str,
drop_params: bool,
) -> Dict:
"""No mapping applied since inputs are in OpenAI spec already"""
return dict(image_edit_optional_params)
def transform_image_edit_request(
self,
model: str,
prompt: str,
image: FileTypes,
image_edit_optional_request_params: Dict,
litellm_params: GenericLiteLLMParams,
headers: dict,
) -> Tuple[Dict, RequestFiles]:
"""
No transform applied since inputs are in OpenAI spec already
This handles buffered readers as images to be sent as multipart/form-data for OpenAI
"""
request = ImageEditRequestParams(
model=model,
image=image,
prompt=prompt,
**image_edit_optional_request_params,
)
request_dict = cast(Dict, request)
#########################################################
# Separate images as `files` and send other parameters as `data`
#########################################################
_images = request_dict.get("image") or []
data_without_images = {k: v for k, v in request_dict.items() if k != "image"}
files_list: List[Tuple[str, Any]] = []
for _image in _images:
if isinstance(_image, BufferedReader):
files_list.append(("image[]", (_image.name, _image, "image/png")))
else:
files_list.append(("image[]", (_image, "image/png")))
return data_without_images, files_list
def transform_image_edit_response(
self,
model: str,
raw_response: httpx.Response,
logging_obj: LiteLLMLoggingObj,
) -> ImageResponse:
"""No transform applied since outputs are in OpenAI spec already"""
try:
raw_response_json = raw_response.json()
except Exception:
raise OpenAIError(
message=raw_response.text, status_code=raw_response.status_code
)
return ImageResponse(**raw_response_json)
def validate_environment(
self,
headers: dict,
model: str,
api_key: Optional[str] = None,
) -> dict:
api_key = (
api_key
or litellm.api_key
or litellm.openai_key
or get_secret_str("OPENAI_API_KEY")
)
headers.update(
{
"Authorization": f"Bearer {api_key}",
}
)
return headers
def get_complete_url(
self,
api_base: Optional[str],
litellm_params: dict,
) -> str:
"""
Get the endpoint for OpenAI responses API
"""
api_base = (
api_base
or litellm.api_base
or get_secret_str("OPENAI_BASE_URL")
or get_secret_str("OPENAI_API_BASE")
or "https://api.openai.com/v1"
)
# Remove trailing slashes
api_base = api_base.rstrip("/")
return f"{api_base}/images/edits"

4
litellm/llms/openai/image_variations/handler.py
View file

@ -50,7 +50,7 @@ class OpenAIImageVariationsHandler:
data: dict,
headers: dict,
model: Optional[str],
timeout: float,
timeout: Optional[float],
max_retries: int,
logging_obj: LiteLLMLoggingObj,
model_response: ImageResponse,
@ -123,7 +123,7 @@ class OpenAIImageVariationsHandler:
api_base: str,
model: Optional[str],
image: FileTypes,
timeout: float,
timeout: Optional[float],
custom_llm_provider: str,
logging_obj: LiteLLMLoggingObj,
optional_params: dict,

513
litellm/main.py
View file

@ -183,12 +183,10 @@ from .types.llms.openai import (
ChatCompletionPredictionContentParam,
ChatCompletionUserMessage,
HttpxBinaryResponseContent,
ImageGenerationRequestQuality,
OpenAIModerationResponse,
OpenAIWebSearchOptions,
)
from .types.utils import (
LITELLM_IMAGE_VARIATION_PROVIDERS,
AdapterCompletionStreamWrapper,
ChatCompletionMessageToolCall,
CompletionTokensDetails,
@ -204,7 +202,6 @@ encoding = tiktoken.get_encoding("cl100k_base")
from litellm.utils import (
Choices,
EmbeddingResponse,
ImageResponse,
Message,
ModelResponse,
TextChoices,
@ -4578,516 +4575,6 @@ async def amoderation(
)
##### Image Generation #######################
@client
async def aimage_generation(*args, **kwargs) -> ImageResponse:
"""
Asynchronously calls the `image_generation` function with the given arguments and keyword arguments.
Parameters:
- `args` (tuple): Positional arguments to be passed to the `image_generation` function.
- `kwargs` (dict): Keyword arguments to be passed to the `image_generation` function.
Returns:
- `response` (Any): The response returned by the `image_generation` function.
"""
loop = asyncio.get_event_loop()
model = args[0] if len(args) > 0 else kwargs["model"]
### PASS ARGS TO Image Generation ###
kwargs["aimg_generation"] = True
custom_llm_provider = None
try:
# Use a partial function to pass your keyword arguments
func = partial(image_generation, *args, **kwargs)
# Add the context to the function
ctx = contextvars.copy_context()
func_with_context = partial(ctx.run, func)
_, custom_llm_provider, _, _ = get_llm_provider(
model=model, api_base=kwargs.get("api_base", None)
)
# Await normally
init_response = await loop.run_in_executor(None, func_with_context)
if isinstance(init_response, dict) or isinstance(
init_response, ImageResponse
): ## CACHING SCENARIO
if isinstance(init_response, dict):
init_response = ImageResponse(**init_response)
response = init_response
elif asyncio.iscoroutine(init_response):
response = await init_response # type: ignore
else:
# Call the synchronous function using run_in_executor
response = await loop.run_in_executor(None, func_with_context)
return response
except Exception as e:
custom_llm_provider = custom_llm_provider or "openai"
raise exception_type(
model=model,
custom_llm_provider=custom_llm_provider,
original_exception=e,
completion_kwargs=args,
extra_kwargs=kwargs,
)
@client
def image_generation( # noqa: PLR0915
prompt: str,
model: Optional[str] = None,
n: Optional[int] = None,
quality: Optional[Union[str, ImageGenerationRequestQuality]] = None,
response_format: Optional[str] = None,
size: Optional[str] = None,
style: Optional[str] = None,
user: Optional[str] = None,
timeout=600, # default to 10 minutes
api_key: Optional[str] = None,
api_base: Optional[str] = None,
api_version: Optional[str] = None,
custom_llm_provider=None,
**kwargs,
) -> ImageResponse:
"""
Maps the https://api.openai.com/v1/images/generations endpoint.
Currently supports just Azure + OpenAI.
"""
try:
args = locals()
aimg_generation = kwargs.get("aimg_generation", False)
litellm_call_id = kwargs.get("litellm_call_id", None)
logger_fn = kwargs.get("logger_fn", None)
mock_response: Optional[str] = kwargs.get("mock_response", None) # type: ignore
proxy_server_request = kwargs.get("proxy_server_request", None)
azure_ad_token_provider = kwargs.get("azure_ad_token_provider", None)
model_info = kwargs.get("model_info", None)
metadata = kwargs.get("metadata", {})
litellm_logging_obj: LiteLLMLoggingObj = kwargs.get("litellm_logging_obj") # type: ignore
client = kwargs.get("client", None)
extra_headers = kwargs.get("extra_headers", None)
headers: dict = kwargs.get("headers", None) or {}
base_model = kwargs.get("base_model", None)
if extra_headers is not None:
headers.update(extra_headers)
model_response: ImageResponse = litellm.utils.ImageResponse()
dynamic_api_key: Optional[str] = None
if model is not None or custom_llm_provider is not None:
model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider(
model=model, # type: ignore
custom_llm_provider=custom_llm_provider,
api_base=api_base,
)
else:
model = "dall-e-2"
custom_llm_provider = "openai" # default to dall-e-2 on openai
model_response._hidden_params["model"] = model
openai_params = [
"user",
"request_timeout",
"api_base",
"api_version",
"api_key",
"deployment_id",
"organization",
"base_url",
"default_headers",
"timeout",
"max_retries",
"n",
"quality",
"size",
"style",
]
litellm_params = all_litellm_params
default_params = openai_params + litellm_params
non_default_params = {
k: v for k, v in kwargs.items() if k not in default_params
} # model-specific params - pass them straight to the model/provider
image_generation_config: Optional[BaseImageGenerationConfig] = None
if (
custom_llm_provider is not None
and custom_llm_provider in LlmProviders._member_map_.values()
):
image_generation_config = (
ProviderConfigManager.get_provider_image_generation_config(
model=base_model or model,
provider=LlmProviders(custom_llm_provider),
)
)
optional_params = get_optional_params_image_gen(
model=base_model or model,
n=n,
quality=quality,
response_format=response_format,
size=size,
style=style,
user=user,
custom_llm_provider=custom_llm_provider,
provider_config=image_generation_config,
**non_default_params,
)
litellm_params_dict = get_litellm_params(**kwargs)
logging: Logging = litellm_logging_obj
logging.update_environment_variables(
model=model,
user=user,
optional_params=optional_params,
litellm_params={
"timeout": timeout,
"azure": False,
"litellm_call_id": litellm_call_id,
"logger_fn": logger_fn,
"proxy_server_request": proxy_server_request,
"model_info": model_info,
"metadata": metadata,
"preset_cache_key": None,
"stream_response": {},
},
custom_llm_provider=custom_llm_provider,
)
if "custom_llm_provider" not in logging.model_call_details:
logging.model_call_details["custom_llm_provider"] = custom_llm_provider
if mock_response is not None:
return mock_image_generation(model=model, mock_response=mock_response)
if custom_llm_provider == "azure":
# azure configs
api_type = get_secret_str("AZURE_API_TYPE") or "azure"
api_base = api_base or litellm.api_base or get_secret_str("AZURE_API_BASE")
api_version = (
api_version
or litellm.api_version
or get_secret_str("AZURE_API_VERSION")
)
api_key = (
api_key
or litellm.api_key
or litellm.azure_key
or get_secret_str("AZURE_OPENAI_API_KEY")
or get_secret_str("AZURE_API_KEY")
)
azure_ad_token = optional_params.pop(
"azure_ad_token", None
) or get_secret_str("AZURE_AD_TOKEN")
default_headers = {
"Content-Type": "application/json;",
"api-key": api_key,
}
for k, v in default_headers.items():
if k not in headers:
headers[k] = v
model_response = azure_chat_completions.image_generation(
model=model,
prompt=prompt,
timeout=timeout,
api_key=api_key,
api_base=api_base,
azure_ad_token=azure_ad_token,
azure_ad_token_provider=azure_ad_token_provider,
logging_obj=litellm_logging_obj,
optional_params=optional_params,
model_response=model_response,
api_version=api_version,
aimg_generation=aimg_generation,
client=client,
headers=headers,
litellm_params=litellm_params_dict,
)
elif (
custom_llm_provider == "openai"
or custom_llm_provider in litellm.openai_compatible_providers
):
model_response = openai_chat_completions.image_generation(
model=model,
prompt=prompt,
timeout=timeout,
api_key=api_key or dynamic_api_key,
api_base=api_base,
logging_obj=litellm_logging_obj,
optional_params=optional_params,
model_response=model_response,
aimg_generation=aimg_generation,
client=client,
)
elif custom_llm_provider == "bedrock":
if model is None:
raise Exception("Model needs to be set for bedrock")
model_response = bedrock_image_generation.image_generation( # type: ignore
model=model,
prompt=prompt,
timeout=timeout,
logging_obj=litellm_logging_obj,
optional_params=optional_params,
model_response=model_response,
aimg_generation=aimg_generation,
client=client,
)
elif custom_llm_provider == "vertex_ai":
vertex_ai_project = (
optional_params.pop("vertex_project", None)
or optional_params.pop("vertex_ai_project", None)
or litellm.vertex_project
or get_secret_str("VERTEXAI_PROJECT")
)
vertex_ai_location = (
optional_params.pop("vertex_location", None)
or optional_params.pop("vertex_ai_location", None)
or litellm.vertex_location
or get_secret_str("VERTEXAI_LOCATION")
)
vertex_credentials = (
optional_params.pop("vertex_credentials", None)
or optional_params.pop("vertex_ai_credentials", None)
or get_secret_str("VERTEXAI_CREDENTIALS")
)
api_base = (
api_base
or litellm.api_base
or get_secret_str("VERTEXAI_API_BASE")
or get_secret_str("VERTEX_API_BASE")
)
model_response = vertex_image_generation.image_generation(
model=model,
prompt=prompt,
timeout=timeout,
logging_obj=litellm_logging_obj,
optional_params=optional_params,
model_response=model_response,
vertex_project=vertex_ai_project,
vertex_location=vertex_ai_location,
vertex_credentials=vertex_credentials,
aimg_generation=aimg_generation,
api_base=api_base,
client=client,
)
elif (
custom_llm_provider in litellm._custom_providers
): # Assume custom LLM provider
# Get the Custom Handler
custom_handler: Optional[CustomLLM] = None
for item in litellm.custom_provider_map:
if item["provider"] == custom_llm_provider:
custom_handler = item["custom_handler"]
if custom_handler is None:
raise LiteLLMUnknownProvider(
model=model, custom_llm_provider=custom_llm_provider
)
## ROUTE LLM CALL ##
if aimg_generation is True:
async_custom_client: Optional[AsyncHTTPHandler] = None
if client is not None and isinstance(client, AsyncHTTPHandler):
async_custom_client = client
## CALL FUNCTION
model_response = custom_handler.aimage_generation( # type: ignore
model=model,
prompt=prompt,
api_key=api_key,
api_base=api_base,
model_response=model_response,
optional_params=optional_params,
logging_obj=litellm_logging_obj,
timeout=timeout,
client=async_custom_client,
)
else:
custom_client: Optional[HTTPHandler] = None
if client is not None and isinstance(client, HTTPHandler):
custom_client = client
## CALL FUNCTION
model_response = custom_handler.image_generation(
model=model,
prompt=prompt,
api_key=api_key,
api_base=api_base,
model_response=model_response,
optional_params=optional_params,
logging_obj=litellm_logging_obj,
timeout=timeout,
client=custom_client,
)
return model_response
except Exception as e:
## Map to OpenAI Exception
raise exception_type(
model=model,
custom_llm_provider=custom_llm_provider,
original_exception=e,
completion_kwargs=locals(),
extra_kwargs=kwargs,
)
@client
async def aimage_variation(*args, **kwargs) -> ImageResponse:
"""
Asynchronously calls the `image_variation` function with the given arguments and keyword arguments.
Parameters:
- `args` (tuple): Positional arguments to be passed to the `image_variation` function.
- `kwargs` (dict): Keyword arguments to be passed to the `image_variation` function.
Returns:
- `response` (Any): The response returned by the `image_variation` function.
"""
loop = asyncio.get_event_loop()
model = kwargs.get("model", None)
custom_llm_provider = kwargs.get("custom_llm_provider", None)
### PASS ARGS TO Image Generation ###
kwargs["async_call"] = True
try:
# Use a partial function to pass your keyword arguments
func = partial(image_variation, *args, **kwargs)
# Add the context to the function
ctx = contextvars.copy_context()
func_with_context = partial(ctx.run, func)
if custom_llm_provider is None and model is not None:
_, custom_llm_provider, _, _ = get_llm_provider(
model=model, api_base=kwargs.get("api_base", None)
)
# Await normally
init_response = await loop.run_in_executor(None, func_with_context)
if isinstance(init_response, dict) or isinstance(
init_response, ImageResponse
): ## CACHING SCENARIO
if isinstance(init_response, dict):
init_response = ImageResponse(**init_response)
response = init_response
elif asyncio.iscoroutine(init_response):
response = await init_response # type: ignore
else:
# Call the synchronous function using run_in_executor
response = await loop.run_in_executor(None, func_with_context)
return response
except Exception as e:
custom_llm_provider = custom_llm_provider or "openai"
raise exception_type(
model=model,
custom_llm_provider=custom_llm_provider,
original_exception=e,
completion_kwargs=args,
extra_kwargs=kwargs,
)
@client
def image_variation(
image: FileTypes,
model: str = "dall-e-2", # set to dall-e-2 by default - like OpenAI.
n: int = 1,
response_format: Literal["url", "b64_json"] = "url",
size: Optional[str] = None,
user: Optional[str] = None,
**kwargs,
) -> ImageResponse:
# get non-default params
client = kwargs.get("client", None)
# get logging object
litellm_logging_obj = cast(LiteLLMLoggingObj, kwargs.get("litellm_logging_obj"))
# get the litellm params
litellm_params = get_litellm_params(**kwargs)
# get the custom llm provider
model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider(
model=model,
custom_llm_provider=litellm_params.get("custom_llm_provider", None),
api_base=litellm_params.get("api_base", None),
api_key=litellm_params.get("api_key", None),
)
# route to the correct provider w/ the params
try:
llm_provider = LlmProviders(custom_llm_provider)
image_variation_provider = LITELLM_IMAGE_VARIATION_PROVIDERS(llm_provider)
except ValueError:
raise ValueError(
f"Invalid image variation provider: {custom_llm_provider}. Supported providers are: {LITELLM_IMAGE_VARIATION_PROVIDERS}"
)
model_response = ImageResponse()
response: Optional[ImageResponse] = None
provider_config = ProviderConfigManager.get_provider_model_info(
model=model or "", # openai defaults to dall-e-2
provider=llm_provider,
)
if provider_config is None:
raise ValueError(
f"image variation provider has no known model info config - required for getting api keys, etc.: {custom_llm_provider}. Supported providers are: {LITELLM_IMAGE_VARIATION_PROVIDERS}"
)
api_key = provider_config.get_api_key(litellm_params.get("api_key", None))
api_base = provider_config.get_api_base(litellm_params.get("api_base", None))
if image_variation_provider == LITELLM_IMAGE_VARIATION_PROVIDERS.OPENAI:
if api_key is None:
raise ValueError("API key is required for OpenAI image variations")
if api_base is None:
raise ValueError("API base is required for OpenAI image variations")
response = openai_image_variations.image_variations(
model_response=model_response,
api_key=api_key,
api_base=api_base,
model=model,
image=image,
timeout=litellm_params.get("timeout", None),
custom_llm_provider=custom_llm_provider,
logging_obj=litellm_logging_obj,
optional_params={},
litellm_params=litellm_params,
)
elif image_variation_provider == LITELLM_IMAGE_VARIATION_PROVIDERS.TOPAZ:
if api_key is None:
raise ValueError("API key is required for Topaz image variations")
if api_base is None:
raise ValueError("API base is required for Topaz image variations")
response = base_llm_aiohttp_handler.image_variations(
model_response=model_response,
api_key=api_key,
api_base=api_base,
model=model,
image=image,
timeout=litellm_params.get("timeout", None),
custom_llm_provider=custom_llm_provider,
logging_obj=litellm_logging_obj,
optional_params={},
litellm_params=litellm_params,
client=client,
)
# return the response
if response is None:
raise ValueError(
f"Invalid image variation provider: {custom_llm_provider}. Supported providers are: {LITELLM_IMAGE_VARIATION_PROVIDERS}"
)
return response
##### Transcription #######################

13
litellm/model_prices_and_context_window_backup.json
View file

@ -3936,6 +3936,19 @@
"supports_vision": true,
"supports_tool_choice": true
},
"xai/grok-3": {
"max_tokens": 131072,
"max_input_tokens": 131072,
"max_output_tokens": 131072,
"input_cost_per_token": 0.000003,
"output_cost_per_token": 0.000015,
"litellm_provider": "xai",
"mode": "chat",
"supports_function_calling": true,
"supports_tool_choice": true,
"supports_response_schema": false,
"source": "https://x.ai/api#pricing"
},
"xai/grok-3-beta": {
"max_tokens": 131072,
"max_input_tokens": 131072,

31
litellm/types/images/main.py Normal file
View file

@ -0,0 +1,31 @@
from typing import Any, Dict, List, Literal, Optional, TypedDict, Union
from litellm.types.utils import FileTypes
class ImageEditOptionalRequestParams(TypedDict, total=False):
"""
TypedDict for Optional parameters supported by OpenAI's image edit API.
Params here: https://platform.openai.com/docs/api-reference/images/createEdit
"""
background: Optional[Literal["transparent", "opaque", "auto"]]
mask: Optional[str]
n: Optional[int]
quality: Optional[Literal["high", "medium", "low", "standard", "auto"]]
response_format: Optional[Literal["url", "b64_json"]]
size: Optional[str]
user: Optional[str]
class ImageEditRequestParams(ImageEditOptionalRequestParams, total=False):
"""
TypedDict for request parameters supported by OpenAI's image edit API.
Params here: https://platform.openai.com/docs/api-reference/images/createEdit
"""
image: FileTypes
prompt: str
model: Optional[str]

15
litellm/utils.py
View file

@ -229,6 +229,7 @@ from litellm.llms.base_llm.chat.transformation import BaseConfig
from litellm.llms.base_llm.completion.transformation import BaseTextCompletionConfig
from litellm.llms.base_llm.embedding.transformation import BaseEmbeddingConfig
from litellm.llms.base_llm.files.transformation import BaseFilesConfig
from litellm.llms.base_llm.image_edit.transformation import BaseImageEditConfig
from litellm.llms.base_llm.image_generation.transformation import (
BaseImageGenerationConfig,
)
@ -1967,6 +1968,7 @@ def supports_prompt_caching(
key="supports_prompt_caching",
)
def supports_computer_use(
model: str, custom_llm_provider: Optional[str] = None
) -> bool:
@ -6681,6 +6683,19 @@ class ProviderConfigManager:
return GeminiRealtimeConfig()
return None
@staticmethod
def get_provider_image_edit_config(
model: str,
provider: LlmProviders,
) -> Optional[BaseImageEditConfig]:
if LlmProviders.OPENAI == provider:
from litellm.llms.openai.image_edit.transformation import (
OpenAIImageEditConfig,
)
return OpenAIImageEditConfig()
return None
def get_end_user_id_for_cost_tracking(
litellm_params: dict,

13
model_prices_and_context_window.json
View file

@ -3936,6 +3936,19 @@
"supports_vision": true,
"supports_tool_choice": true
},
"xai/grok-3": {
"max_tokens": 131072,
"max_input_tokens": 131072,
"max_output_tokens": 131072,
"input_cost_per_token": 0.000003,
"output_cost_per_token": 0.000015,
"litellm_provider": "xai",
"mode": "chat",
"supports_function_calling": true,
"supports_tool_choice": true,
"supports_response_schema": false,
"source": "https://x.ai/api#pricing"
},
"xai/grok-3-beta": {
"max_tokens": 131072,
"max_input_tokens": 131072,

BIN
tests/image_gen_tests/ishaan_github.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 604 KiB

BIN
tests/image_gen_tests/litellm_site.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 757 KiB

BIN
tests/image_gen_tests/test_image_edit.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 2.6 MiB

57
tests/image_gen_tests/test_image_edits.py Normal file
View file

@ -0,0 +1,57 @@
import logging
import os
import sys
import traceback
import pytest
import base64
sys.path.insert(
0, os.path.abspath("../..")
) # Adds the parent directory to the system path
import litellm
from litellm.utils import ImageResponse
# Get the current directory of the file being run
pwd = os.path.dirname(os.path.realpath(__file__))
TEST_IMAGES = [
open(os.path.join(pwd, "ishaan_github.png"), "rb"),
open(os.path.join(pwd, "litellm_site.png"), "rb"),
]
@pytest.mark.parametrize("sync_mode", [True])
@pytest.mark.asyncio
async def test_openai_image_edit_litellm_sdk(sync_mode):
from litellm import image_edit, aimage_edit
litellm._turn_on_debug()
prompt = """
Create a studio ghibli style image that combines all the reference images. Make sure the person looks like a CTO.
"""
if sync_mode:
result = image_edit(
prompt=prompt,
model="gpt-image-1",
image=TEST_IMAGES,
)
else:
result = await aimage_edit(
prompt=prompt,
model="gpt-image-1",
image=TEST_IMAGES,
)
print("result from image edit", result)
# Validate the response meets expected schema
ImageResponse.model_validate(result)
if isinstance(result, ImageResponse):
image_base64 = result.data[0].b64_json
image_bytes = base64.b64decode(image_base64)
# Save the image to a file
with open("test_image_edit.png", "wb") as f:
f.write(image_bytes)

4
ui/litellm-dashboard/src/components/mcp_tools/columns.tsx
View file

@ -173,7 +173,9 @@ export function ToolTestPanel({
/>
</div>
</div>
) : (
) : schema.properties === undefined ? (
<p className="text-xs">None</p>
): (
Object.entries(schema.properties).map(([key, prop]) => (
<div key={key} className="mb-4">
<label className="block text-sm font-medium text-gray-700 mb-1">

405
ui/litellm-dashboard/src/components/organizations.tsx
View file

@ -147,211 +147,218 @@ const OrganizationsTable: React.FC<OrganizationsTableProps> = ({
}
return (
<TabGroup className="gap-2 p-8 h-[75vh] w-full mt-2">
<TabList className="flex justify-between mt-2 w-full items-center">
<div className="flex">
<Tab>Your Organizations</Tab>
</div>
<div className="flex items-center space-x-2">
{lastRefreshed && <Text>Last Refreshed: {lastRefreshed}</Text>}
<Icon
icon={RefreshIcon}
variant="shadow"
size="xs"
className="self-center"
onClick={handleRefreshClick}
/>
</div>
</TabList>
<TabPanels>
<TabPanel>
<Text>
Click on &ldquo;Organization ID&rdquo; to view organization details.
</Text>
<Grid numItems={1} className="gap-2 pt-2 pb-2 h-[75vh] w-full mt-2">
<Col numColSpan={1}>
<Card className="w-full mx-auto flex-auto overflow-y-auto max-h-[50vh]">
<Table>
<TableHead>
<TableRow>
<TableHeaderCell>Organization ID</TableHeaderCell>
<TableHeaderCell>Organization Name</TableHeaderCell>
<TableHeaderCell>Created</TableHeaderCell>
<TableHeaderCell>Spend (USD)</TableHeaderCell>
<TableHeaderCell>Budget (USD)</TableHeaderCell>
<TableHeaderCell>Models</TableHeaderCell>
<TableHeaderCell>TPM / RPM Limits</TableHeaderCell>
<TableHeaderCell>Info</TableHeaderCell>
<TableHeaderCell>Actions</TableHeaderCell>
</TableRow>
</TableHead>
<TableBody>
{organizations && organizations.length > 0
? organizations
.sort((a, b) => new Date(b.created_at).getTime() - new Date(a.created_at).getTime())
.map((org: Organization) => (
<TableRow key={org.organization_id}>
<TableCell>
<div className="overflow-hidden">
<Tooltip title={org.organization_id}>
<Button
size="xs"
variant="light"
className="font-mono text-blue-500 bg-blue-50 hover:bg-blue-100 text-xs font-normal px-2 py-0.5 text-left overflow-hidden truncate max-w-[200px]"
onClick={() => setSelectedOrgId(org.organization_id)}
>
{org.organization_id?.slice(0, 7)}...
</Button>
</Tooltip>
</div>
</TableCell>
<TableCell>{org.organization_alias}</TableCell>
<TableCell>
{org.created_at ? new Date(org.created_at).toLocaleDateString() : "N/A"}
</TableCell>
<TableCell>{org.spend}</TableCell>
<TableCell>
{org.litellm_budget_table?.max_budget !== null && org.litellm_budget_table?.max_budget !== undefined ? org.litellm_budget_table?.max_budget : "No limit"}
</TableCell>
<TableCell>
{Array.isArray(org.models) && (
<div className="flex flex-col">
{org.models.length === 0 ? (
<Badge size="xs" className="mb-1" color="red">
All Proxy Models
</Badge>
) : (
org.models.map((model, index) =>
model === "all-proxy-models" ? (
<Badge key={index} size="xs" className="mb-1" color="red">
All Proxy Models
</Badge>
) : (
<Badge key={index} size="xs" className="mb-1" color="blue">
{model.length > 30
? `${getModelDisplayName(model).slice(0, 30)}...`
: getModelDisplayName(model)}
</Badge>
)
)
)}
</div>
)}
</TableCell>
<TableCell>
<Text>
TPM: {org.litellm_budget_table?.tpm_limit ? org.litellm_budget_table?.tpm_limit : "Unlimited"}
<br />
RPM: {org.litellm_budget_table?.rpm_limit ? org.litellm_budget_table?.rpm_limit : "Unlimited"}
</Text>
</TableCell>
<TableCell>
<Text>{org.members?.length || 0} Members</Text>
</TableCell>
<TableCell>
{userRole === "Admin" && (
<>
<Icon
icon={PencilAltIcon}
size="sm"
onClick={() => {
setSelectedOrgId(org.organization_id);
setEditOrg(true);
}}
/>
<Icon
onClick={() => handleDelete(org.organization_id)}
icon={TrashIcon}
size="sm"
/>
</>
)}
</TableCell>
</TableRow>
))
: null}
</TableBody>
</Table>
</Card>
</Col>
{(userRole === "Admin" || userRole === "Org Admin") && (
<Col numColSpan={1}>
<Button
className="mx-auto"
onClick={() => setIsOrgModalVisible(true)}
<div className="w-full mx-4 h-[75vh]">
<Grid numItems={1} className="gap-2 p-8 w-full mt-2">
<Col numColSpan={1} className="flex flex-col gap-2">
{(userRole === "Admin" || userRole === "Org Admin") && (
<>
<Button
className="w-fit"
onClick={() => setIsOrgModalVisible(true)}
>
+ Create New Organization
</Button>
<Modal
title="Create Organization"
visible={isOrgModalVisible}
width={800}
footer={null}
onCancel={handleCancel}
>
<Form
form={form}
onFinish={handleCreate}
labelCol={{ span: 8 }}
wrapperCol={{ span: 16 }}
labelAlign="left"
>
+ Create New Organization
</Button>
<Modal
title="Create Organization"
visible={isOrgModalVisible}
width={800}
footer={null}
onCancel={handleCancel}
>
<Form
form={form}
onFinish={handleCreate}
labelCol={{ span: 8 }}
wrapperCol={{ span: 16 }}
labelAlign="left"
<Form.Item
label="Organization Name"
name="organization_alias"
rules={[{ required: true, message: "Please input an organization name" }]}
>
<Form.Item
label="Organization Name"
name="organization_alias"
rules={[{ required: true, message: "Please input an organization name" }]}
<TextInput placeholder="" />
</Form.Item>
<Form.Item label="Models" name="models">
<Select2
mode="multiple"
placeholder="Select models"
style={{ width: "100%" }}
>
<TextInput placeholder="" />
</Form.Item>
<Form.Item label="Models" name="models">
<Select2
mode="multiple"
placeholder="Select models"
style={{ width: "100%" }}
>
<Select2.Option key="all-proxy-models" value="all-proxy-models">
All Proxy Models
<Select2.Option key="all-proxy-models" value="all-proxy-models">
All Proxy Models
</Select2.Option>
{userModels && userModels.length > 0 && userModels.map((model) => (
<Select2.Option key={model} value={model}>
{getModelDisplayName(model)}
</Select2.Option>
{userModels && userModels.length > 0 && userModels.map((model) => (
<Select2.Option key={model} value={model}>
{getModelDisplayName(model)}
</Select2.Option>
))}
</Select2>
</Form.Item>
))}
</Select2>
</Form.Item>
<Form.Item label="Max Budget (USD)" name="max_budget">
<NumericalInput step={0.01} precision={2} width={200} />
</Form.Item>
<Form.Item label="Reset Budget" name="budget_duration">
<Select2 defaultValue={null} placeholder="n/a">
<Select2.Option value="24h">daily</Select2.Option>
<Select2.Option value="7d">weekly</Select2.Option>
<Select2.Option value="30d">monthly</Select2.Option>
</Select2>
</Form.Item>
<Form.Item label="Tokens per minute Limit (TPM)" name="tpm_limit">
<NumericalInput step={1} width={400} />
</Form.Item>
<Form.Item label="Requests per minute Limit (RPM)" name="rpm_limit">
<NumericalInput step={1} width={400} />
</Form.Item>
<Form.Item label="Max Budget (USD)" name="max_budget">
<NumericalInput step={0.01} precision={2} width={200} />
</Form.Item>
<Form.Item label="Reset Budget" name="budget_duration">
<Select2 defaultValue={null} placeholder="n/a">
<Select2.Option value="24h">daily</Select2.Option>
<Select2.Option value="7d">weekly</Select2.Option>
<Select2.Option value="30d">monthly</Select2.Option>
</Select2>
</Form.Item>
<Form.Item label="Tokens per minute Limit (TPM)" name="tpm_limit">
<NumericalInput step={1} width={400} />
</Form.Item>
<Form.Item label="Requests per minute Limit (RPM)" name="rpm_limit">
<NumericalInput step={1} width={400} />
</Form.Item>
<Form.Item label="Metadata" name="metadata">
<Input.TextArea rows={4} />
</Form.Item>
<Form.Item label="Metadata" name="metadata">
<Input.TextArea rows={4} />
</Form.Item>
<div style={{ textAlign: "right", marginTop: "10px" }}>
<Button type="submit">Create Organization</Button>
</div>
</Form>
</Modal>
</Col>
)}
</Grid>
</TabPanel>
</TabPanels>
<div style={{ textAlign: "right", marginTop: "10px" }}>
<Button type="submit">Create Organization</Button>
</div>
</Form>
</Modal>
</>
)}
<TabGroup className="gap-2 h-[75vh] w-full">
<TabList className="flex justify-between mt-2 w-full items-center">
<div className="flex">
<Tab>Your Organizations</Tab>
</div>
<div className="flex items-center space-x-2">
{lastRefreshed && <Text>Last Refreshed: {lastRefreshed}</Text>}
<Icon
icon={RefreshIcon}
variant="shadow"
size="xs"
className="self-center"
onClick={handleRefreshClick}
/>
</div>
</TabList>
<TabPanels>
<TabPanel>
<Text>
Click on &ldquo;Organization ID&rdquo; to view organization details.
</Text>
<Grid numItems={1} className="gap-2 pt-2 pb-2 h-[75vh] w-full mt-2">
<Col numColSpan={1}>
<Card className="w-full mx-auto flex-auto overflow-y-auto max-h-[50vh]">
<Table>
<TableHead>
<TableRow>
<TableHeaderCell>Organization ID</TableHeaderCell>
<TableHeaderCell>Organization Name</TableHeaderCell>
<TableHeaderCell>Created</TableHeaderCell>
<TableHeaderCell>Spend (USD)</TableHeaderCell>
<TableHeaderCell>Budget (USD)</TableHeaderCell>
<TableHeaderCell>Models</TableHeaderCell>
<TableHeaderCell>TPM / RPM Limits</TableHeaderCell>
<TableHeaderCell>Info</TableHeaderCell>
<TableHeaderCell>Actions</TableHeaderCell>
</TableRow>
</TableHead>
<TableBody>
{organizations && organizations.length > 0
? organizations
.sort((a, b) => new Date(b.created_at).getTime() - new Date(a.created_at).getTime())
.map((org: Organization) => (
<TableRow key={org.organization_id}>
<TableCell>
<div className="overflow-hidden">
<Tooltip title={org.organization_id}>
<Button
size="xs"
variant="light"
className="font-mono text-blue-500 bg-blue-50 hover:bg-blue-100 text-xs font-normal px-2 py-0.5 text-left overflow-hidden truncate max-w-[200px]"
onClick={() => setSelectedOrgId(org.organization_id)}
>
{org.organization_id?.slice(0, 7)}...
</Button>
</Tooltip>
</div>
</TableCell>
<TableCell>{org.organization_alias}</TableCell>
<TableCell>
{org.created_at ? new Date(org.created_at).toLocaleDateString() : "N/A"}
</TableCell>
<TableCell>{org.spend}</TableCell>
<TableCell>
{org.litellm_budget_table?.max_budget !== null && org.litellm_budget_table?.max_budget !== undefined ? org.litellm_budget_table?.max_budget : "No limit"}
</TableCell>
<TableCell>
{Array.isArray(org.models) && (
<div className="flex flex-col">
{org.models.length === 0 ? (
<Badge size="xs" className="mb-1" color="red">
All Proxy Models
</Badge>
) : (
org.models.map((model, index) =>
model === "all-proxy-models" ? (
<Badge key={index} size="xs" className="mb-1" color="red">
All Proxy Models
</Badge>
) : (
<Badge key={index} size="xs" className="mb-1" color="blue">
{model.length > 30
? `${getModelDisplayName(model).slice(0, 30)}...`
: getModelDisplayName(model)}
</Badge>
)
)
)}
</div>
)}
</TableCell>
<TableCell>
<Text>
TPM: {org.litellm_budget_table?.tpm_limit ? org.litellm_budget_table?.tpm_limit : "Unlimited"}
<br />
RPM: {org.litellm_budget_table?.rpm_limit ? org.litellm_budget_table?.rpm_limit : "Unlimited"}
</Text>
</TableCell>
<TableCell>
<Text>{org.members?.length || 0} Members</Text>
</TableCell>
<TableCell>
{userRole === "Admin" && (
<>
<Icon
icon={PencilAltIcon}
size="sm"
onClick={() => {
setSelectedOrgId(org.organization_id);
setEditOrg(true);
}}
/>
<Icon
onClick={() => handleDelete(org.organization_id)}
icon={TrashIcon}
size="sm"
/>
</>
)}
</TableCell>
</TableRow>
))
: null}
</TableBody>
</Table>
</Card>
</Col>
</Grid>
</TabPanel>
</TabPanels>
</TabGroup>
</Col>
</Grid>
{isDeleteModalOpen ? (
<div className="fixed z-10 inset-0 overflow-y-auto">
@ -389,7 +396,7 @@ const OrganizationsTable: React.FC<OrganizationsTableProps> = ({
</div>
</div>
) : <></>}
</TabGroup>
</div>
);
};

907
ui/litellm-dashboard/src/components/teams.tsx
View file

@ -423,221 +423,230 @@ const Teams: React.FC<TeamProps> = ({
return (
<div className="w-full mx-4 h-[75vh]">
{selectedTeamId ? (
<TeamInfoView
teamId={selectedTeamId}
onUpdate={(data) => {
setTeams(teams => {
if (teams == null) {
return teams;
}
return teams.map(team => {
if (data.team_id === team.team_id) {
return updateExistingKeys(team, data)
}
<Grid numItems={1} className="gap-2 p-8 w-full mt-2">
<Col numColSpan={1} className="flex flex-col gap-2">
<Button
className="w-fit"
onClick={() => setIsTeamModalVisible(true)}
>
+ Create New Team
</Button>
{selectedTeamId ? (
<TeamInfoView
teamId={selectedTeamId}
onUpdate={(data) => {
setTeams(teams => {
if (teams == null) {
return teams;
}
return team
})
})
return teams.map(team => {
if (data.team_id === team.team_id) {
return updateExistingKeys(team, data)
}
return team
})
})
}}
onClose={() => {
setSelectedTeamId(null);
setEditTeam(false);
}}
accessToken={accessToken}
is_team_admin={is_team_admin(teams?.find((team) => team.team_id === selectedTeamId))}
is_proxy_admin={userRole == "Admin"}
userModels={userModels}
editTeam={editTeam}
/>
) : (
<TabGroup className="gap-2 p-8 h-[75vh] w-full mt-2">
<TabList className="flex justify-between mt-2 w-full items-center">
<div className="flex">
<Tab>Your Teams</Tab>
<Tab>Available Teams</Tab>
{isAdminRole(userRole || "") && <Tab>Default Team Settings</Tab>}
</div>
<div className="flex items-center space-x-2">
{lastRefreshed && <Text>Last Refreshed: {lastRefreshed}</Text>}
<Icon
icon={RefreshIcon} // Modify as necessary for correct icon name
variant="shadow"
size="xs"
className="self-center"
onClick={handleRefreshClick}
/>
</div>
</TabList>
<TabPanels>
<TabPanel>
<Text>
Click on &ldquo;Team ID&rdquo; to view team details <b>and</b> manage team members.
</Text>
<Grid numItems={1} className="gap-2 pt-2 pb-2 h-[75vh] w-full mt-2">
<Col numColSpan={1}>
<Card className="w-full mx-auto flex-auto overflow-hidden overflow-y-auto max-h-[50vh]">
<div className="border-b px-6 py-4">
<div className="flex flex-col space-y-4">
{/* Search and Filter Controls */}
<div className="flex flex-wrap items-center gap-3">
{/* Team Alias Search */}
<div className="relative w-64">
<input
type="text"
placeholder="Search by Team Name..."
className="w-full px-3 py-2 pl-8 border rounded-md text-sm focus:outline-none focus:ring-2 focus:ring-blue-500 focus:border-blue-500"
value={filters.team_alias}
onChange={(e) => handleFilterChange('team_alias', e.target.value)}
/>
<svg
className="absolute left-2.5 top-2.5 h-4 w-4 text-gray-500"
fill="none"
stroke="currentColor"
viewBox="0 0 24 24"
>
<path
strokeLinecap="round"
strokeLinejoin="round"
strokeWidth={2}
d="M21 21l-6-6m2-5a7 7 0 11-14 0 7 7 0 0114 0z"
/>
</svg>
</div>
{/* Filter Button */}
<button
className={`px-3 py-2 text-sm border rounded-md hover:bg-gray-50 flex items-center gap-2 ${showFilters ? 'bg-gray-100' : ''}`}
onClick={() => setShowFilters(!showFilters)}
>
<svg
className="w-4 h-4"
fill="none"
stroke="currentColor"
viewBox="0 0 24 24"
>
<path
strokeLinecap="round"
strokeLinejoin="round"
strokeWidth={2}
d="M3 4a1 1 0 011-1h16a1 1 0 011 1v2.586a1 1 0 01-.293.707l-6.414 6.414a1 1 0 00-.293.707V17l-4 4v-6.586a1 1 0 00-.293-.707L3.293 7.293A1 1 0 013 6.586V4z"
/>
</svg>
Filters
{(filters.team_id || filters.team_alias || filters.organization_id) && (
<span className="w-2 h-2 rounded-full bg-blue-500"></span>
)}
</button>
{/* Reset Filters Button */}
<button
className="px-3 py-2 text-sm border rounded-md hover:bg-gray-50 flex items-center gap-2"
onClick={handleFilterReset}
>
<svg
className="w-4 h-4"
fill="none"
stroke="currentColor"
viewBox="0 0 24 24"
>
<path
strokeLinecap="round"
strokeLinejoin="round"
strokeWidth={2}
d="M4 4v5h.582m15.356 2A8.001 8.001 0 004.582 9m0 0H9m11 11v-5h-.581m0 0a8.003 8.003 0 01-15.357-2m15.357 2H15"
/>
</svg>
Reset Filters
</button>
}}
onClose={() => {
setSelectedTeamId(null);
setEditTeam(false);
}}
accessToken={accessToken}
is_team_admin={is_team_admin(teams?.find((team) => team.team_id === selectedTeamId))}
is_proxy_admin={userRole == "Admin"}
userModels={userModels}
editTeam={editTeam}
/>
) : (
<TabGroup className="gap-2 h-[75vh] w-full">
<TabList className="flex justify-between mt-2 w-full items-center">
<div className="flex">
<Tab>Your Teams</Tab>
<Tab>Available Teams</Tab>
{isAdminRole(userRole || "") && <Tab>Default Team Settings</Tab>}
</div>
<div className="flex items-center space-x-2">
{lastRefreshed && <Text>Last Refreshed: {lastRefreshed}</Text>}
<Icon
icon={RefreshIcon} // Modify as necessary for correct icon name
variant="shadow"
size="xs"
className="self-center"
onClick={handleRefreshClick}
/>
</div>
</TabList>
<TabPanels>
<TabPanel>
<Text>
Click on &ldquo;Team ID&rdquo; to view team details <b>and</b> manage team members.
</Text>
<Grid numItems={1} className="gap-2 pt-2 pb-2 h-[75vh] w-full mt-2">
<Col numColSpan={1}>
<Card className="w-full mx-auto flex-auto overflow-hidden overflow-y-auto max-h-[50vh]">
<div className="border-b px-6 py-4">
<div className="flex flex-col space-y-4">
{/* Search and Filter Controls */}
<div className="flex flex-wrap items-center gap-3">
{/* Team Alias Search */}
<div className="relative w-64">
<input
type="text"
placeholder="Search by Team Name..."
className="w-full px-3 py-2 pl-8 border rounded-md text-sm focus:outline-none focus:ring-2 focus:ring-blue-500 focus:border-blue-500"
value={filters.team_alias}
onChange={(e) => handleFilterChange('team_alias', e.target.value)}
/>
<svg
className="absolute left-2.5 top-2.5 h-4 w-4 text-gray-500"
fill="none"
stroke="currentColor"
viewBox="0 0 24 24"
>
<path
strokeLinecap="round"
strokeLinejoin="round"
strokeWidth={2}
d="M21 21l-6-6m2-5a7 7 0 11-14 0 7 7 0 0114 0z"
/>
</svg>
</div>
{/* Additional Filters */}
{showFilters && (
<div className="flex flex-wrap items-center gap-3 mt-3">
{/* Team ID Search */}
<div className="relative w-64">
<input
type="text"
placeholder="Enter Team ID"
className="w-full px-3 py-2 pl-8 border rounded-md text-sm focus:outline-none focus:ring-2 focus:ring-blue-500 focus:border-blue-500"
value={filters.team_id}
onChange={(e) => handleFilterChange('team_id', e.target.value)}
/>
<svg
className="absolute left-2.5 top-2.5 h-4 w-4 text-gray-500"
fill="none"
stroke="currentColor"
viewBox="0 0 24 24"
>
<path
strokeLinecap="round"
strokeLinejoin="round"
strokeWidth={2}
d="M5.121 17.804A13.937 13.937 0 0112 16c2.5 0 4.847.655 6.879 1.804M15 10a3 3 0 11-6 0 3 3 0 016 0zm6 2a9 9 0 11-18 0 9 9 0 0118 0z"
/>
</svg>
</div>
{/* Filter Button */}
<button
className={`px-3 py-2 text-sm border rounded-md hover:bg-gray-50 flex items-center gap-2 ${showFilters ? 'bg-gray-100' : ''}`}
onClick={() => setShowFilters(!showFilters)}
>
<svg
className="w-4 h-4"
fill="none"
stroke="currentColor"
viewBox="0 0 24 24"
>
<path
strokeLinecap="round"
strokeLinejoin="round"
strokeWidth={2}
d="M3 4a1 1 0 011-1h16a1 1 0 011 1v2.586a1 1 0 01-.293.707l-6.414 6.414a1 1 0 00-.293.707V17l-4 4v-6.586a1 1 0 00-.293-.707L3.293 7.293A1 1 0 013 6.586V4z"
/>
</svg>
Filters
{(filters.team_id || filters.team_alias || filters.organization_id) && (
<span className="w-2 h-2 rounded-full bg-blue-500"></span>
)}
</button>
{/* Organization Dropdown */}
<div className="w-64">
<Select
value={filters.organization_id || ""}
onValueChange={(value) => handleFilterChange('organization_id', value)}
placeholder="Select Organization"
>
{organizations?.map((org) => (
<SelectItem key={org.organization_id} value={org.organization_id || ""}>
{org.organization_alias || org.organization_id}
</SelectItem>
))}
</Select>
{/* Reset Filters Button */}
<button
className="px-3 py-2 text-sm border rounded-md hover:bg-gray-50 flex items-center gap-2"
onClick={handleFilterReset}
>
<svg
className="w-4 h-4"
fill="none"
stroke="currentColor"
viewBox="0 0 24 24"
>
<path
strokeLinecap="round"
strokeLinejoin="round"
strokeWidth={2}
d="M4 4v5h.582m15.356 2A8.001 8.001 0 004.582 9m0 0H9m11 11v-5h-.581m0 0a8.003 8.003 0 01-15.357-2m15.357 2H15"
/>
</svg>
Reset Filters
</button>
</div>
{/* Additional Filters */}
{showFilters && (
<div className="flex flex-wrap items-center gap-3 mt-3">
{/* Team ID Search */}
<div className="relative w-64">
<input
type="text"
placeholder="Enter Team ID"
className="w-full px-3 py-2 pl-8 border rounded-md text-sm focus:outline-none focus:ring-2 focus:ring-blue-500 focus:border-blue-500"
value={filters.team_id}
onChange={(e) => handleFilterChange('team_id', e.target.value)}
/>
<svg
className="absolute left-2.5 top-2.5 h-4 w-4 text-gray-500"
fill="none"
stroke="currentColor"
viewBox="0 0 24 24"
>
<path
strokeLinecap="round"
strokeLinejoin="round"
strokeWidth={2}
d="M5.121 17.804A13.937 13.937 0 0112 16c2.5 0 4.847.655 6.879 1.804M15 10a3 3 0 11-6 0 3 3 0 016 0zm6 2a9 9 0 11-18 0 9 9 0 0118 0z"
/>
</svg>
</div>
{/* Organization Dropdown */}
<div className="w-64">
<Select
value={filters.organization_id || ""}
onValueChange={(value) => handleFilterChange('organization_id', value)}
placeholder="Select Organization"
>
{organizations?.map((org) => (
<SelectItem key={org.organization_id} value={org.organization_id || ""}>
{org.organization_alias || org.organization_id}
</SelectItem>
))}
</Select>
</div>
</div>
)}
</div>
</div>
)}
</div>
</div>
<Table>
<TableHead>
<TableRow>
<TableHeaderCell>Team Name</TableHeaderCell>
<TableHeaderCell>Team ID</TableHeaderCell>
<TableHeaderCell>Created</TableHeaderCell>
<TableHeaderCell>Spend (USD)</TableHeaderCell>
<TableHeaderCell>Budget (USD)</TableHeaderCell>
<TableHeaderCell>Models</TableHeaderCell>
<TableHeaderCell>Organization</TableHeaderCell>
<TableHeaderCell>Info</TableHeaderCell>
</TableRow>
</TableHead>
<Table>
<TableHead>
<TableRow>
<TableHeaderCell>Team Name</TableHeaderCell>
<TableHeaderCell>Team ID</TableHeaderCell>
<TableHeaderCell>Created</TableHeaderCell>
<TableHeaderCell>Spend (USD)</TableHeaderCell>
<TableHeaderCell>Budget (USD)</TableHeaderCell>
<TableHeaderCell>Models</TableHeaderCell>
<TableHeaderCell>Organization</TableHeaderCell>
<TableHeaderCell>Info</TableHeaderCell>
</TableRow>
</TableHead>
<TableBody>
{teams && teams.length > 0
? teams
.filter((team) => {
if (!currentOrg) return true;
return team.organization_id === currentOrg.organization_id;
})
.sort((a, b) => new Date(b.created_at).getTime() - new Date(a.created_at).getTime())
.map((team: any) => (
<TableRow key={team.team_id}>
<TableCell
style={{
maxWidth: "4px",
whiteSpace: "pre-wrap",
overflow: "hidden",
}}
>
{team["team_alias"]}
</TableCell>
<TableCell>
<div className="overflow-hidden">
<Tooltip title={team.team_id}>
<Button
size="xs"
variant="light"
className="font-mono text-blue-500 bg-blue-50 hover:bg-blue-100 text-xs font-normal px-2 py-0.5 text-left overflow-hidden truncate max-w-[200px]"
<TableBody>
{teams && teams.length > 0
? teams
.filter((team) => {
if (!currentOrg) return true;
return team.organization_id === currentOrg.organization_id;
})
.sort((a, b) => new Date(b.created_at).getTime() - new Date(a.created_at).getTime())
.map((team: any) => (
<TableRow key={team.team_id}>
<TableCell
style={{
maxWidth: "4px",
whiteSpace: "pre-wrap",
overflow: "hidden",
}}
>
{team["team_alias"]}
</TableCell>
<TableCell>
<div className="overflow-hidden">
<Tooltip title={team.team_id}>
<Button
size="xs"
variant="light"
className="font-mono text-blue-500 bg-blue-50 hover:bg-blue-100 text-xs font-normal px-2 py-0.5 text-left overflow-hidden truncate max-w-[200px]"
onClick={() => {
// Add click handler
@ -782,258 +791,254 @@ const Teams: React.FC<TeamProps> = ({
<div className="absolute inset-0 bg-gray-500 opacity-75"></div>
</div>
{/* Modal Panel */}
<span
className="hidden sm:inline-block sm:align-middle sm:h-screen"
aria-hidden="true"
>
&#8203;
</span>
{/* Modal Panel */}
<span
className="hidden sm:inline-block sm:align-middle sm:h-screen"
aria-hidden="true"
>
&#8203;
</span>
{/* Confirmation Modal Content */}
<div className="inline-block align-bottom bg-white rounded-lg text-left overflow-hidden shadow-xl transform transition-all sm:my-8 sm:align-middle sm:max-w-lg sm:w-full">
<div className="bg-white px-4 pt-5 pb-4 sm:p-6 sm:pb-4">
<div className="sm:flex sm:items-start">
<div className="mt-3 text-center sm:mt-0 sm:ml-4 sm:text-left">
<h3 className="text-lg leading-6 font-medium text-gray-900">
Delete Team
</h3>
<div className="mt-2">
<p className="text-sm text-gray-500">
Are you sure you want to delete this team ?
</p>
{/* Confirmation Modal Content */}
<div className="inline-block align-bottom bg-white rounded-lg text-left overflow-hidden shadow-xl transform transition-all sm:my-8 sm:align-middle sm:max-w-lg sm:w-full">
<div className="bg-white px-4 pt-5 pb-4 sm:p-6 sm:pb-4">
<div className="sm:flex sm:items-start">
<div className="mt-3 text-center sm:mt-0 sm:ml-4 sm:text-left">
<h3 className="text-lg leading-6 font-medium text-gray-900">
Delete Team
</h3>
<div className="mt-2">
<p className="text-sm text-gray-500">
Are you sure you want to delete this team ?
</p>
</div>
</div>
</div>
</div>
<div className="bg-gray-50 px-4 py-3 sm:px-6 sm:flex sm:flex-row-reverse">
<Button
onClick={confirmDelete}
color="red"
className="ml-2"
>
Delete
</Button>
<Button onClick={cancelDelete}>Cancel</Button>
</div>
</div>
</div>
</div>
<div className="bg-gray-50 px-4 py-3 sm:px-6 sm:flex sm:flex-row-reverse">
<Button
onClick={confirmDelete}
color="red"
className="ml-2"
)}
</Card>
</Col>
{userRole == "Admin" || userRole == "Org Admin"? (
<Col numColSpan={1}>
<Modal
title="Create Team"
visible={isTeamModalVisible}
width={800}
footer={null}
onOk={handleOk}
onCancel={handleCancel}
>
<Form
form={form}
onFinish={handleCreate}
labelCol={{ span: 8 }}
wrapperCol={{ span: 16 }}
labelAlign="left"
>
<>
<Form.Item
label="Team Name"
name="team_alias"
rules={[
{ required: true, message: "Please input a team name" },
]}
>
Delete
</Button>
<Button onClick={cancelDelete}>Cancel</Button>
</div>
</div>
</div>
</div>
)}
</Card>
</Col>
{userRole == "Admin" || userRole == "Org Admin"? (
<Col numColSpan={1}>
<Button
className="mx-auto"
onClick={() => setIsTeamModalVisible(true)}
>
+ Create New Team
</Button>
<Modal
title="Create Team"
visible={isTeamModalVisible}
width={800}
footer={null}
onOk={handleOk}
onCancel={handleCancel}
>
<Form
form={form}
onFinish={handleCreate}
labelCol={{ span: 8 }}
wrapperCol={{ span: 16 }}
labelAlign="left"
>
<>
<Form.Item
label="Team Name"
name="team_alias"
rules={[
{ required: true, message: "Please input a team name" },
]}
>
<TextInput placeholder="" />
</Form.Item>
<Form.Item
label={
<span>
Organization{' '}
<Tooltip title={
<span>
Organizations can have multiple teams. Learn more about{' '}
<a
href="https://docs.litellm.ai/docs/proxy/user_management_heirarchy"
target="_blank"
rel="noopener noreferrer"
style={{ color: '#1890ff', textDecoration: 'underline' }}
onClick={(e) => e.stopPropagation()}
>
user management hierarchy
</a>
</span>
}>
<InfoCircleOutlined style={{ marginLeft: '4px' }} />
</Tooltip>
</span>
}
name="organization_id"
initialValue={currentOrg ? currentOrg.organization_id : null}
className="mt-8"
>
<Select2
showSearch
allowClear
placeholder="Search or select an Organization"
onChange={(value) => {
form.setFieldValue('organization_id', value);
setCurrentOrgForCreateTeam(organizations?.find((org) => org.organization_id === value) || null);
}}
filterOption={(input, option) => {
if (!option) return false;
const optionValue = option.children?.toString() || '';
return optionValue.toLowerCase().includes(input.toLowerCase());
}}
optionFilterProp="children"
>
{organizations?.map((org) => (
<Select2.Option key={org.organization_id} value={org.organization_id}>
<span className="font-medium">{org.organization_alias}</span>{" "}
<span className="text-gray-500">({org.organization_id})</span>
</Select2.Option>
))}
</Select2>
</Form.Item>
<Form.Item label={
<span>
Models{' '}
<Tooltip title="These are the models that your selected team has access to">
<InfoCircleOutlined style={{ marginLeft: '4px' }} />
</Tooltip>
</span>
} name="models">
<Select2
mode="multiple"
placeholder="Select models"
style={{ width: "100%" }}
>
<Select2.Option
key="all-proxy-models"
value="all-proxy-models"
>
All Proxy Models
</Select2.Option>
{modelsToPick.map((model) => (
<Select2.Option key={model} value={model}>
{getModelDisplayName(model)}
</Select2.Option>
))}
</Select2>
</Form.Item>
<Form.Item label="Max Budget (USD)" name="max_budget">
<NumericalInput step={0.01} precision={2} width={200} />
</Form.Item>
<Form.Item
className="mt-8"
label="Reset Budget"
name="budget_duration"
>
<Select2 defaultValue={null} placeholder="n/a">
<Select2.Option value="24h">daily</Select2.Option>
<Select2.Option value="7d">weekly</Select2.Option>
<Select2.Option value="30d">monthly</Select2.Option>
</Select2>
</Form.Item>
<Form.Item
label="Tokens per minute Limit (TPM)"
name="tpm_limit"
>
<NumericalInput step={1} width={400} />
</Form.Item>
<Form.Item
label="Requests per minute Limit (RPM)"
name="rpm_limit"
>
<NumericalInput step={1} width={400} />
</Form.Item>
<Accordion className="mt-20 mb-8">
<AccordionHeader>
<b>Additional Settings</b>
</AccordionHeader>
<AccordionBody>
<Form.Item
label="Team ID"
name="team_id"
help="ID of the team you want to create. If not provided, it will be generated automatically."
>
<TextInput
onChange={(e) => {
e.target.value = e.target.value.trim();
}}
/>
</Form.Item>
<Form.Item label="Metadata" name="metadata" help="Additional team metadata. Enter metadata as JSON object.">
<Input.TextArea rows={4} />
</Form.Item>
<Form.Item
label={
<span>
Guardrails{' '}
<Tooltip title="Setup your first guardrail">
<a
href="https://docs.litellm.ai/docs/proxy/guardrails/quick_start"
target="_blank"
rel="noopener noreferrer"
onClick={(e) => e.stopPropagation()}
>
<TextInput placeholder="" />
</Form.Item>
<Form.Item
label={
<span>
Organization{' '}
<Tooltip title={
<span>
Organizations can have multiple teams. Learn more about{' '}
<a
href="https://docs.litellm.ai/docs/proxy/user_management_heirarchy"
target="_blank"
rel="noopener noreferrer"
style={{ color: '#1890ff', textDecoration: 'underline' }}
onClick={(e) => e.stopPropagation()}
>
user management hierarchy
</a>
</span>
}>
<InfoCircleOutlined style={{ marginLeft: '4px' }} />
</a>
</Tooltip>
</span>
}
name="guardrails"
className="mt-8"
help="Select existing guardrails or enter new ones"
>
<Select2
mode="tags"
style={{ width: '100%' }}
placeholder="Select or enter guardrails"
options={guardrailsList.map(name => ({ value: name, label: name }))}
/>
</Form.Item>
</AccordionBody>
</Accordion>
</>
<div style={{ textAlign: "right", marginTop: "10px" }}>
<Button2 htmlType="submit">Create Team</Button2>
</div>
</Form>
</Modal>
</Col>
) : null}
</Grid>
</TabPanel>
<TabPanel>
<AvailableTeamsPanel
accessToken={accessToken}
userID={userID}
/>
</TabPanel>
{isAdminRole(userRole || "") && (
<TabPanel>
<TeamSSOSettings
accessToken={accessToken}
userID={userID || ""}
userRole={userRole || ""}
/>
</TabPanel>
)}
</TabPanels>
</Tooltip>
</span>
}
name="organization_id"
initialValue={currentOrg ? currentOrg.organization_id : null}
className="mt-8"
>
<Select2
showSearch
allowClear
placeholder="Search or select an Organization"
onChange={(value) => {
form.setFieldValue('organization_id', value);
setCurrentOrgForCreateTeam(organizations?.find((org) => org.organization_id === value) || null);
}}
filterOption={(input, option) => {
if (!option) return false;
const optionValue = option.children?.toString() || '';
return optionValue.toLowerCase().includes(input.toLowerCase());
}}
optionFilterProp="children"
>
{organizations?.map((org) => (
<Select2.Option key={org.organization_id} value={org.organization_id}>
<span className="font-medium">{org.organization_alias}</span>{" "}
<span className="text-gray-500">({org.organization_id})</span>
</Select2.Option>
))}
</Select2>
</Form.Item>
<Form.Item label={
<span>
Models{' '}
<Tooltip title="These are the models that your selected team has access to">
<InfoCircleOutlined style={{ marginLeft: '4px' }} />
</Tooltip>
</span>
} name="models">
<Select2
mode="multiple"
placeholder="Select models"
style={{ width: "100%" }}
>
<Select2.Option
key="all-proxy-models"
value="all-proxy-models"
>
All Proxy Models
</Select2.Option>
{modelsToPick.map((model) => (
<Select2.Option key={model} value={model}>
{getModelDisplayName(model)}
</Select2.Option>
))}
</Select2>
</Form.Item>
</TabGroup>)}
<Form.Item label="Max Budget (USD)" name="max_budget">
<NumericalInput step={0.01} precision={2} width={200} />
</Form.Item>
<Form.Item
className="mt-8"
label="Reset Budget"
name="budget_duration"
>
<Select2 defaultValue={null} placeholder="n/a">
<Select2.Option value="24h">daily</Select2.Option>
<Select2.Option value="7d">weekly</Select2.Option>
<Select2.Option value="30d">monthly</Select2.Option>
</Select2>
</Form.Item>
<Form.Item
label="Tokens per minute Limit (TPM)"
name="tpm_limit"
>
<NumericalInput step={1} width={400} />
</Form.Item>
<Form.Item
label="Requests per minute Limit (RPM)"
name="rpm_limit"
>
<NumericalInput step={1} width={400} />
</Form.Item>
<Accordion className="mt-20 mb-8">
<AccordionHeader>
<b>Additional Settings</b>
</AccordionHeader>
<AccordionBody>
<Form.Item
label="Team ID"
name="team_id"
help="ID of the team you want to create. If not provided, it will be generated automatically."
>
<TextInput
onChange={(e) => {
e.target.value = e.target.value.trim();
}}
/>
</Form.Item>
<Form.Item label="Metadata" name="metadata" help="Additional team metadata. Enter metadata as JSON object.">
<Input.TextArea rows={4} />
</Form.Item>
<Form.Item
label={
<span>
Guardrails{' '}
<Tooltip title="Setup your first guardrail">
<a
href="https://docs.litellm.ai/docs/proxy/guardrails/quick_start"
target="_blank"
rel="noopener noreferrer"
onClick={(e) => e.stopPropagation()}
>
<InfoCircleOutlined style={{ marginLeft: '4px' }} />
</a>
</Tooltip>
</span>
}
name="guardrails"
className="mt-8"
help="Select existing guardrails or enter new ones"
>
<Select2
mode="tags"
style={{ width: '100%' }}
placeholder="Select or enter guardrails"
options={guardrailsList.map(name => ({ value: name, label: name }))}
/>
</Form.Item>
</AccordionBody>
</Accordion>
</>
<div style={{ textAlign: "right", marginTop: "10px" }}>
<Button2 htmlType="submit">Create Team</Button2>
</div>
</Form>
</Modal>
</Col>
) : null}
</Grid>
</TabPanel>
<TabPanel>
<AvailableTeamsPanel
accessToken={accessToken}
userID={userID}
/>
</TabPanel>
{isAdminRole(userRole || "") && (
<TabPanel>
<TeamSSOSettings
accessToken={accessToken}
userID={userID || ""}
userRole={userRole || ""}
/>
</TabPanel>
)}
</TabPanels>
</TabGroup>)}
</Col>
</Grid>
</div>
);
};