scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-07 02:59:05 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 103

Incorporte review comments

Browse source
This commit is contained in:
Muhammad Sannan Nasir 2025-07-11 22:34:57 +05:00
parent 44ba3971d6
commit 37bd51bd54
12 changed files with 41 additions and 351 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
docs/my-website/Dockerfile
View file

@ -4,6 +4,6 @@ COPY . /app
WORKDIR /app
RUN pip install -r requirements.txt
EXPOSE 3000
EXPOSE $PORT
CMD litellm --host 0.0.0.0 --port 3000 --workers 10 --config config.yaml
CMD litellm --host 0.0.0.0 --port $PORT --workers 10 --config config.yaml

4
docs/my-website/docs/providers/digitalocean.md → docs/my-website/docs/providers/gradient_ai.md
View file

@ -1,11 +1,11 @@
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
# GradientAI GradientAI
# GradientAI
https://digitalocean.com/products/genai
LiteLLM provides native support for GradientAI GenAI models.
LiteLLM provides native support for GradientAI models.
To use a GradientAI model, specify it as `gradient_ai/<model-name>` in your LiteLLM requests.

16
docs/my-website/docs/proxy/guardrails/secret_detection.md
View file

@ -1,9 +1,9 @@
# ✨ Secret Detection/Redaction (Enterprise-only)
❓ Use this to REDACT API Keys, Secrets sent in requests to an LLM.
❓ Use this to REDACT API Keys, Secrets sent in requests to an LLM.
Example if you want to redact the value of `OPENAI_API_KEY` in the following request
#### Incoming Request
#### Incoming Request
```json
{
@ -31,13 +31,13 @@ Example if you want to redact the value of `OPENAI_API_KEY` in the following req
**Usage**
**Step 1** Add this to your config.yaml
**Step 1** Add this to your config.yaml
```yaml
guardrails:
- guardrail_name: "my-custom-name"
litellm_params:
guardrail: "hide-secrets" # supported values: "aporia", "lakera", ..
guardrail: "hide-secrets" # supported values: "aporia", "lakera", ..
mode: "pre_call"
```
@ -125,7 +125,7 @@ guardrails:
**2. Start proxy**
Run with `--detailed_debug` for more detailed logs. Use in dev only.
Run with `--detailed_debug` for more detailed logs. Use in dev only.
```bash
litellm --config /path/to/config.yaml --detailed_debug
@ -157,7 +157,7 @@ Look for this in your logs, to confirm your changes worked as expected.
No secrets detected on input.
```
### Default Config Used
### Default Config Used
```
_default_detect_secrets_config = {
@ -259,8 +259,8 @@ _default_detect_secrets_config = {
"path": _custom_plugins_path + "/defined_networking_api_token.py",
},
{
"name": "GradientAIDetector",
"path": _custom_plugins_path + "/gradient_ai.py",
"name": "DigitaloceanDetector",
"path": _custom_plugins_path + "/digitalocean.py",
},
{
"name": "DopplerApiTokenDetector",

4
enterprise/litellm_enterprise/enterprise_callbacks/secret_detection.py
View file

@ -123,8 +123,8 @@ _default_detect_secrets_config = {
"path": _custom_plugins_path + "/defined_networking_api_token.py",
},
{
"name": "GradientAItector",
"path": _custom_plugins_path + "/gradient_aipy",
"name": "DigitaloceanDetector",
"path": _custom_plugins_path + "/digitalocean.py",
},
{
"name": "DopplerApiTokenDetector",

8
enterprise/litellm_enterprise/enterprise_callbacks/secrets_plugins/digitalocean.py
View file

@ -1,5 +1,5 @@
"""
This plugin searches for GradientAIokens.
This plugin searches for DigitalOcean tokens.
"""
import re
@ -7,12 +7,12 @@ import re
from detect_secrets.plugins.base import RegexBasedDetector
class GradientAItector(RegexBasedDetector):
"""Scans for various GradientAI Tokens."""
class DigitaloceanDetector(RegexBasedDetector):
"""Scans for various DigitalOcean Tokens."""
@property
def secret_type(self) -> str:
return "GradientAI Token"
return "DigitalOcean Token"
@property
def denylist(self) -> list[re.Pattern]:

3
litellm/main.py
View file

@ -3264,8 +3264,7 @@ def completion( # type: ignore # noqa: PLR0915
headers=headers,
encoding=encoding,
api_key=api_key,
logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements
client=client,
logging_obj=logging,
)
elif custom_llm_provider == "custom":

15
litellm/utils.py
View file

@ -3822,17 +3822,6 @@ def get_optional_params( # noqa: PLR0915
else False
),
)
elif custom_llm_provider == "gradient_ai":
optional_params = litellm.GradientAIConfig().map_openai_params(
non_default_params=non_default_params,
optional_params=optional_params,
model=model,
drop_params=(
drop_params
if drop_params is not None and isinstance(drop_params, bool)
else False
),
)
elif custom_llm_provider == "openai":
optional_params = litellm.OpenAIConfig().map_openai_params(
non_default_params=non_default_params,
@ -6855,8 +6844,8 @@ class ProviderConfigManager:
return litellm.LiteLLMProxyChatConfig()
elif litellm.LlmProviders.OPENAI == provider:
return litellm.OpenAIGPTConfig()
elif litellm.LlmProviders.GRADIENT_AI == provider:
return litellm.GradientAIConfig()
elif litellm.LlmProviders.DIGITALOCEAN == provider:
return litellm.DigitalOceanConfig()
elif litellm.LlmProviders.NSCALE == provider:
return litellm.NscaleConfig()
return None

311
model_prices_and_context_window.json
View file

@ -8393,14 +8393,8 @@
"mode": "chat",
"supports_function_calling": true,
"supports_vision": true,
"tool_use_system_prompt_tokens": 159,
"supports_assistant_prefill": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_reasoning": true,
"supports_computer_use": true
"supports_tool_choice": true
},
"vertex_ai/claude-3-5-sonnet-v2@20241022": {
"supports_computer_use": true,
@ -8412,15 +8406,10 @@
"litellm_provider": "vertex_ai-anthropic_models",
"mode": "chat",
"supports_function_calling": true,
"supports_vision": true,
"tool_use_system_prompt_tokens": 159,
"supports_assistant_prefill": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_reasoning": true,
"supports_computer_use": true
"supports_vision": true,
"supports_assistant_prefill": true,
"supports_tool_choice": true
},
"vertex_ai/claude-3-7-sonnet@20250219": {
"supports_computer_use": true,
@ -8434,15 +8423,15 @@
"litellm_provider": "vertex_ai-anthropic_models",
"mode": "chat",
"supports_function_calling": true,
"supports_pdf_input": true,
"supports_vision": true,
"tool_use_system_prompt_tokens": 159,
"supports_assistant_prefill": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"deprecation_date": "2025-06-01",
"supports_reasoning": true,
"supports_computer_use": true
"supports_tool_choice": true
},
"vertex_ai/claude-opus-4": {
"max_tokens": 32000,
@ -11765,124 +11754,6 @@
"supports_pdf_input": true,
"supports_tool_choice": true
},
"apac.anthropic.claude-3-5-sonnet-20240620-v1:0": {
"max_tokens": 4096,
"max_input_tokens": 200000,
"max_output_tokens": 4096,
"input_cost_per_token": 3e-06,
"output_cost_per_token": 1.5e-05,
"litellm_provider": "bedrock",
"mode": "chat",
"supports_function_calling": true,
"supports_response_schema": true,
"supports_vision": true,
"supports_pdf_input": true,
"supports_tool_choice": true
},
"apac.anthropic.claude-3-5-sonnet-20241022-v2:0": {
"max_tokens": 8192,
"max_input_tokens": 200000,
"max_output_tokens": 8192,
"input_cost_per_token": 3e-06,
"output_cost_per_token": 1.5e-05,
"cache_creation_input_token_cost": 3.75e-06,
"cache_read_input_token_cost": 3e-07,
"litellm_provider": "bedrock",
"mode": "chat",
"supports_function_calling": true,
"supports_vision": true,
"supports_assistant_prefill": true,
"supports_computer_use": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_response_schema": true,
"supports_tool_choice": true
},
"apac.anthropic.claude-sonnet-4-20250514-v1:0": {
"max_tokens": 64000,
"max_input_tokens": 200000,
"max_output_tokens": 64000,
"input_cost_per_token": 3e-06,
"output_cost_per_token": 1.5e-05,
"search_context_cost_per_query": {
"search_context_size_low": 0.01,
"search_context_size_medium": 0.01,
"search_context_size_high": 0.01
},
"cache_creation_input_token_cost": 3.75e-06,
"cache_read_input_token_cost": 3e-07,
"litellm_provider": "bedrock_converse",
"mode": "chat",
"supports_function_calling": true,
"supports_vision": true,
"tool_use_system_prompt_tokens": 159,
"supports_assistant_prefill": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_reasoning": true,
"supports_computer_use": true
},
"apac.anthropic.claude-3-5-sonnet-20240620-v1:0": {
"max_tokens": 4096,
"max_input_tokens": 200000,
"max_output_tokens": 4096,
"input_cost_per_token": 3e-06,
"output_cost_per_token": 1.5e-05,
"litellm_provider": "bedrock",
"mode": "chat",
"supports_function_calling": true,
"supports_response_schema": true,
"supports_vision": true,
"supports_pdf_input": true,
"supports_tool_choice": true
},
"apac.anthropic.claude-3-5-sonnet-20241022-v2:0": {
"max_tokens": 8192,
"max_input_tokens": 200000,
"max_output_tokens": 8192,
"input_cost_per_token": 3e-06,
"output_cost_per_token": 1.5e-05,
"cache_creation_input_token_cost": 3.75e-06,
"cache_read_input_token_cost": 3e-07,
"litellm_provider": "bedrock",
"mode": "chat",
"supports_function_calling": true,
"supports_vision": true,
"supports_assistant_prefill": true,
"supports_computer_use": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_response_schema": true,
"supports_tool_choice": true
},
"apac.anthropic.claude-sonnet-4-20250514-v1:0": {
"max_tokens": 64000,
"max_input_tokens": 200000,
"max_output_tokens": 64000,
"input_cost_per_token": 3e-06,
"output_cost_per_token": 1.5e-05,
"search_context_cost_per_query": {
"search_context_size_low": 0.01,
"search_context_size_medium": 0.01,
"search_context_size_high": 0.01
},
"cache_creation_input_token_cost": 3.75e-06,
"cache_read_input_token_cost": 3e-07,
"litellm_provider": "bedrock_converse",
"mode": "chat",
"supports_function_calling": true,
"supports_vision": true,
"tool_use_system_prompt_tokens": 159,
"supports_assistant_prefill": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_reasoning": true,
"supports_computer_use": true
},
"eu.anthropic.claude-3-5-haiku-20241022-v1:0": {
"max_tokens": 8192,
"max_input_tokens": 200000,
@ -12908,174 +12779,6 @@
"supports_function_calling": true,
"supports_tool_choice": false
},
"meta.llama4-maverick-17b-instruct-v1:0": {
"max_tokens": 4096,
"max_input_tokens": 128000,
"max_output_tokens": 4096,
"input_cost_per_token": 2.4e-07,
"input_cost_per_token_batches": 1.2e-07,
"output_cost_per_token": 9.7e-07,
"output_cost_per_token_batches": 4.85e-07,
"litellm_provider": "bedrock_converse",
"mode": "chat",
"supports_function_calling": true,
"supports_tool_choice": false,
"supported_modalities": [
"text",
"image"
],
"supported_output_modalities": [
"text",
"code"
]
},
"us.meta.llama4-maverick-17b-instruct-v1:0": {
"max_tokens": 4096,
"max_input_tokens": 128000,
"max_output_tokens": 4096,
"input_cost_per_token": 2.4e-07,
"input_cost_per_token_batches": 1.2e-07,
"output_cost_per_token": 9.7e-07,
"output_cost_per_token_batches": 4.85e-07,
"litellm_provider": "bedrock_converse",
"mode": "chat",
"supports_function_calling": true,
"supports_tool_choice": false,
"supported_modalities": [
"text",
"image"
],
"supported_output_modalities": [
"text",
"code"
]
},
"meta.llama4-scout-17b-instruct-v1:0": {
"max_tokens": 4096,
"max_input_tokens": 128000,
"max_output_tokens": 4096,
"input_cost_per_token": 1.7e-07,
"input_cost_per_token_batches": 8.5e-08,
"output_cost_per_token": 6.6e-07,
"output_cost_per_token_batches": 3.3e-07,
"litellm_provider": "bedrock_converse",
"mode": "chat",
"supports_function_calling": true,
"supports_tool_choice": false,
"supported_modalities": [
"text",
"image"
],
"supported_output_modalities": [
"text",
"code"
]
},
"us.meta.llama4-scout-17b-instruct-v1:0": {
"max_tokens": 4096,
"max_input_tokens": 128000,
"max_output_tokens": 4096,
"input_cost_per_token": 1.7e-07,
"input_cost_per_token_batches": 8.5e-08,
"output_cost_per_token": 6.6e-07,
"output_cost_per_token_batches": 3.3e-07,
"litellm_provider": "bedrock_converse",
"mode": "chat",
"supports_function_calling": true,
"supports_tool_choice": false,
"supported_modalities": [
"text",
"image"
],
"supported_output_modalities": [
"text",
"code"
]
},
"meta.llama4-maverick-17b-instruct-v1:0": {
"max_tokens": 4096,
"max_input_tokens": 128000,
"max_output_tokens": 4096,
"input_cost_per_token": 2.4e-07,
"input_cost_per_token_batches": 1.2e-07,
"output_cost_per_token": 9.7e-07,
"output_cost_per_token_batches": 4.85e-07,
"litellm_provider": "bedrock_converse",
"mode": "chat",
"supports_function_calling": true,
"supports_tool_choice": false,
"supported_modalities": [
"text",
"image"
],
"supported_output_modalities": [
"text",
"code"
]
},
"us.meta.llama4-maverick-17b-instruct-v1:0": {
"max_tokens": 4096,
"max_input_tokens": 128000,
"max_output_tokens": 4096,
"input_cost_per_token": 2.4e-07,
"input_cost_per_token_batches": 1.2e-07,
"output_cost_per_token": 9.7e-07,
"output_cost_per_token_batches": 4.85e-07,
"litellm_provider": "bedrock_converse",
"mode": "chat",
"supports_function_calling": true,
"supports_tool_choice": false,
"supported_modalities": [
"text",
"image"
],
"supported_output_modalities": [
"text",
"code"
]
},
"meta.llama4-scout-17b-instruct-v1:0": {
"max_tokens": 4096,
"max_input_tokens": 128000,
"max_output_tokens": 4096,
"input_cost_per_token": 1.7e-07,
"input_cost_per_token_batches": 8.5e-08,
"output_cost_per_token": 6.6e-07,
"output_cost_per_token_batches": 3.3e-07,
"litellm_provider": "bedrock_converse",
"mode": "chat",
"supports_function_calling": true,
"supports_tool_choice": false,
"supported_modalities": [
"text",
"image"
],
"supported_output_modalities": [
"text",
"code"
]
},
"us.meta.llama4-scout-17b-instruct-v1:0": {
"max_tokens": 4096,
"max_input_tokens": 128000,
"max_output_tokens": 4096,
"input_cost_per_token": 1.7e-07,
"input_cost_per_token_batches": 8.5e-08,
"output_cost_per_token": 6.6e-07,
"output_cost_per_token_batches": 3.3e-07,
"litellm_provider": "bedrock_converse",
"mode": "chat",
"supports_function_calling": true,
"supports_tool_choice": false,
"supported_modalities": [
"text",
"image"
],
"supported_output_modalities": [
"text",
"code"
]
},
"512-x-512/50-steps/stability.stable-diffusion-xl-v0": {
"max_tokens": 77,
"max_input_tokens": 77,

24
proxy_server_config.yaml
View file

@ -18,7 +18,7 @@ model_list:
api_version: "2023-05-15"
api_key: os.environ/AZURE_API_KEY # The `os.environ/` prefix tells litellm to read this from the env. See https://docs.litellm.ai/docs/simple_proxy#load-api-keys-from-vault
- model_name: gpt-3.5-turbo-large
litellm_params:
litellm_params:
model: "gpt-3.5-turbo-1106"
api_key: os.environ/OPENAI_API_KEY
rpm: 480
@ -36,9 +36,9 @@ model_list:
- model_name: sagemaker-completion-model
litellm_params:
model: sagemaker/berri-benchmarking-Llama-2-70b-chat-hf-4
input_cost_per_second: 0.000420
input_cost_per_second: 0.000420
- model_name: text-embedding-ada-002
litellm_params:
litellm_params:
model: azure/azure-embedding-model
api_key: os.environ/AZURE_API_KEY
api_base: https://openai-gpt-4-test-v-1.openai.azure.com/
@ -106,7 +106,7 @@ model_list:
litellm_params:
model: openai/*
api_key: os.environ/OPENAI_API_KEY
# provider specific wildcard routing
- model_name: "anthropic/*"
@ -140,20 +140,20 @@ model_list:
- model_name: "gradient_ai/*"
litellm_params:
model: "gradient_ai/*"
api_key: os.environ/_API_KEY
api_key: os.environ/GRADIENT_AI_API_KEY
api_base: os.environ/GRADIENT_AI_AGENT_ENDPOINT
litellm_settings:
# set_verbose: True # Uncomment this if you want to see verbose logs; not recommended in production
drop_params: True
# max_budget: 100
# max_budget: 100
# budget_duration: 30d
num_retries: 5
request_timeout: 600
telemetry: False
context_window_fallbacks: [{"gpt-3.5-turbo": ["gpt-3.5-turbo-large"]}]
default_team_settings:
default_team_settings:
- team_id: team-1
success_callback: ["langfuse"]
failure_callback: ["langfuse"]
@ -185,14 +185,14 @@ files_settings:
api_key: os.environ/OPENAI_API_KEY
router_settings:
routing_strategy: usage-based-routing-v2
routing_strategy: usage-based-routing-v2
redis_host: os.environ/REDIS_HOST
redis_password: os.environ/REDIS_PASSWORD
redis_port: os.environ/REDIS_PORT
enable_pre_call_checks: true
model_group_alias: {"my-special-fake-model-alias-name": "fake-openai-endpoint-3"}
model_group_alias: {"my-special-fake-model-alias-name": "fake-openai-endpoint-3"}
general_settings:
general_settings:
master_key: sk-1234 # [OPTIONAL] Use to enforce auth on proxy. See - https://docs.litellm.ai/docs/proxy/virtual_keys
store_model_in_db: True
proxy_budget_rescheduler_min_time: 60
@ -205,7 +205,7 @@ general_settings:
- path: "/v1/rerank" # route you want to add to LiteLLM Proxy Server
target: "https://api.cohere.com/v1/rerank" # URL this route should forward requests to
headers: # headers to forward to this URL
content-type: application/json # (Optional) Extra Headers to pass to this endpoint
content-type: application/json # (Optional) Extra Headers to pass to this endpoint
accept: application/json
forward_headers: True
@ -213,4 +213,4 @@ general_settings:
# settings for using redis caching
# REDIS_HOST: redis-16337.c322.us-east-1-2.ec2.cloud.redislabs.com
# REDIS_PORT: "16337"
# REDIS_PASSWORD:
# REDIS_PASSWORD:

0
tests/litellm/llms/digitalocean/chat/test_digitalocean_chat_transformation.py → tests/litellm/llms/digitalocean/chat/test_gradient_ai_chat_transformation.py
View file

0
ui/litellm-dashboard/out/assets/logos/digitalocean.svg → ui/litellm-dashboard/out/assets/logos/gradientai.svg
View file

Side by side Swipe Overlay

Before

Width:  |  Height:  |  Size: 7 KiB

After

Width:  |  Height:  |  Size: 7 KiB

3
ui/litellm-dashboard/src/components/add_model/provider_specific_fields.tsx
View file

@ -358,7 +358,6 @@ const PROVIDER_CREDENTIAL_FIELDS: Record<Providers, ProviderCredentialField[]> =
placeholder: "https://...",
required: true
},
{
key: "base_model",
label: "Base Model",
@ -370,7 +369,7 @@ const PROVIDER_CREDENTIAL_FIELDS: Record<Providers, ProviderCredentialField[]> =
type: "password",
required: true
}
]
],
[Providers.Triton]: [{
key: "api_key",
label: "API Key",