From 37bd51bd54ef4fd52ccc12866e47f8de9476d597 Mon Sep 17 00:00:00 2001 From: Muhammad Sannan Nasir Date: Fri, 11 Jul 2025 22:34:57 +0500 Subject: [PATCH] Incorporte review comments --- docs/my-website/Dockerfile | 4 +- .../{digitalocean.md => gradient_ai.md} | 4 +- .../docs/proxy/guardrails/secret_detection.md | 16 +- .../enterprise_callbacks/secret_detection.py | 4 +- .../secrets_plugins/digitalocean.py | 8 +- litellm/main.py | 3 +- litellm/utils.py | 15 +- model_prices_and_context_window.json | 311 +----------------- proxy_server_config.yaml | 24 +- ...> test_gradient_ai_chat_transformation.py} | 0 .../{digitalocean.svg => gradientai.svg} | 0 .../add_model/provider_specific_fields.tsx | 3 +- 12 files changed, 41 insertions(+), 351 deletions(-) rename docs/my-website/docs/providers/{digitalocean.md => gradient_ai.md} (97%) rename tests/litellm/llms/digitalocean/chat/{test_digitalocean_chat_transformation.py => test_gradient_ai_chat_transformation.py} (100%) rename ui/litellm-dashboard/out/assets/logos/{digitalocean.svg => gradientai.svg} (100%) diff --git a/docs/my-website/Dockerfile b/docs/my-website/Dockerfile index a1989a9dd97..87d1537237d 100644 --- a/docs/my-website/Dockerfile +++ b/docs/my-website/Dockerfile @@ -4,6 +4,6 @@ COPY . /app WORKDIR /app RUN pip install -r requirements.txt -EXPOSE 3000 +EXPOSE $PORT -CMD litellm --host 0.0.0.0 --port 3000 --workers 10 --config config.yaml \ No newline at end of file +CMD litellm --host 0.0.0.0 --port $PORT --workers 10 --config config.yaml \ No newline at end of file diff --git a/docs/my-website/docs/providers/digitalocean.md b/docs/my-website/docs/providers/gradient_ai.md similarity index 97% rename from docs/my-website/docs/providers/digitalocean.md rename to docs/my-website/docs/providers/gradient_ai.md index 334d655a40c..d9364c62ea5 100644 --- a/docs/my-website/docs/providers/digitalocean.md +++ b/docs/my-website/docs/providers/gradient_ai.md @@ -1,11 +1,11 @@ import Tabs from '@theme/Tabs'; import TabItem from '@theme/TabItem'; -# GradientAI GradientAI +# GradientAI https://digitalocean.com/products/genai -LiteLLM provides native support for GradientAI GenAI models. +LiteLLM provides native support for GradientAI models. To use a GradientAI model, specify it as `gradient_ai/` in your LiteLLM requests. diff --git a/docs/my-website/docs/proxy/guardrails/secret_detection.md b/docs/my-website/docs/proxy/guardrails/secret_detection.md index bcd848b4d31..a70c35d96af 100644 --- a/docs/my-website/docs/proxy/guardrails/secret_detection.md +++ b/docs/my-website/docs/proxy/guardrails/secret_detection.md @@ -1,9 +1,9 @@ # ✨ Secret Detection/Redaction (Enterprise-only) -❓ Use this to REDACT API Keys, Secrets sent in requests to an LLM. +❓ Use this to REDACT API Keys, Secrets sent in requests to an LLM. Example if you want to redact the value of `OPENAI_API_KEY` in the following request -#### Incoming Request +#### Incoming Request ```json { @@ -31,13 +31,13 @@ Example if you want to redact the value of `OPENAI_API_KEY` in the following req **Usage** -**Step 1** Add this to your config.yaml +**Step 1** Add this to your config.yaml ```yaml guardrails: - guardrail_name: "my-custom-name" litellm_params: - guardrail: "hide-secrets" # supported values: "aporia", "lakera", .. + guardrail: "hide-secrets" # supported values: "aporia", "lakera", .. mode: "pre_call" ``` @@ -125,7 +125,7 @@ guardrails: **2. Start proxy** -Run with `--detailed_debug` for more detailed logs. Use in dev only. +Run with `--detailed_debug` for more detailed logs. Use in dev only. ```bash litellm --config /path/to/config.yaml --detailed_debug @@ -157,7 +157,7 @@ Look for this in your logs, to confirm your changes worked as expected. No secrets detected on input. ``` -### Default Config Used +### Default Config Used ``` _default_detect_secrets_config = { @@ -259,8 +259,8 @@ _default_detect_secrets_config = { "path": _custom_plugins_path + "/defined_networking_api_token.py", }, { - "name": "GradientAIDetector", - "path": _custom_plugins_path + "/gradient_ai.py", + "name": "DigitaloceanDetector", + "path": _custom_plugins_path + "/digitalocean.py", }, { "name": "DopplerApiTokenDetector", diff --git a/enterprise/litellm_enterprise/enterprise_callbacks/secret_detection.py b/enterprise/litellm_enterprise/enterprise_callbacks/secret_detection.py index 74265e91b51..8a7a82df686 100644 --- a/enterprise/litellm_enterprise/enterprise_callbacks/secret_detection.py +++ b/enterprise/litellm_enterprise/enterprise_callbacks/secret_detection.py @@ -123,8 +123,8 @@ _default_detect_secrets_config = { "path": _custom_plugins_path + "/defined_networking_api_token.py", }, { - "name": "GradientAItector", - "path": _custom_plugins_path + "/gradient_aipy", + "name": "DigitaloceanDetector", + "path": _custom_plugins_path + "/digitalocean.py", }, { "name": "DopplerApiTokenDetector", diff --git a/enterprise/litellm_enterprise/enterprise_callbacks/secrets_plugins/digitalocean.py b/enterprise/litellm_enterprise/enterprise_callbacks/secrets_plugins/digitalocean.py index 45fd3710e6d..5ffc4f600e7 100644 --- a/enterprise/litellm_enterprise/enterprise_callbacks/secrets_plugins/digitalocean.py +++ b/enterprise/litellm_enterprise/enterprise_callbacks/secrets_plugins/digitalocean.py @@ -1,5 +1,5 @@ """ -This plugin searches for GradientAIokens. +This plugin searches for DigitalOcean tokens. """ import re @@ -7,12 +7,12 @@ import re from detect_secrets.plugins.base import RegexBasedDetector -class GradientAItector(RegexBasedDetector): - """Scans for various GradientAI Tokens.""" +class DigitaloceanDetector(RegexBasedDetector): + """Scans for various DigitalOcean Tokens.""" @property def secret_type(self) -> str: - return "GradientAI Token" + return "DigitalOcean Token" @property def denylist(self) -> list[re.Pattern]: diff --git a/litellm/main.py b/litellm/main.py index fec5f2e4703..e8c71a46d8b 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -3264,8 +3264,7 @@ def completion( # type: ignore # noqa: PLR0915 headers=headers, encoding=encoding, api_key=api_key, - logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements - client=client, + logging_obj=logging, ) elif custom_llm_provider == "custom": diff --git a/litellm/utils.py b/litellm/utils.py index b4fcb1e528c..eb407c2291a 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -3822,17 +3822,6 @@ def get_optional_params( # noqa: PLR0915 else False ), ) - elif custom_llm_provider == "gradient_ai": - optional_params = litellm.GradientAIConfig().map_openai_params( - non_default_params=non_default_params, - optional_params=optional_params, - model=model, - drop_params=( - drop_params - if drop_params is not None and isinstance(drop_params, bool) - else False - ), - ) elif custom_llm_provider == "openai": optional_params = litellm.OpenAIConfig().map_openai_params( non_default_params=non_default_params, @@ -6855,8 +6844,8 @@ class ProviderConfigManager: return litellm.LiteLLMProxyChatConfig() elif litellm.LlmProviders.OPENAI == provider: return litellm.OpenAIGPTConfig() - elif litellm.LlmProviders.GRADIENT_AI == provider: - return litellm.GradientAIConfig() + elif litellm.LlmProviders.DIGITALOCEAN == provider: + return litellm.DigitalOceanConfig() elif litellm.LlmProviders.NSCALE == provider: return litellm.NscaleConfig() return None diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 83ad266763c..a94c5db717b 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -8393,14 +8393,8 @@ "mode": "chat", "supports_function_calling": true, "supports_vision": true, - "tool_use_system_prompt_tokens": 159, "supports_assistant_prefill": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_reasoning": true, - "supports_computer_use": true + "supports_tool_choice": true }, "vertex_ai/claude-3-5-sonnet-v2@20241022": { "supports_computer_use": true, @@ -8412,15 +8406,10 @@ "litellm_provider": "vertex_ai-anthropic_models", "mode": "chat", "supports_function_calling": true, - "supports_vision": true, - "tool_use_system_prompt_tokens": 159, - "supports_assistant_prefill": true, "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_reasoning": true, - "supports_computer_use": true + "supports_vision": true, + "supports_assistant_prefill": true, + "supports_tool_choice": true }, "vertex_ai/claude-3-7-sonnet@20250219": { "supports_computer_use": true, @@ -8434,15 +8423,15 @@ "litellm_provider": "vertex_ai-anthropic_models", "mode": "chat", "supports_function_calling": true, + "supports_pdf_input": true, "supports_vision": true, "tool_use_system_prompt_tokens": 159, "supports_assistant_prefill": true, - "supports_pdf_input": true, "supports_prompt_caching": true, "supports_response_schema": true, - "supports_tool_choice": true, + "deprecation_date": "2025-06-01", "supports_reasoning": true, - "supports_computer_use": true + "supports_tool_choice": true }, "vertex_ai/claude-opus-4": { "max_tokens": 32000, @@ -11765,124 +11754,6 @@ "supports_pdf_input": true, "supports_tool_choice": true }, - "apac.anthropic.claude-3-5-sonnet-20240620-v1:0": { - "max_tokens": 4096, - "max_input_tokens": 200000, - "max_output_tokens": 4096, - "input_cost_per_token": 3e-06, - "output_cost_per_token": 1.5e-05, - "litellm_provider": "bedrock", - "mode": "chat", - "supports_function_calling": true, - "supports_response_schema": true, - "supports_vision": true, - "supports_pdf_input": true, - "supports_tool_choice": true - }, - "apac.anthropic.claude-3-5-sonnet-20241022-v2:0": { - "max_tokens": 8192, - "max_input_tokens": 200000, - "max_output_tokens": 8192, - "input_cost_per_token": 3e-06, - "output_cost_per_token": 1.5e-05, - "cache_creation_input_token_cost": 3.75e-06, - "cache_read_input_token_cost": 3e-07, - "litellm_provider": "bedrock", - "mode": "chat", - "supports_function_calling": true, - "supports_vision": true, - "supports_assistant_prefill": true, - "supports_computer_use": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_response_schema": true, - "supports_tool_choice": true - }, - "apac.anthropic.claude-sonnet-4-20250514-v1:0": { - "max_tokens": 64000, - "max_input_tokens": 200000, - "max_output_tokens": 64000, - "input_cost_per_token": 3e-06, - "output_cost_per_token": 1.5e-05, - "search_context_cost_per_query": { - "search_context_size_low": 0.01, - "search_context_size_medium": 0.01, - "search_context_size_high": 0.01 - }, - "cache_creation_input_token_cost": 3.75e-06, - "cache_read_input_token_cost": 3e-07, - "litellm_provider": "bedrock_converse", - "mode": "chat", - "supports_function_calling": true, - "supports_vision": true, - "tool_use_system_prompt_tokens": 159, - "supports_assistant_prefill": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_reasoning": true, - "supports_computer_use": true - }, - "apac.anthropic.claude-3-5-sonnet-20240620-v1:0": { - "max_tokens": 4096, - "max_input_tokens": 200000, - "max_output_tokens": 4096, - "input_cost_per_token": 3e-06, - "output_cost_per_token": 1.5e-05, - "litellm_provider": "bedrock", - "mode": "chat", - "supports_function_calling": true, - "supports_response_schema": true, - "supports_vision": true, - "supports_pdf_input": true, - "supports_tool_choice": true - }, - "apac.anthropic.claude-3-5-sonnet-20241022-v2:0": { - "max_tokens": 8192, - "max_input_tokens": 200000, - "max_output_tokens": 8192, - "input_cost_per_token": 3e-06, - "output_cost_per_token": 1.5e-05, - "cache_creation_input_token_cost": 3.75e-06, - "cache_read_input_token_cost": 3e-07, - "litellm_provider": "bedrock", - "mode": "chat", - "supports_function_calling": true, - "supports_vision": true, - "supports_assistant_prefill": true, - "supports_computer_use": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_response_schema": true, - "supports_tool_choice": true - }, - "apac.anthropic.claude-sonnet-4-20250514-v1:0": { - "max_tokens": 64000, - "max_input_tokens": 200000, - "max_output_tokens": 64000, - "input_cost_per_token": 3e-06, - "output_cost_per_token": 1.5e-05, - "search_context_cost_per_query": { - "search_context_size_low": 0.01, - "search_context_size_medium": 0.01, - "search_context_size_high": 0.01 - }, - "cache_creation_input_token_cost": 3.75e-06, - "cache_read_input_token_cost": 3e-07, - "litellm_provider": "bedrock_converse", - "mode": "chat", - "supports_function_calling": true, - "supports_vision": true, - "tool_use_system_prompt_tokens": 159, - "supports_assistant_prefill": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_reasoning": true, - "supports_computer_use": true - }, "eu.anthropic.claude-3-5-haiku-20241022-v1:0": { "max_tokens": 8192, "max_input_tokens": 200000, @@ -12908,174 +12779,6 @@ "supports_function_calling": true, "supports_tool_choice": false }, - "meta.llama4-maverick-17b-instruct-v1:0": { - "max_tokens": 4096, - "max_input_tokens": 128000, - "max_output_tokens": 4096, - "input_cost_per_token": 2.4e-07, - "input_cost_per_token_batches": 1.2e-07, - "output_cost_per_token": 9.7e-07, - "output_cost_per_token_batches": 4.85e-07, - "litellm_provider": "bedrock_converse", - "mode": "chat", - "supports_function_calling": true, - "supports_tool_choice": false, - "supported_modalities": [ - "text", - "image" - ], - "supported_output_modalities": [ - "text", - "code" - ] - }, - "us.meta.llama4-maverick-17b-instruct-v1:0": { - "max_tokens": 4096, - "max_input_tokens": 128000, - "max_output_tokens": 4096, - "input_cost_per_token": 2.4e-07, - "input_cost_per_token_batches": 1.2e-07, - "output_cost_per_token": 9.7e-07, - "output_cost_per_token_batches": 4.85e-07, - "litellm_provider": "bedrock_converse", - "mode": "chat", - "supports_function_calling": true, - "supports_tool_choice": false, - "supported_modalities": [ - "text", - "image" - ], - "supported_output_modalities": [ - "text", - "code" - ] - }, - "meta.llama4-scout-17b-instruct-v1:0": { - "max_tokens": 4096, - "max_input_tokens": 128000, - "max_output_tokens": 4096, - "input_cost_per_token": 1.7e-07, - "input_cost_per_token_batches": 8.5e-08, - "output_cost_per_token": 6.6e-07, - "output_cost_per_token_batches": 3.3e-07, - "litellm_provider": "bedrock_converse", - "mode": "chat", - "supports_function_calling": true, - "supports_tool_choice": false, - "supported_modalities": [ - "text", - "image" - ], - "supported_output_modalities": [ - "text", - "code" - ] - }, - "us.meta.llama4-scout-17b-instruct-v1:0": { - "max_tokens": 4096, - "max_input_tokens": 128000, - "max_output_tokens": 4096, - "input_cost_per_token": 1.7e-07, - "input_cost_per_token_batches": 8.5e-08, - "output_cost_per_token": 6.6e-07, - "output_cost_per_token_batches": 3.3e-07, - "litellm_provider": "bedrock_converse", - "mode": "chat", - "supports_function_calling": true, - "supports_tool_choice": false, - "supported_modalities": [ - "text", - "image" - ], - "supported_output_modalities": [ - "text", - "code" - ] - }, - "meta.llama4-maverick-17b-instruct-v1:0": { - "max_tokens": 4096, - "max_input_tokens": 128000, - "max_output_tokens": 4096, - "input_cost_per_token": 2.4e-07, - "input_cost_per_token_batches": 1.2e-07, - "output_cost_per_token": 9.7e-07, - "output_cost_per_token_batches": 4.85e-07, - "litellm_provider": "bedrock_converse", - "mode": "chat", - "supports_function_calling": true, - "supports_tool_choice": false, - "supported_modalities": [ - "text", - "image" - ], - "supported_output_modalities": [ - "text", - "code" - ] - }, - "us.meta.llama4-maverick-17b-instruct-v1:0": { - "max_tokens": 4096, - "max_input_tokens": 128000, - "max_output_tokens": 4096, - "input_cost_per_token": 2.4e-07, - "input_cost_per_token_batches": 1.2e-07, - "output_cost_per_token": 9.7e-07, - "output_cost_per_token_batches": 4.85e-07, - "litellm_provider": "bedrock_converse", - "mode": "chat", - "supports_function_calling": true, - "supports_tool_choice": false, - "supported_modalities": [ - "text", - "image" - ], - "supported_output_modalities": [ - "text", - "code" - ] - }, - "meta.llama4-scout-17b-instruct-v1:0": { - "max_tokens": 4096, - "max_input_tokens": 128000, - "max_output_tokens": 4096, - "input_cost_per_token": 1.7e-07, - "input_cost_per_token_batches": 8.5e-08, - "output_cost_per_token": 6.6e-07, - "output_cost_per_token_batches": 3.3e-07, - "litellm_provider": "bedrock_converse", - "mode": "chat", - "supports_function_calling": true, - "supports_tool_choice": false, - "supported_modalities": [ - "text", - "image" - ], - "supported_output_modalities": [ - "text", - "code" - ] - }, - "us.meta.llama4-scout-17b-instruct-v1:0": { - "max_tokens": 4096, - "max_input_tokens": 128000, - "max_output_tokens": 4096, - "input_cost_per_token": 1.7e-07, - "input_cost_per_token_batches": 8.5e-08, - "output_cost_per_token": 6.6e-07, - "output_cost_per_token_batches": 3.3e-07, - "litellm_provider": "bedrock_converse", - "mode": "chat", - "supports_function_calling": true, - "supports_tool_choice": false, - "supported_modalities": [ - "text", - "image" - ], - "supported_output_modalities": [ - "text", - "code" - ] - }, "512-x-512/50-steps/stability.stable-diffusion-xl-v0": { "max_tokens": 77, "max_input_tokens": 77, diff --git a/proxy_server_config.yaml b/proxy_server_config.yaml index 967cab19f44..18eb48ddfaa 100644 --- a/proxy_server_config.yaml +++ b/proxy_server_config.yaml @@ -18,7 +18,7 @@ model_list: api_version: "2023-05-15" api_key: os.environ/AZURE_API_KEY # The `os.environ/` prefix tells litellm to read this from the env. See https://docs.litellm.ai/docs/simple_proxy#load-api-keys-from-vault - model_name: gpt-3.5-turbo-large - litellm_params: + litellm_params: model: "gpt-3.5-turbo-1106" api_key: os.environ/OPENAI_API_KEY rpm: 480 @@ -36,9 +36,9 @@ model_list: - model_name: sagemaker-completion-model litellm_params: model: sagemaker/berri-benchmarking-Llama-2-70b-chat-hf-4 - input_cost_per_second: 0.000420 + input_cost_per_second: 0.000420 - model_name: text-embedding-ada-002 - litellm_params: + litellm_params: model: azure/azure-embedding-model api_key: os.environ/AZURE_API_KEY api_base: https://openai-gpt-4-test-v-1.openai.azure.com/ @@ -106,7 +106,7 @@ model_list: litellm_params: model: openai/* api_key: os.environ/OPENAI_API_KEY - + # provider specific wildcard routing - model_name: "anthropic/*" @@ -140,20 +140,20 @@ model_list: - model_name: "gradient_ai/*" litellm_params: model: "gradient_ai/*" - api_key: os.environ/_API_KEY + api_key: os.environ/GRADIENT_AI_API_KEY api_base: os.environ/GRADIENT_AI_AGENT_ENDPOINT litellm_settings: # set_verbose: True # Uncomment this if you want to see verbose logs; not recommended in production drop_params: True - # max_budget: 100 + # max_budget: 100 # budget_duration: 30d num_retries: 5 request_timeout: 600 telemetry: False context_window_fallbacks: [{"gpt-3.5-turbo": ["gpt-3.5-turbo-large"]}] - default_team_settings: + default_team_settings: - team_id: team-1 success_callback: ["langfuse"] failure_callback: ["langfuse"] @@ -185,14 +185,14 @@ files_settings: api_key: os.environ/OPENAI_API_KEY router_settings: - routing_strategy: usage-based-routing-v2 + routing_strategy: usage-based-routing-v2 redis_host: os.environ/REDIS_HOST redis_password: os.environ/REDIS_PASSWORD redis_port: os.environ/REDIS_PORT enable_pre_call_checks: true - model_group_alias: {"my-special-fake-model-alias-name": "fake-openai-endpoint-3"} + model_group_alias: {"my-special-fake-model-alias-name": "fake-openai-endpoint-3"} -general_settings: +general_settings: master_key: sk-1234 # [OPTIONAL] Use to enforce auth on proxy. See - https://docs.litellm.ai/docs/proxy/virtual_keys store_model_in_db: True proxy_budget_rescheduler_min_time: 60 @@ -205,7 +205,7 @@ general_settings: - path: "/v1/rerank" # route you want to add to LiteLLM Proxy Server target: "https://api.cohere.com/v1/rerank" # URL this route should forward requests to headers: # headers to forward to this URL - content-type: application/json # (Optional) Extra Headers to pass to this endpoint + content-type: application/json # (Optional) Extra Headers to pass to this endpoint accept: application/json forward_headers: True @@ -213,4 +213,4 @@ general_settings: # settings for using redis caching # REDIS_HOST: redis-16337.c322.us-east-1-2.ec2.cloud.redislabs.com # REDIS_PORT: "16337" - # REDIS_PASSWORD: + # REDIS_PASSWORD: diff --git a/tests/litellm/llms/digitalocean/chat/test_digitalocean_chat_transformation.py b/tests/litellm/llms/digitalocean/chat/test_gradient_ai_chat_transformation.py similarity index 100% rename from tests/litellm/llms/digitalocean/chat/test_digitalocean_chat_transformation.py rename to tests/litellm/llms/digitalocean/chat/test_gradient_ai_chat_transformation.py diff --git a/ui/litellm-dashboard/out/assets/logos/digitalocean.svg b/ui/litellm-dashboard/out/assets/logos/gradientai.svg similarity index 100% rename from ui/litellm-dashboard/out/assets/logos/digitalocean.svg rename to ui/litellm-dashboard/out/assets/logos/gradientai.svg diff --git a/ui/litellm-dashboard/src/components/add_model/provider_specific_fields.tsx b/ui/litellm-dashboard/src/components/add_model/provider_specific_fields.tsx index 236fb0662dc..c43234c4831 100644 --- a/ui/litellm-dashboard/src/components/add_model/provider_specific_fields.tsx +++ b/ui/litellm-dashboard/src/components/add_model/provider_specific_fields.tsx @@ -358,7 +358,6 @@ const PROVIDER_CREDENTIAL_FIELDS: Record = placeholder: "https://...", required: true }, - { key: "base_model", label: "Base Model", @@ -370,7 +369,7 @@ const PROVIDER_CREDENTIAL_FIELDS: Record = type: "password", required: true } - ] + ], [Providers.Triton]: [{ key: "api_key", label: "API Key",