From bf300f8ca78973db285c7eb107e2655550032e5c Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Tue, 22 Jul 2025 18:28:36 -0700 Subject: [PATCH] Revert "Litellm dev 07 21 2025 p1 (#12848)" This reverts commit e4e10aa4edcb9b8e69a99b9e166514be2b3c34b4. --- docs/my-website/docs/benchmarks.md | 75 ++- docs/my-website/docs/projects/HolmesGPT.md | 7 + docs/my-website/docs/providers/azure/azure.md | 64 ++- docs/my-website/docs/providers/groq.md | 2 +- docs/my-website/docs/providers/hyperbolic.md | 331 ++++++++++++ docs/my-website/docs/providers/lambda_ai.md | 280 ++++++++++ docs/my-website/docs/providers/moonshot.md | 14 +- docs/my-website/docs/providers/morph.md | 123 +++++ .../docs/providers/vertex_partner.md | 11 + docs/my-website/docs/proxy/config_settings.md | 2 + .../docs/proxy/guardrails/model_armor.md | 93 ++++ docs/my-website/package.json | 3 +- docs/my-website/sidebars.js | 5 + .../proxy/common_utils/check_batch_cost.py | 27 +- litellm/__init__.py | 119 ++-- litellm/constants.py | 12 + .../get_llm_provider_logic.py | 29 +- litellm/llms/anthropic/chat/handler.py | 5 +- litellm/llms/azure/common_utils.py | 46 +- litellm/llms/github_copilot/authenticator.py | 17 + .../github_copilot/chat/transformation.py | 8 +- litellm/llms/hyperbolic/__init__.py | 0 litellm/llms/hyperbolic/chat/__init__.py | 0 .../llms/hyperbolic/chat/transformation.py | 54 ++ litellm/llms/lambda_ai/__init__.py | 0 litellm/llms/lambda_ai/chat/__init__.py | 0 litellm/llms/lambda_ai/chat/transformation.py | 31 ++ litellm/llms/morph/__init__.py | 0 litellm/llms/morph/chat/__init__.py | 0 litellm/llms/morph/chat/transformation.py | 46 ++ .../vertex_and_google_ai_studio_gemini.py | 3 +- litellm/llms/watsonx/chat/transformation.py | 4 +- litellm/main.py | 4 +- ...odel_prices_and_context_window_backup.json | 510 +++++++++++++++++- litellm/proxy/_new_secret_config.yaml | 26 +- litellm/proxy/proxy_server.py | 12 +- .../proxy_setting_endpoints.py | 2 +- ...rward_clientside_headers_by_model_group.py | 14 +- .../get_azure_ad_token_provider.py | 18 +- litellm/types/llms/recraft.py | 3 +- .../get_azure_ad_token_provider.py | 1 + litellm/types/utils.py | 3 + litellm/utils.py | 6 + model_prices_and_context_window.json | 510 +++++++++++++++++- tests/llm_translation/test_hyperbolic.py | 119 ++++ tests/llm_translation/test_lambda_ai.py | 153 ++++++ tests/llm_translation/test_morph.py | 108 ++++ tests/llm_translation/test_optional_params.py | 2 +- tests/local_testing/test_completion.py | 2 +- tests/local_testing/test_exceptions.py | 69 +-- tests/local_testing/test_function_calling.py | 2 +- tests/local_testing/test_router.py | 10 +- tests/local_testing/test_streaming.py | 3 +- .../llms/azure/test_azure_common_utils.py | 112 +++- .../test_github_copilot_authenticator.py | 11 + .../test_github_copilot_transformation.py | 20 +- tests/test_litellm/test_main.py | 25 - tests/test_openai_endpoints.py | 2 +- .../src/components/team/team_info.tsx | 22 +- 59 files changed, 2938 insertions(+), 242 deletions(-) create mode 100644 docs/my-website/docs/projects/HolmesGPT.md create mode 100644 docs/my-website/docs/providers/hyperbolic.md create mode 100644 docs/my-website/docs/providers/lambda_ai.md create mode 100644 docs/my-website/docs/providers/morph.md create mode 100644 docs/my-website/docs/proxy/guardrails/model_armor.md create mode 100644 litellm/llms/hyperbolic/__init__.py create mode 100644 litellm/llms/hyperbolic/chat/__init__.py create mode 100644 litellm/llms/hyperbolic/chat/transformation.py create mode 100644 litellm/llms/lambda_ai/__init__.py create mode 100644 litellm/llms/lambda_ai/chat/__init__.py create mode 100644 litellm/llms/lambda_ai/chat/transformation.py create mode 100644 litellm/llms/morph/__init__.py create mode 100644 litellm/llms/morph/chat/__init__.py create mode 100644 litellm/llms/morph/chat/transformation.py create mode 100644 tests/llm_translation/test_hyperbolic.py create mode 100644 tests/llm_translation/test_lambda_ai.py create mode 100644 tests/llm_translation/test_morph.py diff --git a/docs/my-website/docs/benchmarks.md b/docs/my-website/docs/benchmarks.md index 817d70b87c2..43ab82b8e61 100644 --- a/docs/my-website/docs/benchmarks.md +++ b/docs/my-website/docs/benchmarks.md @@ -18,13 +18,17 @@ model_list: ### 1 Instance LiteLLM Proxy -In these tests the median latency of directly calling the fake-openai-endpoint is 60ms. +In these tests the baseline latency characteristics are measured against a fake-openai-endpoint. -| Metric | Litellm Proxy (1 Instance) | -|--------|------------------------| -| RPS | 475 | -| Median Latency (ms) | 100 | -| Latency overhead added by LiteLLM Proxy | 40ms | +#### Performance Metrics + +| Metric | Value | +|--------|-------| +| **Requests per Second (RPS)** | 475 | +| **End-to-End Latency P50 (ms)** | 100 | +| **LiteLLM Overhead P50 (ms)** | 3 | +| **LiteLLM Overhead P90 (ms)** | 17 | +| **LiteLLM Overhead P99 (ms)** | 31 | @@ -33,7 +37,8 @@ In these tests the median latency of directly calling the fake-openai-endpoint i --> #### Key Findings -- Single instance: 475 RPS @ 100ms latency +- Single instance: 475 RPS @ 100ms median latency +- LiteLLM adds 3ms P50 overhead, 17ms P90 overhead, 31ms P99 overhead - 2 LiteLLM instances: 950 RPS @ 100ms latency - 4 LiteLLM instances: 1900 RPS @ 100ms latency @@ -54,6 +59,62 @@ Each machine deploying LiteLLM had the following specs: - 2 CPU - 4GB RAM +## How to measure LiteLLM Overhead + +All responses from litellm will include the `x-litellm-overhead-duration-ms` header, this is the latency overhead in milliseconds added by LiteLLM Proxy. + + +If you want to measure this on locust you can use the following code: + +```python showLineNumbers title="Locust Code for measuring LiteLLM Overhead" +import os +import uuid +from locust import HttpUser, task, between, events + +# Custom metric to track LiteLLM overhead duration +overhead_durations = [] + +@events.request.add_listener +def on_request(request_type, name, response_time, response_length, response, context, exception, start_time, url, **kwargs): + if response and hasattr(response, 'headers'): + overhead_duration = response.headers.get('x-litellm-overhead-duration-ms') + if overhead_duration: + try: + duration_ms = float(overhead_duration) + overhead_durations.append(duration_ms) + # Report as custom metric + events.request.fire( + request_type="Custom", + name="LiteLLM Overhead Duration (ms)", + response_time=duration_ms, + response_length=0, + ) + except (ValueError, TypeError): + pass + +class MyUser(HttpUser): + wait_time = between(0.5, 1) # Random wait time between requests + + def on_start(self): + self.api_key = os.getenv('API_KEY', 'sk-1234567890') + self.client.headers.update({'Authorization': f'Bearer {self.api_key}'}) + + @task + def litellm_completion(self): + # no cache hits with this + payload = { + "model": "db-openai-endpoint", + "messages": [{"role": "user", "content": f"{uuid.uuid4()} This is a test there will be no cache hits and we'll fill up the context" * 150}], + "user": "my-new-end-user-1" + } + response = self.client.post("chat/completions", json=payload) + + if response.status_code != 200: + # log the errors in error.txt + with open("error.txt", "a") as error_log: + error_log.write(response.text + "\n") +``` + ## Logging Callbacks diff --git a/docs/my-website/docs/projects/HolmesGPT.md b/docs/my-website/docs/projects/HolmesGPT.md new file mode 100644 index 00000000000..608d526368f --- /dev/null +++ b/docs/my-website/docs/projects/HolmesGPT.md @@ -0,0 +1,7 @@ +# HolmesGPT + +[HolmesGPT](https://github.com/robusta-dev/holmesgpt) is an AI-powered observability tool designed to enhance incident response and troubleshooting processes. It's like your 24/7 on-call assistant, helps you solve alerts faster with Automatic Correlations, Investigations, and More. + +LiteLLM helps HolmesGPT integrate with multiple LLM providers or bring their own model and self-host it. + +🔗 Try HolmesGPT → [https://github.com/robusta-dev/holmesgpt](https://github.com/robusta-dev/holmesgpt) \ No newline at end of file diff --git a/docs/my-website/docs/providers/azure/azure.md b/docs/my-website/docs/providers/azure/azure.md index 5317b744abe..ab4391798f8 100644 --- a/docs/my-website/docs/providers/azure/azure.md +++ b/docs/my-website/docs/providers/azure/azure.md @@ -618,23 +618,43 @@ curl --location 'http://0.0.0.0:4000/chat/completions' \ ### Azure AD Token Refresh - `DefaultAzureCredential` -Use this if you want to use Azure `DefaultAzureCredential` for Authentication on your requests +Use this if you want to use Azure `DefaultAzureCredential` for Authentication on your requests. `DefaultAzureCredential` automatically discovers and uses available Azure credentials from multiple sources. +**Option 1: Explicit DefaultAzureCredential (Recommended)** ```python from litellm import completion from azure.identity import DefaultAzureCredential, get_bearer_token_provider +# DefaultAzureCredential automatically discovers credentials from: +# - Environment variables (AZURE_CLIENT_ID, AZURE_CLIENT_SECRET, AZURE_TENANT_ID) +# - Managed Identity (AKS, Azure VMs, etc.) +# - Azure CLI credentials +# - And other Azure identity sources token_provider = get_bearer_token_provider(DefaultAzureCredential(), "https://cognitiveservices.azure.com/.default") - response = completion( model = "azure/", # model = azure/ api_base = "", # azure api base api_version = "", # azure api version - azure_ad_token_provider=token_provider + azure_ad_token_provider=token_provider, + messages = [{"role": "user", "content": "good morning"}], +) +``` + +**Option 2: LiteLLM Auto-Fallback to DefaultAzureCredential** +```python +import litellm + +# Enable automatic fallback to DefaultAzureCredential +litellm.enable_azure_ad_token_refresh = True + +response = litellm.completion( + model = "azure/", + api_base = "", + api_version = "", messages = [{"role": "user", "content": "good morning"}], ) ``` @@ -642,6 +662,8 @@ response = completion( +**Scenario 1: With Environment Variables (Traditional)** + 1. Add relevant env vars ```bash @@ -663,12 +685,48 @@ litellm_settings: enable_azure_ad_token_refresh: true # 👈 KEY CHANGE ``` +**Scenario 2: Managed Identity (AKS, Azure VMs) - No Hard-coded Credentials Required** + +Perfect for AKS clusters, Azure VMs, or other managed environments where Azure automatically injects credentials. + +```yaml +model_list: + - model_name: gpt-3.5-turbo + litellm_params: + model: azure/your-deployment-name + api_base: https://openai-gpt-4-test-v-1.openai.azure.com/ + +litellm_settings: + enable_azure_ad_token_refresh: true # 👈 KEY CHANGE +``` + +**Scenario 3: Azure CLI Authentication** + +If you're authenticated via `az login`, no additional configuration needed: + +```yaml +model_list: + - model_name: gpt-3.5-turbo + litellm_params: + model: azure/your-deployment-name + api_base: https://openai-gpt-4-test-v-1.openai.azure.com/ + +litellm_settings: + enable_azure_ad_token_refresh: true # 👈 KEY CHANGE +``` + 3. Start proxy ```bash litellm --config /path/to/config.yaml ``` +**How it works**: +- LiteLLM first tries Service Principal authentication (if environment variables are available) +- If that fails, it automatically falls back to `DefaultAzureCredential` +- `DefaultAzureCredential` will use Managed Identity, Azure CLI credentials, or other available Azure identity sources +- This eliminates the need for hard-coded credentials in managed environments like AKS + diff --git a/docs/my-website/docs/providers/groq.md b/docs/my-website/docs/providers/groq.md index 5b74151d736..59668b5eb5f 100644 --- a/docs/my-website/docs/providers/groq.md +++ b/docs/my-website/docs/providers/groq.md @@ -158,7 +158,7 @@ We support ALL Groq models, just set `groq/` as a prefix when sending completion | mixtral-8x7b-32768 | `completion(model="groq/mixtral-8x7b-32768", messages)` | | gemma-7b-it | `completion(model="groq/gemma-7b-it", messages)` | | moonshotai/kimi-k2-instruct | `completion(model="groq/moonshotai/kimi-k2-instruct", messages)` | -| qwen-qwq-32b | `completion(model="groq/qwen-qwq-32b", messages)` | +| qwen3-32b | `completion(model="groq/qwen/qwen3-32b", messages)` | ## Groq - Tool / Function Calling Example diff --git a/docs/my-website/docs/providers/hyperbolic.md b/docs/my-website/docs/providers/hyperbolic.md new file mode 100644 index 00000000000..7bad527fcfe --- /dev/null +++ b/docs/my-website/docs/providers/hyperbolic.md @@ -0,0 +1,331 @@ +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +# Hyperbolic + +## Overview + +| Property | Details | +|-------|-------| +| Description | Hyperbolic provides access to the latest models at a fraction of legacy cloud costs, with OpenAI-compatible APIs for LLMs, image generation, and more. | +| Provider Route on LiteLLM | `hyperbolic/` | +| Link to Provider Doc | [Hyperbolic Documentation ↗](https://docs.hyperbolic.xyz) | +| Base URL | `https://api.hyperbolic.xyz/v1` | +| Supported Operations | [`/chat/completions`](#sample-usage) | + +
+
+ +https://docs.hyperbolic.xyz + +**We support ALL Hyperbolic models, just set `hyperbolic/` as a prefix when sending completion requests** + +## Available Models + +### Language Models + +| Model | Description | Context Window | Pricing per 1M tokens | +|-------|-------------|----------------|----------------------| +| `hyperbolic/deepseek-ai/DeepSeek-V3` | DeepSeek V3 - Fast and efficient | 131,072 tokens | $0.25 | +| `hyperbolic/deepseek-ai/DeepSeek-V3-0324` | DeepSeek V3 March 2024 version | 131,072 tokens | $0.25 | +| `hyperbolic/deepseek-ai/DeepSeek-R1` | DeepSeek R1 - Reasoning model | 131,072 tokens | $2.00 | +| `hyperbolic/deepseek-ai/DeepSeek-R1-0528` | DeepSeek R1 May 2028 version | 131,072 tokens | $0.25 | +| `hyperbolic/Qwen/Qwen2.5-72B-Instruct` | Qwen 2.5 72B Instruct | 131,072 tokens | $0.40 | +| `hyperbolic/Qwen/Qwen2.5-Coder-32B-Instruct` | Qwen 2.5 Coder 32B for code generation | 131,072 tokens | $0.20 | +| `hyperbolic/Qwen/Qwen3-235B-A22B` | Qwen 3 235B A22B variant | 131,072 tokens | $2.00 | +| `hyperbolic/Qwen/QwQ-32B` | Qwen QwQ 32B | 131,072 tokens | $0.20 | +| `hyperbolic/meta-llama/Llama-3.3-70B-Instruct` | Llama 3.3 70B Instruct | 131,072 tokens | $0.80 | +| `hyperbolic/meta-llama/Meta-Llama-3.1-405B-Instruct` | Llama 3.1 405B Instruct | 131,072 tokens | $5.00 | +| `hyperbolic/moonshotai/Kimi-K2-Instruct` | Kimi K2 Instruct | 131,072 tokens | $2.00 | + +## Required Variables + +```python showLineNumbers title="Environment Variables" +os.environ["HYPERBOLIC_API_KEY"] = "" # your Hyperbolic API key +``` + +Get your API key from [Hyperbolic dashboard](https://app.hyperbolic.ai). + +## Usage - LiteLLM Python SDK + +### Non-streaming + +```python showLineNumbers title="Hyperbolic Non-streaming Completion" +import os +import litellm +from litellm import completion + +os.environ["HYPERBOLIC_API_KEY"] = "" # your Hyperbolic API key + +messages = [{"content": "What is the capital of France?", "role": "user"}] + +# Hyperbolic call +response = completion( + model="hyperbolic/Qwen/Qwen2.5-72B-Instruct", + messages=messages +) + +print(response) +``` + +### Streaming + +```python showLineNumbers title="Hyperbolic Streaming Completion" +import os +import litellm +from litellm import completion + +os.environ["HYPERBOLIC_API_KEY"] = "" # your Hyperbolic API key + +messages = [{"content": "Write a short poem about AI", "role": "user"}] + +# Hyperbolic call with streaming +response = completion( + model="hyperbolic/deepseek-ai/DeepSeek-V3", + messages=messages, + stream=True +) + +for chunk in response: + print(chunk) +``` + +### Function Calling + +```python showLineNumbers title="Hyperbolic Function Calling" +import os +import litellm +from litellm import completion + +os.environ["HYPERBOLIC_API_KEY"] = "" # your Hyperbolic API key + +tools = [ + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get the current weather in a location", + "parameters": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA" + }, + "unit": { + "type": "string", + "enum": ["celsius", "fahrenheit"] + } + }, + "required": ["location"] + } + } + } +] + +response = completion( + model="hyperbolic/deepseek-ai/DeepSeek-V3", + messages=[{"role": "user", "content": "What's the weather like in New York?"}], + tools=tools, + tool_choice="auto" +) + +print(response) +``` + +## Usage - LiteLLM Proxy + +Add the following to your LiteLLM Proxy configuration file: + +```yaml showLineNumbers title="config.yaml" +model_list: + - model_name: deepseek-fast + litellm_params: + model: hyperbolic/deepseek-ai/DeepSeek-V3 + api_key: os.environ/HYPERBOLIC_API_KEY + + - model_name: qwen-coder + litellm_params: + model: hyperbolic/Qwen/Qwen2.5-Coder-32B-Instruct + api_key: os.environ/HYPERBOLIC_API_KEY + + - model_name: deepseek-reasoning + litellm_params: + model: hyperbolic/deepseek-ai/DeepSeek-R1 + api_key: os.environ/HYPERBOLIC_API_KEY +``` + +Start your LiteLLM Proxy server: + +```bash showLineNumbers title="Start LiteLLM Proxy" +litellm --config config.yaml + +# RUNNING on http://0.0.0.0:4000 +``` + + + + +```python showLineNumbers title="Hyperbolic via Proxy - Non-streaming" +from openai import OpenAI + +# Initialize client with your proxy URL +client = OpenAI( + base_url="http://localhost:4000", # Your proxy URL + api_key="your-proxy-api-key" # Your proxy API key +) + +# Non-streaming response +response = client.chat.completions.create( + model="deepseek-fast", + messages=[{"role": "user", "content": "Explain quantum computing in simple terms"}] +) + +print(response.choices[0].message.content) +``` + +```python showLineNumbers title="Hyperbolic via Proxy - Streaming" +from openai import OpenAI + +# Initialize client with your proxy URL +client = OpenAI( + base_url="http://localhost:4000", # Your proxy URL + api_key="your-proxy-api-key" # Your proxy API key +) + +# Streaming response +response = client.chat.completions.create( + model="qwen-coder", + messages=[{"role": "user", "content": "Write a Python function to sort a list"}], + stream=True +) + +for chunk in response: + if chunk.choices[0].delta.content is not None: + print(chunk.choices[0].delta.content, end="") +``` + + + + + +```python showLineNumbers title="Hyperbolic via Proxy - LiteLLM SDK" +import litellm + +# Configure LiteLLM to use your proxy +response = litellm.completion( + model="litellm_proxy/deepseek-fast", + messages=[{"role": "user", "content": "What are the benefits of renewable energy?"}], + api_base="http://localhost:4000", + api_key="your-proxy-api-key" +) + +print(response.choices[0].message.content) +``` + +```python showLineNumbers title="Hyperbolic via Proxy - LiteLLM SDK Streaming" +import litellm + +# Configure LiteLLM to use your proxy with streaming +response = litellm.completion( + model="litellm_proxy/qwen-coder", + messages=[{"role": "user", "content": "Implement a binary search algorithm"}], + api_base="http://localhost:4000", + api_key="your-proxy-api-key", + stream=True +) + +for chunk in response: + if hasattr(chunk.choices[0], 'delta') and chunk.choices[0].delta.content is not None: + print(chunk.choices[0].delta.content, end="") +``` + + + + + +```bash showLineNumbers title="Hyperbolic via Proxy - cURL" +curl http://localhost:4000/v1/chat/completions \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer your-proxy-api-key" \ + -d '{ + "model": "deepseek-fast", + "messages": [{"role": "user", "content": "What is machine learning?"}] + }' +``` + +```bash showLineNumbers title="Hyperbolic via Proxy - cURL Streaming" +curl http://localhost:4000/v1/chat/completions \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer your-proxy-api-key" \ + -d '{ + "model": "qwen-coder", + "messages": [{"role": "user", "content": "Write a REST API in Python"}], + "stream": true + }' +``` + + + + +For more detailed information on using the LiteLLM Proxy, see the [LiteLLM Proxy documentation](../providers/litellm_proxy). + +## Supported OpenAI Parameters + +Hyperbolic supports the following OpenAI-compatible parameters: + +| Parameter | Type | Description | +|-----------|------|-------------| +| `messages` | array | **Required**. Array of message objects with 'role' and 'content' | +| `model` | string | **Required**. Model ID (e.g., deepseek-ai/DeepSeek-V3, Qwen/Qwen2.5-72B-Instruct) | +| `stream` | boolean | Optional. Enable streaming responses | +| `temperature` | float | Optional. Sampling temperature (0.0 to 2.0) | +| `top_p` | float | Optional. Nucleus sampling parameter | +| `max_tokens` | integer | Optional. Maximum tokens to generate | +| `frequency_penalty` | float | Optional. Penalize frequent tokens | +| `presence_penalty` | float | Optional. Penalize tokens based on presence | +| `stop` | string/array | Optional. Stop sequences | +| `n` | integer | Optional. Number of completions to generate | +| `tools` | array | Optional. List of available tools/functions | +| `tool_choice` | string/object | Optional. Control tool/function calling | +| `response_format` | object | Optional. Response format specification | +| `seed` | integer | Optional. Random seed for reproducibility | +| `user` | string | Optional. User identifier | + +## Advanced Usage + +### Custom API Base + +If you're using a custom Hyperbolic deployment: + +```python showLineNumbers title="Custom API Base" +import litellm + +response = litellm.completion( + model="hyperbolic/deepseek-ai/DeepSeek-V3", + messages=[{"role": "user", "content": "Hello"}], + api_base="https://your-custom-hyperbolic-endpoint.com/v1", + api_key="your-api-key" +) +``` + +### Rate Limits + +Hyperbolic offers different tiers: +- **Basic**: 60 requests per minute (RPM) +- **Pro**: 600 RPM +- **Enterprise**: Custom limits + +## Pricing + +Hyperbolic offers competitive pay-as-you-go pricing with no hidden fees or long-term commitments. See the model table above for specific pricing per million tokens. + +### Precision Options +- **BF16**: Best precision and performance, suitable for tasks where accuracy is critical +- **FP8**: Optimized for efficiency and speed, ideal for high-throughput applications at lower cost + +## Additional Resources + +- [Hyperbolic Official Documentation](https://docs.hyperbolic.xyz) +- [Hyperbolic Dashboard](https://app.hyperbolic.ai) +- [API Reference](https://docs.hyperbolic.xyz/docs/rest-api) \ No newline at end of file diff --git a/docs/my-website/docs/providers/lambda_ai.md b/docs/my-website/docs/providers/lambda_ai.md new file mode 100644 index 00000000000..91800faab70 --- /dev/null +++ b/docs/my-website/docs/providers/lambda_ai.md @@ -0,0 +1,280 @@ +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +# Lambda AI + +## Overview + +| Property | Details | +|-------|-------| +| Description | Lambda AI provides access to a wide range of open-source language models through their cloud GPU infrastructure, optimized for inference at scale. | +| Provider Route on LiteLLM | `lambda_ai/` | +| Link to Provider Doc | [Lambda AI API Documentation ↗](https://docs.lambda.ai/api) | +| Base URL | `https://api.lambda.ai/v1` | +| Supported Operations | [`/chat/completions`](#sample-usage) | + +
+
+ +https://docs.lambda.ai/api + +**We support ALL Lambda AI models, just set `lambda_ai/` as a prefix when sending completion requests** + +## Available Models + +Lambda AI offers a diverse selection of state-of-the-art open-source models: + +### Large Language Models + +| Model | Description | Context Window | +|-------|-------------|----------------| +| `lambda_ai/llama3.3-70b-instruct-fp8` | Llama 3.3 70B with FP8 quantization | 8,192 tokens | +| `lambda_ai/llama3.1-405b-instruct-fp8` | Llama 3.1 405B with FP8 quantization | 8,192 tokens | +| `lambda_ai/llama3.1-70b-instruct-fp8` | Llama 3.1 70B with FP8 quantization | 8,192 tokens | +| `lambda_ai/llama3.1-8b-instruct` | Llama 3.1 8B instruction-tuned | 8,192 tokens | +| `lambda_ai/llama3.1-nemotron-70b-instruct-fp8` | Llama 3.1 Nemotron 70B | 8,192 tokens | + +### DeepSeek Models + +| Model | Description | Context Window | +|-------|-------------|----------------| +| `lambda_ai/deepseek-llama3.3-70b` | DeepSeek Llama 3.3 70B | 8,192 tokens | +| `lambda_ai/deepseek-r1-0528` | DeepSeek R1 0528 | 8,192 tokens | +| `lambda_ai/deepseek-r1-671b` | DeepSeek R1 671B | 8,192 tokens | +| `lambda_ai/deepseek-v3-0324` | DeepSeek V3 0324 | 8,192 tokens | + +### Hermes Models + +| Model | Description | Context Window | +|-------|-------------|----------------| +| `lambda_ai/hermes3-405b` | Hermes 3 405B | 8,192 tokens | +| `lambda_ai/hermes3-70b` | Hermes 3 70B | 8,192 tokens | +| `lambda_ai/hermes3-8b` | Hermes 3 8B | 8,192 tokens | + +### Coding Models + +| Model | Description | Context Window | +|-------|-------------|----------------| +| `lambda_ai/qwen25-coder-32b-instruct` | Qwen 2.5 Coder 32B | 8,192 tokens | +| `lambda_ai/qwen3-32b-fp8` | Qwen 3 32B with FP8 | 8,192 tokens | + +### Vision Models + +| Model | Description | Context Window | +|-------|-------------|----------------| +| `lambda_ai/llama3.2-11b-vision-instruct` | Llama 3.2 11B with vision capabilities | 8,192 tokens | + +### Specialized Models + +| Model | Description | Context Window | +|-------|-------------|----------------| +| `lambda_ai/llama-4-maverick-17b-128e-instruct-fp8` | Llama 4 Maverick with 128k context | 131,072 tokens | +| `lambda_ai/llama-4-scout-17b-16e-instruct` | Llama 4 Scout with 16k context | 16,384 tokens | +| `lambda_ai/lfm-40b` | LFM 40B model | 8,192 tokens | +| `lambda_ai/lfm-7b` | LFM 7B model | 8,192 tokens | + +## Required Variables + +```python showLineNumbers title="Environment Variables" +os.environ["LAMBDA_API_KEY"] = "" # your Lambda AI API key +``` + +## Usage - LiteLLM Python SDK + +### Non-streaming + +```python showLineNumbers title="Lambda AI Non-streaming Completion" +import os +import litellm +from litellm import completion + +os.environ["LAMBDA_API_KEY"] = "" # your Lambda AI API key + +messages = [{"content": "Hello, how are you?", "role": "user"}] + +# Lambda AI call +response = completion( + model="lambda_ai/llama3.1-8b-instruct", + messages=messages +) + +print(response) +``` + +### Streaming + +```python showLineNumbers title="Lambda AI Streaming Completion" +import os +import litellm +from litellm import completion + +os.environ["LAMBDA_API_KEY"] = "" # your Lambda AI API key + +messages = [{"content": "Write a short story about AI", "role": "user"}] + +# Lambda AI call with streaming +response = completion( + model="lambda_ai/llama3.1-70b-instruct-fp8", + messages=messages, + stream=True +) + +for chunk in response: + print(chunk) +``` + +### Vision/Multimodal Support + +The Llama 3.2 Vision model supports image inputs: + +```python showLineNumbers title="Lambda AI Vision/Multimodal" +import os +import litellm +from litellm import completion + +os.environ["LAMBDA_API_KEY"] = "" # your Lambda AI API key + +messages = [{ + "role": "user", + "content": [ + { + "type": "text", + "text": "What's in this image?" + }, + { + "type": "image_url", + "image_url": { + "url": "https://example.com/image.jpg" + } + } + ] +}] + +# Lambda AI vision model call +response = completion( + model="lambda_ai/llama3.2-11b-vision-instruct", + messages=messages +) + +print(response) +``` + +### Function Calling + +Lambda AI models support function calling: + +```python showLineNumbers title="Lambda AI Function Calling" +import os +import litellm +from litellm import completion + +os.environ["LAMBDA_API_KEY"] = "" # your Lambda AI API key + +# Define tools +tools = [{ + "type": "function", + "function": { + "name": "get_weather", + "description": "Get the current weather in a location", + "parameters": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA" + } + }, + "required": ["location"] + } + } +}] + +messages = [{"role": "user", "content": "What's the weather in Boston?"}] + +# Lambda AI call with function calling +response = completion( + model="lambda_ai/hermes3-70b", + messages=messages, + tools=tools, + tool_choice="auto" +) + +print(response) +``` + +## Usage - LiteLLM Proxy Server + +```yaml showLineNumbers title="config.yaml" +model_list: + - model_name: llama-8b + litellm_params: + model: lambda_ai/llama3.1-8b-instruct + api_key: os.environ/LAMBDA_API_KEY + - model_name: deepseek-70b + litellm_params: + model: lambda_ai/deepseek-llama3.3-70b + api_key: os.environ/LAMBDA_API_KEY + - model_name: hermes-405b + litellm_params: + model: lambda_ai/hermes3-405b + api_key: os.environ/LAMBDA_API_KEY + - model_name: qwen-coder + litellm_params: + model: lambda_ai/qwen25-coder-32b-instruct + api_key: os.environ/LAMBDA_API_KEY +``` + +## Custom API Base + +If you need to use a custom API base URL: + +```python showLineNumbers title="Custom API Base" +import os +import litellm +from litellm import completion + +# Using environment variable +os.environ["LAMBDA_API_BASE"] = "https://custom.lambda-api.com/v1" +os.environ["LAMBDA_API_KEY"] = "" # your API key + +# Or pass directly +response = completion( + model="lambda_ai/llama3.1-8b-instruct", + messages=[{"content": "Hello!", "role": "user"}], + api_base="https://custom.lambda-api.com/v1", + api_key="your-api-key" +) +``` + +## Supported OpenAI Parameters + +Lambda AI supports all standard OpenAI parameters since it's fully OpenAI-compatible: + +- `temperature` +- `max_tokens` +- `top_p` +- `frequency_penalty` +- `presence_penalty` +- `stop` +- `n` +- `stream` +- `tools` +- `tool_choice` +- `response_format` +- `seed` +- `user` +- `logit_bias` + +Example with parameters: + +```python showLineNumbers title="Lambda AI with Parameters" +response = completion( + model="lambda_ai/hermes3-405b", + messages=[{"content": "Explain quantum computing", "role": "user"}], + temperature=0.7, + max_tokens=500, + top_p=0.9, + frequency_penalty=0.2, + presence_penalty=0.1 +) +``` \ No newline at end of file diff --git a/docs/my-website/docs/providers/moonshot.md b/docs/my-website/docs/providers/moonshot.md index ee4f04bd7a0..2e00bae3551 100644 --- a/docs/my-website/docs/providers/moonshot.md +++ b/docs/my-website/docs/providers/moonshot.md @@ -10,7 +10,7 @@ import TabItem from '@theme/TabItem'; | Description | Moonshot AI provides large language models including the moonshot-v1 series and kimi models. | | Provider Route on LiteLLM | `moonshot/` | | Link to Provider Doc | [Moonshot AI ↗](https://platform.moonshot.ai/) | -| Base URL | `https://api.moonshot.cn/` | +| Base URL | `https://api.moonshot.ai/` | | Supported Operations | [`/chat/completions`](#sample-usage) |
@@ -26,6 +26,18 @@ https://platform.moonshot.ai/ os.environ["MOONSHOT_API_KEY"] = "" # your Moonshot AI API key ``` +**ATTENTION:** + +Moonshot AI offers two distinct API endpoints: a global one and a China-specific one. +- Global API Base URL: `https://api.moonshot.ai/v1` (This is the one currently implemented) +- China API Base URL: `https://api.moonshot.cn/v1` + +You can overwrite the base url with: + +``` +os.environ["MOONSHOT_API_BASE"] = "https://api.moonshot.cn/v1" +``` + ## Usage - LiteLLM Python SDK ### Non-streaming diff --git a/docs/my-website/docs/providers/morph.md b/docs/my-website/docs/providers/morph.md new file mode 100644 index 00000000000..e49c60b5665 --- /dev/null +++ b/docs/my-website/docs/providers/morph.md @@ -0,0 +1,123 @@ +# Morph + +LiteLLM supports all models on [Morph](https://morphllm.com) + +## Overview + +Morph provides specialized AI models designed for agentic workflows, particularly excelling at precise code editing and manipulation. Their "Apply" models enable targeted code changes without full file rewrites, making them ideal for AI agents that need to make intelligent, context-aware code modifications. + +## API Key +```python +import os +os.environ["MORPH_API_KEY"] = "your-api-key" +``` + +## Sample Usage + +```python +from litellm import completion + +# set env variable +os.environ["MORPH_API_KEY"] = "your-api-key" + +messages = [ + {"role": "user", "content": "Write a Python function to calculate factorial"} +] + +## Morph v3 Fast - Optimized for speed +response = completion( + model="morph/morph-v3-fast", + messages=messages, +) +print(response) + +## Morph v3 Large - Most capable model +response = completion( + model="morph/morph-v3-large", + messages=messages, +) +print(response) +``` + +## Sample Usage - Streaming +```python +from litellm import completion + +# set env variable +os.environ["MORPH_API_KEY"] = "your-api-key" + +messages = [ + {"role": "user", "content": "Write a Python function to calculate factorial"} +] + +## Morph v3 Fast with streaming +response = completion( + model="morph/morph-v3-fast", + messages=messages, + stream=True, +) + +for chunk in response: + print(chunk) +``` + +## Supported Models + +| Model Name | Function Call | Description | Context Window | +|--------------------------|--------------------------------------------|-----------------------|----------------| +| morph-v3-fast | `completion('morph/morph-v3-fast', messages)` | Fastest model, optimized for quick responses | 16k tokens | +| morph-v3-large | `completion('morph/morph-v3-large', messages)` | Most capable model for complex tasks | 16k tokens | + +## Usage - LiteLLM Proxy Server + +Here's how to use Morph with the LiteLLM Proxy Server: + +1. Save API key in your environment +```bash +export MORPH_API_KEY="your-api-key" +``` + +2. Add model to config.yaml +```yaml +model_list: + - model_name: morph-v3-fast + litellm_params: + model: morph/morph-v3-fast + + - model_name: morph-v3-large + litellm_params: + model: morph/morph-v3-large +``` + +3. Start the proxy server +```bash +litellm --config config.yaml +``` + +## Advanced Usage + +### Setting API Base +```python +import litellm + +# set custom api base +response = completion( + model="morph/morph-v3-large", + messages=[{"role": "user", "content": "Hello, world!"}], + api_base="https://api.morphllm.com/v1" +) +print(response) +``` + +### Setting API Key +```python +import litellm + +# set api key via completion +response = completion( + model="morph/morph-v3-large", + messages=[{"role": "user", "content": "Hello, world!"}], + api_key="your-api-key" +) +print(response) +``` \ No newline at end of file diff --git a/docs/my-website/docs/providers/vertex_partner.md b/docs/my-website/docs/providers/vertex_partner.md index 2600b179f6d..c6e324f2958 100644 --- a/docs/my-website/docs/providers/vertex_partner.md +++ b/docs/my-website/docs/providers/vertex_partner.md @@ -313,11 +313,22 @@ curl --location 'http://0.0.0.0:4000/chat/completions' \ ## VertexAI Mistral API [**Supported OpenAI Params**](https://github.com/BerriAI/litellm/blob/e0f3cd580cb85066f7d36241a03c30aa50a8a31d/litellm/llms/openai.py#L137) + +**LiteLLM Supports all Vertex AI Mistral Models.** Ensure you use the `vertex_ai/mistral-` prefix for all Vertex AI Mistral models. + +Overview + +| Property | Details | +|----------|---------| +| Provider Route | `vertex_ai/mistral-{MODEL}` | +| Vertex Documentation | [Vertex AI - Mistral Models](https://cloud.google.com/vertex-ai/generative-ai/docs/partner-models/mistral) | | Model Name | Function Call | |------------------|--------------------------------------| | mistral-large@latest | `completion('vertex_ai/mistral-large@latest', messages)` | | mistral-large@2407 | `completion('vertex_ai/mistral-large@2407', messages)` | +| mistral-small-2503 | `completion('vertex_ai/mistral-small-2503', messages)` | +| mistral-large-2411 | `completion('vertex_ai/mistral-large-2411', messages)` | | mistral-nemo@latest | `completion('vertex_ai/mistral-nemo@latest', messages)` | | codestral@latest | `completion('vertex_ai/codestral@latest', messages)` | | codestral@@2405 | `completion('vertex_ai/codestral@2405', messages)` | diff --git a/docs/my-website/docs/proxy/config_settings.md b/docs/my-website/docs/proxy/config_settings.md index b1dfac9ac29..8e802f5c689 100644 --- a/docs/my-website/docs/proxy/config_settings.md +++ b/docs/my-website/docs/proxy/config_settings.md @@ -189,6 +189,7 @@ general_settings: | proxy_budget_rescheduler_min_time | int | The minimum time (in seconds) to wait before checking db for budget resets. **Default is 597 seconds** | | proxy_budget_rescheduler_max_time | int | The maximum time (in seconds) to wait before checking db for budget resets. **Default is 605 seconds** | | proxy_batch_write_at | int | Time (in seconds) to wait before batch writing spend logs to the db. **Default is 10 seconds** | +| proxy_batch_polling_interval | int | Time (in seconds) to wait before polling a batch, to check if it's completed. **Default is 6000 seconds (1 hour)** | | alerting_args | dict | Args for Slack Alerting [Doc on Slack Alerting](./alerting.md) | | custom_key_generate | str | Custom function for key generation [Doc on custom key generation](./virtual_keys.md#custom--key-generate) | | allowed_ips | List[str] | List of IPs allowed to access the proxy. If not set, all IPs are allowed. | @@ -622,6 +623,7 @@ router_settings: | PROXY_ADMIN_ID | Admin identifier for proxy server | PROXY_BASE_URL | Base URL for proxy service | PROXY_BATCH_WRITE_AT | Time in seconds to wait before batch writing spend logs to the database. Default is 10 +| PROXY_BATCH_POLLING_INTERVAL | Time in seconds to wait before polling a batch, to check if it's completed. Default is 6000s (1 hour) | PROXY_BUDGET_RESCHEDULER_MAX_TIME | Maximum time in seconds to wait before checking database for budget resets. Default is 605 | PROXY_BUDGET_RESCHEDULER_MIN_TIME | Minimum time in seconds to wait before checking database for budget resets. Default is 597 | PROXY_LOGOUT_URL | URL for logging out of the proxy service diff --git a/docs/my-website/docs/proxy/guardrails/model_armor.md b/docs/my-website/docs/proxy/guardrails/model_armor.md new file mode 100644 index 00000000000..a7463a8eee3 --- /dev/null +++ b/docs/my-website/docs/proxy/guardrails/model_armor.md @@ -0,0 +1,93 @@ +import Image from '@theme/IdealImage'; +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +# Google Cloud Model Armor + +LiteLLM supports Google Cloud Model Armor guardrails via the [Model Armor API](https://cloud.google.com/security-command-center/docs/model-armor-overview). + + +## Supported Guardrails + +- [Model Armor Templates](https://cloud.google.com/security-command-center/docs/manage-model-armor-templates) - Content sanitization and blocking based on configured templates + +## Quick Start +### 1. Define Guardrails on your LiteLLM config.yaml + +Define your guardrails under the `guardrails` section + +```yaml +model_list: + - model_name: gpt-3.5-turbo + litellm_params: + model: openai/gpt-3.5-turbo + api_key: os.environ/OPENAI_API_KEY + +guardrails: + - guardrail_name: model-armor-shield + litellm_params: + guardrail: model_armor + mode: [pre_call, post_call] # Run on both input and output + template_id: "your-template-id" # Required: Your Model Armor template ID + project_id: "your-project-id" # Your GCP project ID + location: "us-central1" # GCP location (default: us-central1) + credentials: "path/to/credentials.json" # Path to service account key + mask_request_content: true # Enable request content masking + mask_response_content: true # Enable response content masking + fail_on_error: true # Fail request if Model Armor errors (default: true) + default_on: true # Run by default for all requests +``` + +#### Supported values for `mode` + +- `pre_call` Run **before** LLM call, on **input** +- `post_call` Run **after** LLM call, on **input & output** + +### 2. Start LiteLLM Gateway + + +```shell +litellm --config config.yaml --detailed_debug +``` + +### 3. Test request + +**[Langchain, OpenAI SDK Usage Examples](../proxy/user_keys#request-format)** + +```shell +curl -i http://localhost:4000/v1/chat/completions \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer sk-npnwjPQciVRok5yNZgKmFQ" \ + -d '{ + "model": "gpt-3.5-turbo", + "messages": [ + {"role": "user", "content": "Hi, my email is test@example.com"} + ], + "guardrails": ["model-armor-shield"] + }' +``` + +## Supported Params + +### Common Params + +- `api_key` - str - Google Cloud service account credentials (optional if using ADC) +- `api_base` - str - Custom Model Armor API endpoint (optional) +- `default_on` - bool - Whether to run the guardrail by default. Default is `false`. +- `mode` - Union[str, list[str]] - Mode to run the guardrail. Either `pre_call` or `post_call`. Default is `pre_call`. + +### Model Armor Specific + +- `template_id` - str - The ID of your Model Armor template (required) +- `project_id` - str - Google Cloud project ID (defaults to credentials project) +- `location` - str - Google Cloud location/region. Default is `us-central1` +- `credentials` - Union[str, dict] - Path to service account JSON file or credentials dictionary +- `api_endpoint` - str - Custom API endpoint for Model Armor (optional) +- `fail_on_error` - bool - Whether to fail requests if Model Armor encounters errors. Default is `true` +- `mask_request_content` - bool - Enable masking of sensitive content in requests. Default is `false` +- `mask_response_content` - bool - Enable masking of sensitive content in responses. Default is `false` + + +## Further Reading + +- [Control Guardrails per API Key](./quick_start#-control-guardrails-per-api-key) \ No newline at end of file diff --git a/docs/my-website/package.json b/docs/my-website/package.json index d6d35d31340..24d212ea2c6 100644 --- a/docs/my-website/package.json +++ b/docs/my-website/package.json @@ -47,6 +47,7 @@ "node": ">=16.14" }, "overrides": { - "webpack-dev-server": ">=5.2.1" + "webpack-dev-server": ">=5.2.1", + "form-data": ">=4.0.4" } } diff --git a/docs/my-website/sidebars.js b/docs/my-website/sidebars.js index 54ff6461c69..1fe19fe979d 100644 --- a/docs/my-website/sidebars.js +++ b/docs/my-website/sidebars.js @@ -39,6 +39,7 @@ const sidebars = { "proxy/guardrails/lasso_security", "proxy/guardrails/guardrails_ai", "proxy/guardrails/lakera_ai", + "proxy/guardrails/model_armor", "proxy/guardrails/openai_moderation", "proxy/guardrails/pangea", "proxy/guardrails/pii_masking_v2", @@ -411,6 +412,7 @@ const sidebars = { "providers/huggingface_rerank", ] }, + "providers/hyperbolic", "providers/databricks", "providers/deepgram", "providers/watsonx", @@ -448,6 +450,8 @@ const sidebars = { "providers/replicate", "providers/togetherai", "providers/v0", + "providers/morph", + "providers/lambda_ai", "providers/novita", "providers/voyage", "providers/jina_ai", @@ -624,6 +628,7 @@ const sidebars = { "projects/llm_cord", "projects/pgai", "projects/GPTLocalhost", + "projects/HolmesGPT" ], }, "extras/code_quality", diff --git a/enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py b/enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py index d8b8efeef47..6edd198cd8e 100644 --- a/enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py +++ b/enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py @@ -83,15 +83,25 @@ class CheckBatchCost: ) continue - response = await self.llm_router.aretrieve_batch( - model=model_id, - batch_id=batch_id, - litellm_metadata={ - "user_api_key_user_id": job.created_by or "default-user-id", - "batch_ignore_default_logging": True, - }, + verbose_proxy_logger.info( + f"Querying model ID: {model_id} for cost and usage of batch ID: {batch_id}" ) + try: + response = await self.llm_router.aretrieve_batch( + model=model_id, + batch_id=batch_id, + litellm_metadata={ + "user_api_key_user_id": job.created_by or "default-user-id", + "batch_ignore_default_logging": True, + }, + ) + except Exception as e: + verbose_proxy_logger.info( + f"Skipping job {unified_object_id} because of error querying model ID: {model_id} for cost and usage of batch ID: {batch_id}: {e}" + ) + continue + ## RETRIEVE THE BATCH JOB OUTPUT FILE managed_files_obj = cast( Optional[_PROXY_LiteLLMManagedFiles], @@ -102,6 +112,9 @@ class CheckBatchCost: and response.output_file_id is not None and managed_files_obj is not None ): + verbose_proxy_logger.info( + f"Batch ID: {batch_id} is complete, tracking cost and usage" + ) # track cost model_file_id_mapping = { response.output_file_id: {model_id: response.output_file_id} diff --git a/litellm/__init__.py b/litellm/__init__.py index 192b210e865..c056e667269 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -144,22 +144,22 @@ prometheus_initialize_budget_metrics: Optional[bool] = False require_auth_for_metrics_endpoint: Optional[bool] = False argilla_batch_size: Optional[int] = None datadog_use_v1: Optional[bool] = False # if you want to use v1 datadog logged payload. -gcs_pub_sub_use_v1: Optional[bool] = ( - False # if you want to use v1 gcs pubsub logged payload -) -generic_api_use_v1: Optional[bool] = ( - False # if you want to use v1 generic api logged payload -) +gcs_pub_sub_use_v1: Optional[ + bool +] = False # if you want to use v1 gcs pubsub logged payload +generic_api_use_v1: Optional[ + bool +] = False # if you want to use v1 generic api logged payload argilla_transformation_object: Optional[Dict[str, Any]] = None -_async_input_callback: List[Union[str, Callable, CustomLogger]] = ( - [] -) # internal variable - async custom callbacks are routed here. -_async_success_callback: List[Union[str, Callable, CustomLogger]] = ( - [] -) # internal variable - async custom callbacks are routed here. -_async_failure_callback: List[Union[str, Callable, CustomLogger]] = ( - [] -) # internal variable - async custom callbacks are routed here. +_async_input_callback: List[ + Union[str, Callable, CustomLogger] +] = [] # internal variable - async custom callbacks are routed here. +_async_success_callback: List[ + Union[str, Callable, CustomLogger] +] = [] # internal variable - async custom callbacks are routed here. +_async_failure_callback: List[ + Union[str, Callable, CustomLogger] +] = [] # internal variable - async custom callbacks are routed here. pre_call_rules: List[Callable] = [] post_call_rules: List[Callable] = [] turn_off_message_logging: Optional[bool] = False @@ -167,18 +167,18 @@ log_raw_request_response: bool = False redact_messages_in_exceptions: Optional[bool] = False redact_user_api_key_info: Optional[bool] = False filter_invalid_headers: Optional[bool] = False -add_user_information_to_llm_headers: Optional[bool] = ( - None # adds user_id, team_id, token hash (params from StandardLoggingMetadata) to request headers -) +add_user_information_to_llm_headers: Optional[ + bool +] = None # adds user_id, team_id, token hash (params from StandardLoggingMetadata) to request headers store_audit_logs = False # Enterprise feature, allow users to see audit logs ### end of callbacks ############# -email: Optional[str] = ( - None # Not used anymore, will be removed in next MAJOR release - https://github.com/BerriAI/litellm/discussions/648 -) -token: Optional[str] = ( - None # Not used anymore, will be removed in next MAJOR release - https://github.com/BerriAI/litellm/discussions/648 -) +email: Optional[ + str +] = None # Not used anymore, will be removed in next MAJOR release - https://github.com/BerriAI/litellm/discussions/648 +token: Optional[ + str +] = None # Not used anymore, will be removed in next MAJOR release - https://github.com/BerriAI/litellm/discussions/648 telemetry = True max_tokens: int = DEFAULT_MAX_TOKENS # OpenAI Defaults drop_params = bool(os.getenv("LITELLM_DROP_PARAMS", False)) @@ -266,15 +266,11 @@ enable_loadbalancing_on_batch_endpoints: Optional[bool] = None enable_caching_on_provider_specific_optional_params: bool = ( False # feature-flag for caching on optional params - e.g. 'top_k' ) -caching: bool = ( - False # Not used anymore, will be removed in next MAJOR release - https://github.com/BerriAI/litellm/discussions/648 -) -caching_with_models: bool = ( - False # # Not used anymore, will be removed in next MAJOR release - https://github.com/BerriAI/litellm/discussions/648 -) -cache: Optional[Cache] = ( - None # cache object <- use this - https://docs.litellm.ai/docs/caching -) +caching: bool = False # Not used anymore, will be removed in next MAJOR release - https://github.com/BerriAI/litellm/discussions/648 +caching_with_models: bool = False # # Not used anymore, will be removed in next MAJOR release - https://github.com/BerriAI/litellm/discussions/648 +cache: Optional[ + Cache +] = None # cache object <- use this - https://docs.litellm.ai/docs/caching default_in_memory_ttl: Optional[float] = None default_redis_ttl: Optional[float] = None default_redis_batch_cache_expiry: Optional[float] = None @@ -282,9 +278,9 @@ model_alias_map: Dict[str, str] = {} model_group_alias_map: Dict[str, str] = {} model_group_settings: Optional["ModelGroupSettings"] = None max_budget: float = 0.0 # set the max budget across all providers -budget_duration: Optional[str] = ( - None # proxy only - resets budget after fixed duration. You can set duration as seconds ("30s"), minutes ("30m"), hours ("30h"), days ("30d"). -) +budget_duration: Optional[ + str +] = None # proxy only - resets budget after fixed duration. You can set duration as seconds ("30s"), minutes ("30m"), hours ("30h"), days ("30d"). default_soft_budget: float = ( DEFAULT_SOFT_BUDGET # by default all litellm proxy keys have a soft budget of 50.0 ) @@ -293,15 +289,11 @@ forward_traceparent_to_llm_provider: bool = False _current_cost = 0.0 # private variable, used if max budget is set error_logs: Dict = {} -add_function_to_prompt: bool = ( - False # if function calling not supported by api, append function call details to system prompt -) +add_function_to_prompt: bool = False # if function calling not supported by api, append function call details to system prompt client_session: Optional[httpx.Client] = None aclient_session: Optional[httpx.AsyncClient] = None model_fallbacks: Optional[List] = None # Deprecated for 'litellm.fallbacks' -model_cost_map_url: str = ( - "https://raw.githubusercontent.com/BerriAI/litellm/main/model_prices_and_context_window.json" -) +model_cost_map_url: str = "https://raw.githubusercontent.com/BerriAI/litellm/main/model_prices_and_context_window.json" suppress_debug_info = False dynamodb_table_name: Optional[str] = None s3_callback_params: Optional[Dict] = None @@ -329,9 +321,7 @@ prometheus_metrics_config: Optional[List] = None disable_add_prefix_to_prompt: bool = ( False # used by anthropic, to disable adding prefix to prompt ) -disable_copilot_system_to_assistant: bool = ( - False # If false (default), converts all 'system' role messages to 'assistant' for GitHub Copilot compatibility. Set to true to disable this behavior. -) +disable_copilot_system_to_assistant: bool = False # If false (default), converts all 'system' role messages to 'assistant' for GitHub Copilot compatibility. Set to true to disable this behavior. public_model_groups: Optional[List[str]] = None public_model_groups_links: Dict[str, str] = {} #### REQUEST PRIORITIZATION ##### @@ -339,17 +329,13 @@ priority_reservation: Optional[Dict[str, float]] = None ######## Networking Settings ######## -use_aiohttp_transport: bool = ( - True # Older variable, aiohttp is now the default. use disable_aiohttp_transport instead. -) +use_aiohttp_transport: bool = True # Older variable, aiohttp is now the default. use disable_aiohttp_transport instead. aiohttp_trust_env: bool = False # set to true to use HTTP_ Proxy settings disable_aiohttp_transport: bool = False # Set this to true to use httpx instead disable_aiohttp_trust_env: bool = ( False # When False, aiohttp will respect HTTP(S)_PROXY env vars ) -force_ipv4: bool = ( - False # when True, litellm will force ipv4 for all LLM requests. Some users have seen httpx ConnectionError when using ipv6. -) +force_ipv4: bool = False # when True, litellm will force ipv4 for all LLM requests. Some users have seen httpx ConnectionError when using ipv6. module_level_aclient = AsyncHTTPHandler( timeout=request_timeout, client_alias="module level aclient" ) @@ -363,13 +349,13 @@ fallbacks: Optional[List] = None context_window_fallbacks: Optional[List] = None content_policy_fallbacks: Optional[List] = None allowed_fails: int = 3 -num_retries_per_request: Optional[int] = ( - None # for the request overall (incl. fallbacks + model retries) -) +num_retries_per_request: Optional[ + int +] = None # for the request overall (incl. fallbacks + model retries) ####### SECRET MANAGERS ##################### -secret_manager_client: Optional[Any] = ( - None # list of instantiated key management clients - e.g. azure kv, infisical, etc. -) +secret_manager_client: Optional[ + Any +] = None # list of instantiated key management clients - e.g. azure kv, infisical, etc. _google_kms_resource_name: Optional[str] = None _key_management_system: Optional[KeyManagementSystem] = None _key_management_settings: KeyManagementSettings = KeyManagementSettings() @@ -505,6 +491,7 @@ moonshot_models: List = [] v0_models: List = [] morph_models: List = [] lambda_ai_models: List = [] +hyperbolic_models: List = [] recraft_models: List = [] def is_bedrock_pricing_only_model(key: str) -> bool: @@ -690,6 +677,8 @@ def add_known_models(): morph_models.append(key) elif value.get("litellm_provider") == "lambda_ai": lambda_ai_models.append(key) + elif value.get("litellm_provider") == "hyperbolic": + hyperbolic_models.append(key) elif value.get("litellm_provider") == "recraft": recraft_models.append(key) @@ -850,6 +839,7 @@ models_by_provider: dict = { "v0": v0_models, "morph": morph_models, "lambda_ai": lambda_ai_models, + "hyperbolic": hyperbolic_models, "recraft": recraft_models, } @@ -1171,6 +1161,9 @@ from .llms.nebius.chat.transformation import NebiusConfig from .llms.dashscope.chat.transformation import DashScopeChatConfig from .llms.moonshot.chat.transformation import MoonshotChatConfig from .llms.v0.chat.transformation import V0ChatConfig +from .llms.morph.chat.transformation import MorphChatConfig +from .llms.lambda_ai.chat.transformation import LambdaAIChatConfig +from .llms.hyperbolic.chat.transformation import HyperbolicChatConfig from .main import * # type: ignore from .integrations import * from .llms.custom_httpx.async_client_cleanup import close_litellm_async_clients @@ -1229,12 +1222,12 @@ from .types.llms.custom_llm import CustomLLMItem from .types.utils import GenericStreamingChunk custom_provider_map: List[CustomLLMItem] = [] -_custom_providers: List[str] = ( - [] -) # internal helper util, used to track names of custom providers -disable_hf_tokenizer_download: Optional[bool] = ( - None # disable huggingface tokenizer download. Defaults to openai clk100 -) +_custom_providers: List[ + str +] = [] # internal helper util, used to track names of custom providers +disable_hf_tokenizer_download: Optional[ + bool +] = None # disable huggingface tokenizer download. Defaults to openai clk100 global_disable_no_log_param: bool = False ### PASSTHROUGH ### diff --git a/litellm/constants.py b/litellm/constants.py index afdd95385cb..970b16617fe 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -279,6 +279,8 @@ LITELLM_CHAT_PROVIDERS = [ "dashscope", "moonshot", "v0", + "morph", + "lambda_ai", ] LITELLM_EMBEDDING_PROVIDERS_SUPPORTING_INPUT_ARRAY_OF_TOKENS = [ @@ -408,6 +410,9 @@ openai_compatible_endpoints: List = [ "https://dashscope-intl.aliyuncs.com/compatible-mode/v1", "https://api.moonshot.ai/v1", "https://api.v0.dev/v1", + "https://api.morphllm.com/v1", + "https://api.lambda.ai/v1", + "https://api.hyperbolic.xyz/v1", ] @@ -446,6 +451,9 @@ openai_compatible_providers: List = [ "dashscope", "moonshot", "v0", + "morph", + "lambda_ai", + "hyperbolic", ] openai_text_completion_compatible_providers: List = ( [ # providers that support `/v1/completions` @@ -458,6 +466,9 @@ openai_text_completion_compatible_providers: List = ( "nebius", "dashscope", "moonshot", + "v0", + "lambda_ai", + "hyperbolic", ] ) _openai_like_providers: List = [ @@ -808,6 +819,7 @@ DEFAULT_CRON_JOB_LOCK_TTL_SECONDS = int( PROXY_BUDGET_RESCHEDULER_MIN_TIME = int( os.getenv("PROXY_BUDGET_RESCHEDULER_MIN_TIME", 597) ) +PROXY_BATCH_POLLING_INTERVAL = int(os.getenv("PROXY_BATCH_POLLING_INTERVAL", 3600)) PROXY_BUDGET_RESCHEDULER_MAX_TIME = int( os.getenv("PROXY_BUDGET_RESCHEDULER_MAX_TIME", 605) ) diff --git a/litellm/litellm_core_utils/get_llm_provider_logic.py b/litellm/litellm_core_utils/get_llm_provider_logic.py index 84c25d49323..4e0a2efb0c6 100644 --- a/litellm/litellm_core_utils/get_llm_provider_logic.py +++ b/litellm/litellm_core_utils/get_llm_provider_logic.py @@ -240,6 +240,12 @@ def get_llm_provider( # noqa: PLR0915 elif endpoint == "https://api.v0.dev/v1": custom_llm_provider = "v0" dynamic_api_key = get_secret_str("V0_API_KEY") + elif endpoint == "https://api.lambda.ai/v1": + custom_llm_provider = "lambda_ai" + dynamic_api_key = get_secret_str("LAMBDA_API_KEY") + elif endpoint == "https://api.hyperbolic.xyz/v1": + custom_llm_provider = "hyperbolic" + dynamic_api_key = get_secret_str("HYPERBOLIC_API_KEY") if api_base is not None and not isinstance(api_base, str): raise Exception( @@ -530,7 +536,7 @@ def _get_openai_compatible_provider_info( # noqa: PLR0915 # DataRobot is OpenAI compatible. ( api_base, - dynamic_api_key + dynamic_api_key, ) = litellm.DataRobotConfig()._get_openai_compatible_provider_info( api_base, api_key ) @@ -691,6 +697,27 @@ def _get_openai_compatible_provider_info( # noqa: PLR0915 ) = litellm.V0ChatConfig()._get_openai_compatible_provider_info( api_base, api_key ) + elif custom_llm_provider == "morph": + ( + api_base, + dynamic_api_key, + ) = litellm.MorphChatConfig()._get_openai_compatible_provider_info( + api_base, api_key + ) + elif custom_llm_provider == "lambda_ai": + ( + api_base, + dynamic_api_key, + ) = litellm.LambdaAIChatConfig()._get_openai_compatible_provider_info( + api_base, api_key + ) + elif custom_llm_provider == "hyperbolic": + ( + api_base, + dynamic_api_key, + ) = litellm.HyperbolicChatConfig()._get_openai_compatible_provider_info( + api_base, api_key + ) if api_base is not None and not isinstance(api_base, str): raise Exception("api base needs to be a string. api_base={}".format(api_base)) diff --git a/litellm/llms/anthropic/chat/handler.py b/litellm/llms/anthropic/chat/handler.py index 04cf66cf021..620d2f05119 100644 --- a/litellm/llms/anthropic/chat/handler.py +++ b/litellm/llms/anthropic/chat/handler.py @@ -27,6 +27,7 @@ from litellm.litellm_core_utils.core_helpers import map_finish_reason from litellm.llms.custom_httpx.http_handler import ( AsyncHTTPHandler, HTTPHandler, + _get_httpx_client, get_async_httpx_client, ) from litellm.types.llms.anthropic import ( @@ -433,7 +434,9 @@ class AnthropicChatCompletion(BaseLLM): else: if client is None or not isinstance(client, HTTPHandler): - client = HTTPHandler(timeout=timeout) # type: ignore + client = _get_httpx_client( + params={"timeout": timeout} + ) else: client = client diff --git a/litellm/llms/azure/common_utils.py b/litellm/llms/azure/common_utils.py index f2a8defe13f..0ed4627908d 100644 --- a/litellm/llms/azure/common_utils.py +++ b/litellm/llms/azure/common_utils.py @@ -278,6 +278,7 @@ def get_azure_ad_token( 3. From username and password 4. From OIDC token 5. From a service principal with secret workflow + 6. From DefaultAzureCredential Args: litellm_params: Dictionary containing authentication parameters @@ -352,18 +353,27 @@ def get_azure_ad_token( azure_tenant_id=tenant_id, scope=scope, ) - # Try to get token provider from service principal + # Try to get token provider from service principal or DefaultAzureCredential elif ( azure_ad_token_provider is None and litellm.enable_azure_ad_token_refresh is True ): verbose_logger.debug( - "Using Azure AD token provider based on Service Principal with Secret workflow for Azure Auth" + "Using Azure AD token provider based on Service Principal with Secret workflow or DefaultAzureCredential for Azure Auth" ) try: azure_ad_token_provider = get_azure_ad_token_provider(azure_scope=scope) except ValueError: verbose_logger.debug("Azure AD Token Provider could not be used.") + + ######################################################### + # If litellm.enable_azure_ad_token_refresh is True and no other token provider is available, + # try to get DefaultAzureCredential provider + ######################################################### + if azure_ad_token_provider is None and azure_ad_token is None: + azure_ad_token_provider = BaseAzureLLM._try_get_default_azure_credential_provider( + scope=scope, + ) # Execute the token provider to get the token if available if azure_ad_token_provider and callable(azure_ad_token_provider): @@ -387,6 +397,38 @@ def get_azure_ad_token( class BaseAzureLLM(BaseOpenAILLM): + @staticmethod + def _try_get_default_azure_credential_provider( + scope: str, + ) -> Optional[Callable[[], str]]: + """ + Try to get DefaultAzureCredential provider + + Args: + scope: Azure scope for the token + + Returns: + Token provider callable if DefaultAzureCredential is enabled and available, None otherwise + """ + from litellm.types.secret_managers.get_azure_ad_token_provider import ( + AzureCredentialType, + ) + + verbose_logger.debug( + "Attempting to use DefaultAzureCredential for Azure Auth" + ) + + try: + azure_ad_token_provider = get_azure_ad_token_provider( + azure_scope=scope, + azure_credential=AzureCredentialType.DefaultAzureCredential, + ) + verbose_logger.debug("Successfully obtained Azure AD token provider using DefaultAzureCredential") + return azure_ad_token_provider + except Exception as e: + verbose_logger.debug(f"DefaultAzureCredential failed: {str(e)}") + return None + def get_azure_openai_client( self, api_key: Optional[str], diff --git a/litellm/llms/github_copilot/authenticator.py b/litellm/llms/github_copilot/authenticator.py index 8ec9cd544f4..7d7ef522a43 100644 --- a/litellm/llms/github_copilot/authenticator.py +++ b/litellm/llms/github_copilot/authenticator.py @@ -132,6 +132,23 @@ class Authenticator: status_code=401, ) + def get_api_base(self) -> Optional[str]: + """ + Get the API endpoint from the api-key.json file. + + Returns: + Optional[str]: The GitHub Copilot API endpoint, or None if not found. + """ + try: + with open(self.api_key_file, "r") as f: + api_key_info = json.load(f) + endpoints = api_key_info.get("endpoints", {}) + api_endpoint = endpoints.get("api") + return api_endpoint + except (IOError, json.JSONDecodeError, KeyError) as e: + verbose_logger.warning(f"Error reading API endpoint from file: {str(e)}") + return None + def _refresh_api_key(self) -> Dict[str, Any]: """ Refresh the API key using the access token. diff --git a/litellm/llms/github_copilot/chat/transformation.py b/litellm/llms/github_copilot/chat/transformation.py index b0aaaf9a513..eff8acb5f54 100644 --- a/litellm/llms/github_copilot/chat/transformation.py +++ b/litellm/llms/github_copilot/chat/transformation.py @@ -1,4 +1,4 @@ -from typing import Optional, Tuple +from typing import Any, Optional, Tuple, cast from litellm.exceptions import AuthenticationError from litellm.llms.openai.openai import OpenAIConfig @@ -25,7 +25,7 @@ class GithubCopilotConfig(OpenAIConfig): api_key: Optional[str], custom_llm_provider: str, ) -> Tuple[Optional[str], Optional[str], str]: - api_base = self.GITHUB_COPILOT_API_BASE + dynamic_api_base = self.authenticator.get_api_base() or self.GITHUB_COPILOT_API_BASE try: dynamic_api_key = self.authenticator.get_api_key() except GetAPIKeyError as e: @@ -34,7 +34,7 @@ class GithubCopilotConfig(OpenAIConfig): llm_provider=custom_llm_provider, message=str(e), ) - return api_base, dynamic_api_key, custom_llm_provider + return dynamic_api_base, dynamic_api_key, custom_llm_provider def _transform_messages( self, @@ -46,5 +46,5 @@ class GithubCopilotConfig(OpenAIConfig): if not disable_copilot_system_to_assistant: for message in messages: if "role" in message and message["role"] == "system": - message["role"] = "assistant" + cast(Any, message)["role"] = "assistant" return messages diff --git a/litellm/llms/hyperbolic/__init__.py b/litellm/llms/hyperbolic/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/litellm/llms/hyperbolic/chat/__init__.py b/litellm/llms/hyperbolic/chat/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/litellm/llms/hyperbolic/chat/transformation.py b/litellm/llms/hyperbolic/chat/transformation.py new file mode 100644 index 00000000000..48af9fa68a0 --- /dev/null +++ b/litellm/llms/hyperbolic/chat/transformation.py @@ -0,0 +1,54 @@ +""" +Translate from OpenAI's `/v1/chat/completions` to Hyperbolic's `/v1/chat/completions` +""" + +from typing import Optional, Tuple + +from litellm.secret_managers.main import get_secret_str + +from ...openai_like.chat.transformation import OpenAILikeChatConfig + + +class HyperbolicChatConfig(OpenAILikeChatConfig): + """ + Hyperbolic is OpenAI-compatible with standard endpoints + """ + + @property + def custom_llm_provider(self) -> Optional[str]: + return "hyperbolic" + + def _get_openai_compatible_provider_info( + self, api_base: Optional[str], api_key: Optional[str] + ) -> Tuple[Optional[str], Optional[str]]: + # Hyperbolic is openai compatible, we just need to set the api_base + api_base = ( + api_base + or get_secret_str("HYPERBOLIC_API_BASE") + or "https://api.hyperbolic.xyz/v1" # Default Hyperbolic API base URL + ) # type: ignore + dynamic_api_key = api_key or get_secret_str("HYPERBOLIC_API_KEY") + return api_base, dynamic_api_key + + def get_supported_openai_params(self, model: str) -> list: + """ + Hyperbolic supports standard OpenAI parameters + Reference: https://docs.hyperbolic.xyz/docs/rest-api + """ + return [ + "messages", # Required + "model", # Required + "stream", # Optional + "temperature", # Optional + "top_p", # Optional + "max_tokens", # Optional + "frequency_penalty", # Optional + "presence_penalty", # Optional + "stop", # Optional + "n", # Optional + "tools", # Optional + "tool_choice", # Optional + "response_format", # Optional + "seed", # Optional + "user", # Optional + ] diff --git a/litellm/llms/lambda_ai/__init__.py b/litellm/llms/lambda_ai/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/litellm/llms/lambda_ai/chat/__init__.py b/litellm/llms/lambda_ai/chat/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/litellm/llms/lambda_ai/chat/transformation.py b/litellm/llms/lambda_ai/chat/transformation.py new file mode 100644 index 00000000000..2d481d66824 --- /dev/null +++ b/litellm/llms/lambda_ai/chat/transformation.py @@ -0,0 +1,31 @@ +""" +Translate from OpenAI's `/v1/chat/completions` to Lambda's `/v1/chat/completions` +""" + +from typing import Optional, Tuple + +from litellm.secret_managers.main import get_secret_str + +from ...openai_like.chat.transformation import OpenAILikeChatConfig + + +class LambdaAIChatConfig(OpenAILikeChatConfig): + """ + Lambda AI is OpenAI-compatible with standard endpoints + """ + + @property + def custom_llm_provider(self) -> Optional[str]: + return "lambda_ai" + + def _get_openai_compatible_provider_info( + self, api_base: Optional[str], api_key: Optional[str] + ) -> Tuple[Optional[str], Optional[str]]: + # Lambda AI is openai compatible, we just need to set the api_base + api_base = ( + api_base + or get_secret_str("LAMBDA_API_BASE") + or "https://api.lambda.ai/v1" # Default Lambda API base URL + ) # type: ignore + dynamic_api_key = api_key or get_secret_str("LAMBDA_API_KEY") + return api_base, dynamic_api_key \ No newline at end of file diff --git a/litellm/llms/morph/__init__.py b/litellm/llms/morph/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/litellm/llms/morph/chat/__init__.py b/litellm/llms/morph/chat/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/litellm/llms/morph/chat/transformation.py b/litellm/llms/morph/chat/transformation.py new file mode 100644 index 00000000000..f37ed8e45e3 --- /dev/null +++ b/litellm/llms/morph/chat/transformation.py @@ -0,0 +1,46 @@ +""" +Transform request from OpenAI format to Morph format. + +[TODO] Docs: Morph supports the OpenAI API format. +https://docs.morphllm.com/quickstart +""" + +from typing import Optional, Tuple + +from litellm.secret_managers.main import get_secret_str + +from ...openai_like.chat.transformation import OpenAILikeChatConfig + + +class MorphChatConfig(OpenAILikeChatConfig): + """ + Transform request from OpenAI format to Morph format. + """ + + @property + def custom_llm_provider(self) -> Optional[str]: + return "morph" + + def _get_openai_compatible_provider_info( + self, api_base: Optional[str], api_key: Optional[str] + ) -> Tuple[Optional[str], Optional[str]]: + api_base = ( + api_base + or get_secret_str("MORPH_API_BASE") + or "https://api.morphllm.com/v1" # default api base + ) + dynamic_api_key = api_key or get_secret_str("MORPH_API_KEY") + return api_base, dynamic_api_key + + def get_supported_openai_params(self, model: str) -> list: + return [ + "messages", + "model", + "stream", + ] + + def pre_call(self, messages: list, model: str, api_key: str, api_base: str): + """ + Hook for any pre-processing before the API call. + """ + return \ No newline at end of file diff --git a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py index d09599e8789..e4a68dd82e0 100644 --- a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py +++ b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py @@ -35,6 +35,7 @@ from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMExcepti from litellm.llms.custom_httpx.http_handler import ( AsyncHTTPHandler, HTTPHandler, + _get_httpx_client, get_async_httpx_client, ) from litellm.types.llms.anthropic import AnthropicThinkingParam @@ -1881,7 +1882,7 @@ class VertexLLM(VertexBase): if isinstance(timeout, float) or isinstance(timeout, int): timeout = httpx.Timeout(timeout) _params["timeout"] = timeout - client = HTTPHandler(**_params) # type: ignore + client = _get_httpx_client(params=_params) else: client = client diff --git a/litellm/llms/watsonx/chat/transformation.py b/litellm/llms/watsonx/chat/transformation.py index 71d8bba4ef4..6b0dd5a39ae 100644 --- a/litellm/llms/watsonx/chat/transformation.py +++ b/litellm/llms/watsonx/chat/transformation.py @@ -25,7 +25,7 @@ class IBMWatsonXChatConfig(IBMWatsonXMixin, OpenAIGPTConfig): "seed", # equivalent to random_seed "stream", # equivalent to stream "tools", - "tool_choice", # equivalent to tool_choice + tool_choice_options + "tool_choice", # equivalent to tool_choice + tool_choice_option "logprobs", "top_logprobs", "n", @@ -61,7 +61,7 @@ class IBMWatsonXChatConfig(IBMWatsonXMixin, OpenAIGPTConfig): _tool_choice = non_default_params.pop("tool_choice", None) if self.is_tool_choice_option(_tool_choice): - optional_params["tool_choice_options"] = _tool_choice + optional_params["tool_choice_option"] = _tool_choice elif _tool_choice is not None: optional_params["tool_choice"] = _tool_choice return super().map_openai_params( diff --git a/litellm/main.py b/litellm/main.py index e0d41260a3f..e0b76d57a6a 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -3486,13 +3486,13 @@ async def acompletion_with_retries(*args, **kwargs): retry_strategy = kwargs.pop("retry_strategy", "constant_retry") original_function = kwargs.pop("original_function", completion) if retry_strategy == "exponential_backoff_retry": - retryer = tenacity.AsyncRetrying( + retryer = tenacity.Retrying( wait=tenacity.wait_exponential(multiplier=1, max=10), stop=tenacity.stop_after_attempt(num_retries), reraise=True, ) else: - retryer = tenacity.AsyncRetrying( + retryer = tenacity.Retrying( stop=tenacity.stop_after_attempt(num_retries), reraise=True ) return await retryer(original_function, *args, **kwargs) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 43ef9f798ab..4f22818792b 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -5417,12 +5417,12 @@ "supports_tool_choice": true, "deprecation_date": "2025-01-06" }, - "groq/qwen-qwq-32b": { - "max_tokens": 128000, - "max_input_tokens": 128000, - "max_output_tokens": 128000, + "groq/qwen/qwen3-32b": { + "max_tokens": 131000, + "max_input_tokens": 131000, + "max_output_tokens": 131000, "input_cost_per_token": 2.9e-07, - "output_cost_per_token": 3.9e-07, + "output_cost_per_token": 5.9e-07, "litellm_provider": "groq", "mode": "chat", "supports_function_calling": true, @@ -11039,6 +11039,16 @@ "mode": "chat", "supports_tool_choice": true }, + "openrouter/qwen/qwen-vl-plus": { + "max_tokens": 8192, + "max_input_tokens": 8192, + "max_output_tokens": 2048, + "input_cost_per_token": 0.00021, + "output_cost_per_token": 0.00063, + "litellm_provider": "openrouter", + "mode": "chat", + "supports_tool_choice": false + }, "openrouter/switchpoint/router": { "max_tokens": 131072, "max_input_tokens": 131072, @@ -14603,6 +14613,19 @@ "supports_tool_choice": false, "supports_response_schema": true }, + "fireworks_ai/accounts/fireworks/models/kimi-k2-instruct": { + "max_tokens": 131072, + "max_input_tokens": 131072, + "max_output_tokens": 16384, + "input_cost_per_token": 0.6e-06, + "output_cost_per_token": 2.5e-06, + "litellm_provider": "fireworks_ai", + "mode": "chat", + "supports_function_calling": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "source": "https://fireworks.ai/models/fireworks/kimi-k2-instruct" + }, "fireworks_ai/accounts/fireworks/models/llama-v3p1-405b-instruct": { "max_tokens": 16384, "max_input_tokens": 128000, @@ -14921,13 +14944,478 @@ "supports_system_messages": true, "supports_tool_choice": true }, - "voyage/voyage-01": { - "max_tokens": 4096, - "max_input_tokens": 4096, + "lambda_ai/deepseek-llama3.3-70b": { + "max_tokens": 131072, + "max_input_tokens": 131072, + "max_output_tokens": 131072, + "input_cost_per_token": 2e-07, + "output_cost_per_token": 6e-07, + "litellm_provider": "lambda_ai", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_reasoning": true + }, + "lambda_ai/deepseek-r1-0528": { + "max_tokens": 131072, + "max_input_tokens": 131072, + "max_output_tokens": 131072, + "input_cost_per_token": 2e-07, + "output_cost_per_token": 6e-07, + "litellm_provider": "lambda_ai", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_reasoning": true + }, + "lambda_ai/deepseek-r1-671b": { + "max_tokens": 131072, + "max_input_tokens": 131072, + "max_output_tokens": 131072, + "input_cost_per_token": 8e-07, + "output_cost_per_token": 8e-07, + "litellm_provider": "lambda_ai", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_reasoning": true + }, + "lambda_ai/deepseek-v3-0324": { + "max_tokens": 131072, + "max_input_tokens": 131072, + "max_output_tokens": 131072, + "input_cost_per_token": 2e-07, + "output_cost_per_token": 6e-07, + "litellm_provider": "lambda_ai", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, + "lambda_ai/hermes3-405b": { + "max_tokens": 131072, + "max_input_tokens": 131072, + "max_output_tokens": 131072, + "input_cost_per_token": 8e-07, + "output_cost_per_token": 8e-07, + "litellm_provider": "lambda_ai", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, + "lambda_ai/hermes3-70b": { + "max_tokens": 131072, + "max_input_tokens": 131072, + "max_output_tokens": 131072, + "input_cost_per_token": 1.2e-07, + "output_cost_per_token": 3e-07, + "litellm_provider": "lambda_ai", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, + "lambda_ai/hermes3-8b": { + "max_tokens": 131072, + "max_input_tokens": 131072, + "max_output_tokens": 131072, + "input_cost_per_token": 2.5e-08, + "output_cost_per_token": 4e-08, + "litellm_provider": "lambda_ai", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, + "lambda_ai/lfm-40b": { + "max_tokens": 131072, + "max_input_tokens": 131072, + "max_output_tokens": 131072, "input_cost_per_token": 1e-07, - "output_cost_per_token": 0.0, - "litellm_provider": "voyage", - "mode": "embedding" + "output_cost_per_token": 2e-07, + "litellm_provider": "lambda_ai", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, + "lambda_ai/lfm-7b": { + "max_tokens": 131072, + "max_input_tokens": 131072, + "max_output_tokens": 131072, + "input_cost_per_token": 2.5e-08, + "output_cost_per_token": 4e-08, + "litellm_provider": "lambda_ai", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, + "lambda_ai/llama-4-maverick-17b-128e-instruct-fp8": { + "max_tokens": 131072, + "max_input_tokens": 131072, + "max_output_tokens": 8192, + "input_cost_per_token": 5e-08, + "output_cost_per_token": 1e-07, + "litellm_provider": "lambda_ai", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, + "lambda_ai/llama-4-scout-17b-16e-instruct": { + "max_tokens": 16384, + "max_input_tokens": 16384, + "max_output_tokens": 8192, + "input_cost_per_token": 5e-08, + "output_cost_per_token": 1e-07, + "litellm_provider": "lambda_ai", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, + "lambda_ai/llama3.1-405b-instruct-fp8": { + "max_tokens": 131072, + "max_input_tokens": 131072, + "max_output_tokens": 131072, + "input_cost_per_token": 8e-07, + "output_cost_per_token": 8e-07, + "litellm_provider": "lambda_ai", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, + "lambda_ai/llama3.1-70b-instruct-fp8": { + "max_tokens": 131072, + "max_input_tokens": 131072, + "max_output_tokens": 131072, + "input_cost_per_token": 1.2e-07, + "output_cost_per_token": 3e-07, + "litellm_provider": "lambda_ai", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, + "lambda_ai/llama3.1-8b-instruct": { + "max_tokens": 131072, + "max_input_tokens": 131072, + "max_output_tokens": 131072, + "input_cost_per_token": 2.5e-08, + "output_cost_per_token": 4e-08, + "litellm_provider": "lambda_ai", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, + "lambda_ai/llama3.1-nemotron-70b-instruct-fp8": { + "max_tokens": 131072, + "max_input_tokens": 131072, + "max_output_tokens": 131072, + "input_cost_per_token": 1.2e-07, + "output_cost_per_token": 3e-07, + "litellm_provider": "lambda_ai", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, + "lambda_ai/llama3.2-11b-vision-instruct": { + "max_tokens": 131072, + "max_input_tokens": 131072, + "max_output_tokens": 131072, + "input_cost_per_token": 1.5e-08, + "output_cost_per_token": 2.5e-08, + "litellm_provider": "lambda_ai", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_vision": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, + "lambda_ai/llama3.2-3b-instruct": { + "max_tokens": 131072, + "max_input_tokens": 131072, + "max_output_tokens": 131072, + "input_cost_per_token": 1.5e-08, + "output_cost_per_token": 2.5e-08, + "litellm_provider": "lambda_ai", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, + "lambda_ai/llama3.3-70b-instruct-fp8": { + "max_tokens": 131072, + "max_input_tokens": 131072, + "max_output_tokens": 131072, + "input_cost_per_token": 1.2e-07, + "output_cost_per_token": 3e-07, + "litellm_provider": "lambda_ai", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, + "lambda_ai/qwen25-coder-32b-instruct": { + "max_tokens": 131072, + "max_input_tokens": 131072, + "max_output_tokens": 131072, + "input_cost_per_token": 5e-08, + "output_cost_per_token": 1e-07, + "litellm_provider": "lambda_ai", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, + "lambda_ai/qwen3-32b-fp8": { + "max_tokens": 131072, + "max_input_tokens": 131072, + "max_output_tokens": 131072, + "input_cost_per_token": 5e-08, + "output_cost_per_token": 1e-07, + "litellm_provider": "lambda_ai", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_reasoning": true + }, + "hyperbolic/moonshotai/Kimi-K2-Instruct": { + "max_tokens": 131072, + "max_input_tokens": 131072, + "max_output_tokens": 131072, + "input_cost_per_token": 2e-06, + "output_cost_per_token": 2e-06, + "litellm_provider": "hyperbolic", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, + "hyperbolic/deepseek-ai/DeepSeek-R1-0528": { + "max_tokens": 131072, + "max_input_tokens": 131072, + "max_output_tokens": 131072, + "input_cost_per_token": 2.5e-07, + "output_cost_per_token": 2.5e-07, + "litellm_provider": "hyperbolic", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, + "hyperbolic/Qwen/Qwen3-235B-A22B": { + "max_tokens": 131072, + "max_input_tokens": 131072, + "max_output_tokens": 131072, + "input_cost_per_token": 2e-06, + "output_cost_per_token": 2e-06, + "litellm_provider": "hyperbolic", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, + "hyperbolic/deepseek-ai/DeepSeek-V3-0324": { + "max_tokens": 32768, + "max_input_tokens": 32768, + "max_output_tokens": 32768, + "input_cost_per_token": 4e-07, + "output_cost_per_token": 4e-07, + "litellm_provider": "hyperbolic", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, + "hyperbolic/Qwen/QwQ-32B": { + "max_tokens": 131072, + "max_input_tokens": 131072, + "max_output_tokens": 131072, + "input_cost_per_token": 2e-07, + "output_cost_per_token": 2e-07, + "litellm_provider": "hyperbolic", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, + "hyperbolic/deepseek-ai/DeepSeek-R1": { + "max_tokens": 32768, + "max_input_tokens": 32768, + "max_output_tokens": 32768, + "input_cost_per_token": 4e-07, + "output_cost_per_token": 4e-07, + "litellm_provider": "hyperbolic", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, + "hyperbolic/deepseek-ai/DeepSeek-V3": { + "max_tokens": 32768, + "max_input_tokens": 32768, + "max_output_tokens": 32768, + "input_cost_per_token": 2e-07, + "output_cost_per_token": 2e-07, + "litellm_provider": "hyperbolic", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, + "hyperbolic/meta-llama/Llama-3.3-70B-Instruct": { + "max_tokens": 131072, + "max_input_tokens": 131072, + "max_output_tokens": 131072, + "input_cost_per_token": 1.2e-07, + "output_cost_per_token": 3e-07, + "litellm_provider": "hyperbolic", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, + "hyperbolic/Qwen/Qwen2.5-Coder-32B-Instruct": { + "max_tokens": 32768, + "max_input_tokens": 32768, + "max_output_tokens": 32768, + "input_cost_per_token": 1.2e-07, + "output_cost_per_token": 3e-07, + "litellm_provider": "hyperbolic", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, + "hyperbolic/meta-llama/Llama-3.2-3B-Instruct": { + "max_tokens": 32768, + "max_input_tokens": 32768, + "max_output_tokens": 32768, + "input_cost_per_token": 1.2e-07, + "output_cost_per_token": 3e-07, + "litellm_provider": "hyperbolic", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, + "hyperbolic/Qwen/Qwen2.5-72B-Instruct": { + "max_tokens": 131072, + "max_input_tokens": 131072, + "max_output_tokens": 131072, + "input_cost_per_token": 1.2e-07, + "output_cost_per_token": 3e-07, + "litellm_provider": "hyperbolic", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, + "hyperbolic/meta-llama/Meta-Llama-3-70B-Instruct": { + "max_tokens": 131072, + "max_input_tokens": 131072, + "max_output_tokens": 131072, + "input_cost_per_token": 1.2e-07, + "output_cost_per_token": 3e-07, + "litellm_provider": "hyperbolic", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, + "hyperbolic/NousResearch/Hermes-3-Llama-3.1-70B": { + "max_tokens": 32768, + "max_input_tokens": 32768, + "max_output_tokens": 32768, + "input_cost_per_token": 1.2e-07, + "output_cost_per_token": 3e-07, + "litellm_provider": "hyperbolic", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, + "hyperbolic/meta-llama/Meta-Llama-3.1-405B-Instruct": { + "max_tokens": 32768, + "max_input_tokens": 32768, + "max_output_tokens": 32768, + "input_cost_per_token": 1.2e-07, + "output_cost_per_token": 3e-07, + "litellm_provider": "hyperbolic", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, + "hyperbolic/meta-llama/Meta-Llama-3.1-8B-Instruct": { + "max_tokens": 32768, + "max_input_tokens": 32768, + "max_output_tokens": 32768, + "input_cost_per_token": 1.2e-07, + "output_cost_per_token": 3e-07, + "litellm_provider": "hyperbolic", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, + "hyperbolic/meta-llama/Meta-Llama-3.1-70B-Instruct": { + "max_tokens": 32768, + "max_input_tokens": 32768, + "max_output_tokens": 32768, + "input_cost_per_token": 1.2e-07, + "output_cost_per_token": 3e-07, + "litellm_provider": "hyperbolic", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true }, "voyage/voyage-lite-01": { "max_tokens": 4096, diff --git a/litellm/proxy/_new_secret_config.yaml b/litellm/proxy/_new_secret_config.yaml index b2642d8a600..00d3ef1adfe 100644 --- a/litellm/proxy/_new_secret_config.yaml +++ b/litellm/proxy/_new_secret_config.yaml @@ -1,25 +1,11 @@ model_list: - - model_name: gpt-3.5-turbo-allow + - model_name: gpt-4o-mini-batch litellm_params: - model: gpt-3.5-turbo + model: azure/gpt-4o-mini + api_key: os.environ/AZURE_API_KEY_HIDDEN + api_base: os.environ/AZURE_API_BASE_HIDDEN model_info: version: 2 - - model_name: gpt-3.5-turbo-disallow - litellm_params: - model: gpt-3.5-turbo - model_info: - version: 2 - - model_name: zapier-byok-provider/openai/* - litellm_params: - model: openai/* - api_base: http://0.0.0.0:8090 - - model_name: openai/gpt-4o-mini - litellm_params: - model: openai/gpt-4o-mini -litellm_settings: - model_group_alias: {"gpt-3.5-turbo-custom": "gpt-3.5-turbo-disallow"} - model_group_settings: - forward_client_headers_to_llm_api: - - "gpt-3.5-turbo-allow" - - "zapier-byok-provider/openai/*" +general_settings: + proxy_batch_polling_interval: 10 \ No newline at end of file diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 50a044d9835..c2bf2f46081 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -132,6 +132,7 @@ from litellm.constants import ( DEFAULT_MODEL_CREATED_AT_TIME, LITELLM_PROXY_ADMIN_NAME, PROMETHEUS_FALLBACK_STATS_SEND_TIME_HOURS, + PROXY_BATCH_POLLING_INTERVAL, PROXY_BATCH_WRITE_AT, PROXY_BUDGET_RESCHEDULER_MAX_TIME, PROXY_BUDGET_RESCHEDULER_MIN_TIME, @@ -538,7 +539,7 @@ async def proxy_shutdown_event(): @asynccontextmanager async def proxy_startup_event(app: FastAPI): - global prisma_client, master_key, use_background_health_checks, llm_router, llm_model_list, general_settings, proxy_budget_rescheduler_min_time, proxy_budget_rescheduler_max_time, litellm_proxy_admin_name, db_writer_client, store_model_in_db, premium_user, _license_check + global prisma_client, master_key, use_background_health_checks, llm_router, llm_model_list, general_settings, proxy_budget_rescheduler_min_time, proxy_budget_rescheduler_max_time, litellm_proxy_admin_name, db_writer_client, store_model_in_db, premium_user, _license_check, proxy_batch_polling_interval import json init_verbose_loggers() @@ -940,6 +941,7 @@ litellm_proxy_admin_name = LITELLM_PROXY_ADMIN_NAME ui_access_mode: Union[Literal["admin", "all"], Dict] = "all" proxy_budget_rescheduler_min_time = PROXY_BUDGET_RESCHEDULER_MIN_TIME proxy_budget_rescheduler_max_time = PROXY_BUDGET_RESCHEDULER_MAX_TIME +proxy_batch_polling_interval = PROXY_BATCH_POLLING_INTERVAL proxy_batch_write_at = PROXY_BATCH_WRITE_AT litellm_master_key_hash = None disable_spend_logs = False @@ -1697,7 +1699,7 @@ class ProxyConfig: """ Load config values into proxy global state """ - global master_key, user_config_file_path, otel_logging, user_custom_auth, user_custom_auth_path, user_custom_key_generate, user_custom_sso, user_custom_ui_sso_sign_in_handler, use_background_health_checks, health_check_interval, use_queue, proxy_budget_rescheduler_max_time, proxy_budget_rescheduler_min_time, ui_access_mode, litellm_master_key_hash, proxy_batch_write_at, disable_spend_logs, prompt_injection_detection_obj, redis_usage_cache, store_model_in_db, premium_user, open_telemetry_logger, health_check_details, callback_settings + global master_key, user_config_file_path, otel_logging, user_custom_auth, user_custom_auth_path, user_custom_key_generate, user_custom_sso, user_custom_ui_sso_sign_in_handler, use_background_health_checks, health_check_interval, use_queue, proxy_budget_rescheduler_max_time, proxy_budget_rescheduler_min_time, ui_access_mode, litellm_master_key_hash, proxy_batch_write_at, disable_spend_logs, prompt_injection_detection_obj, redis_usage_cache, store_model_in_db, premium_user, open_telemetry_logger, health_check_details, callback_settings, proxy_batch_polling_interval config: dict = await self.get_config(config_file_path=config_file_path) @@ -2040,6 +2042,10 @@ class ProxyConfig: proxy_budget_rescheduler_max_time = general_settings.get( "proxy_budget_rescheduler_max_time", proxy_budget_rescheduler_max_time ) + ## BATCH POLLING INTERVAL ## + proxy_batch_polling_interval = general_settings.get( + "proxy_batch_polling_interval", proxy_batch_polling_interval + ) ## BATCH WRITER ## proxy_batch_write_at = general_settings.get( "proxy_batch_write_at", proxy_batch_write_at @@ -3564,7 +3570,7 @@ class ProxyStartupEvent: scheduler.add_job( check_batch_cost_job.check_batch_cost, "interval", - seconds=3600, # these can run infrequently, as batch jobs take time to complete + seconds=proxy_batch_polling_interval, # these can run infrequently, as batch jobs take time to complete ) except Exception: diff --git a/litellm/proxy/ui_crud_endpoints/proxy_setting_endpoints.py b/litellm/proxy/ui_crud_endpoints/proxy_setting_endpoints.py index 14f34b15858..a3aafda2232 100644 --- a/litellm/proxy/ui_crud_endpoints/proxy_setting_endpoints.py +++ b/litellm/proxy/ui_crud_endpoints/proxy_setting_endpoints.py @@ -480,7 +480,7 @@ async def update_sso_settings(sso_config: SSOConfig): elif field_name == "ui_access_mode" and value is not None: config["general_settings"]["ui_access_mode"] = value - elif field_name in env_var_mapping and value is not None: + elif field_name in env_var_mapping and value is not None and len(value) > 0: env_var_name = env_var_mapping[field_name] # Update in config config["environment_variables"][env_var_name] = value diff --git a/litellm/router_utils/forward_clientside_headers_by_model_group.py b/litellm/router_utils/forward_clientside_headers_by_model_group.py index 2e1a066a6cc..87c2e48f689 100644 --- a/litellm/router_utils/forward_clientside_headers_by_model_group.py +++ b/litellm/router_utils/forward_clientside_headers_by_model_group.py @@ -31,18 +31,6 @@ class ForwardClientSideHeadersByModelGroup(CustomLogger): "model_group_alias": model_group_alias, } - def filter_headers(self, headers: Dict[str, Any]) -> Dict[str, Any]: - """ - Filter the headers to only include the headers that are forwarded to the LLM API. - - E.g. passing 'connection': 'keep-alive' will cause the request to hang, and not be acknowledged on the other side. - """ - return { - k: v - for k, v in headers.items() - if k.lower() not in ["connection", "content-length"] - } - async def async_pre_call_deployment_hook( self, kwargs: Dict[str, Any], call_type: Optional[CallTypes] ) -> Optional[dict]: @@ -78,7 +66,7 @@ class ForwardClientSideHeadersByModelGroup(CustomLogger): in litellm.model_group_settings.forward_client_headers_to_llm_api ): kwargs.setdefault("headers", {}).update( - self.filter_headers(kwargs["secret_fields"]["raw_headers"]) + kwargs["secret_fields"]["raw_headers"] ) return kwargs diff --git a/litellm/secret_managers/get_azure_ad_token_provider.py b/litellm/secret_managers/get_azure_ad_token_provider.py index e4b749a6598..1ce7264b0b9 100644 --- a/litellm/secret_managers/get_azure_ad_token_provider.py +++ b/litellm/secret_managers/get_azure_ad_token_provider.py @@ -6,7 +6,10 @@ from litellm.types.secret_managers.get_azure_ad_token_provider import ( ) -def get_azure_ad_token_provider(azure_scope: Optional[str] = None) -> Callable[[], str]: +def get_azure_ad_token_provider( + azure_scope: Optional[str] = None, + azure_credential: Optional[AzureCredentialType] = None, +) -> Callable[[], str]: """ Get Azure AD token provider based on Service Principal with Secret workflow. @@ -27,6 +30,7 @@ def get_azure_ad_token_provider(azure_scope: Optional[str] = None) -> Callable[[ from azure.identity import ( CertificateCredential, ClientSecretCredential, + DefaultAzureCredential, ManagedIdentityCredential, get_bearer_token_provider, ) @@ -37,14 +41,17 @@ def get_azure_ad_token_provider(azure_scope: Optional[str] = None) -> Callable[[ or "https://cognitiveservices.azure.com/.default" ) - cred: str = os.environ.get( - "AZURE_CREDENTIAL", AzureCredentialType.ClientSecretCredential + cred: str = ( + azure_credential.value if azure_credential else None + or os.environ.get("AZURE_CREDENTIAL", AzureCredentialType.ClientSecretCredential) + or AzureCredentialType.ClientSecretCredential ) credential: Optional[ Union[ ClientSecretCredential, ManagedIdentityCredential, CertificateCredential, + DefaultAzureCredential, Any, ] ] = None @@ -62,10 +69,15 @@ def get_azure_ad_token_provider(azure_scope: Optional[str] = None) -> Callable[[ tenant_id=os.environ["AZURE_TENANT_ID"], certificate_path=os.environ["AZURE_CERTIFICATE_PATH"], ) + elif cred == AzureCredentialType.DefaultAzureCredential: + # DefaultAzureCredential doesn't require explicit environment variables + # It automatically discovers credentials from the environment (managed identity, CLI, etc.) + credential = DefaultAzureCredential() else: cred_cls = getattr(identity, cred) credential = cred_cls() if credential is None: raise ValueError("No credential provided") + return get_bearer_token_provider(credential, azure_scope) diff --git a/litellm/types/llms/recraft.py b/litellm/types/llms/recraft.py index 8de63549c32..f4429dcf52f 100644 --- a/litellm/types/llms/recraft.py +++ b/litellm/types/llms/recraft.py @@ -32,4 +32,5 @@ class RecraftImageEditRequestParams(TypedDict, total=False): substyle: Optional[str] # Additional style specification response_format: Optional[str] # Format of returned images: url or b64_json negative_prompt: Optional[str] # Description of undesired elements - controls: Optional[Dict] # Custom parameters to tweak generation process \ No newline at end of file + controls: Optional[Dict] # Custom parameters to tweak generation process + controls: Optional[Dict] diff --git a/litellm/types/secret_managers/get_azure_ad_token_provider.py b/litellm/types/secret_managers/get_azure_ad_token_provider.py index f318b4333bd..5d2f7409f95 100644 --- a/litellm/types/secret_managers/get_azure_ad_token_provider.py +++ b/litellm/types/secret_managers/get_azure_ad_token_provider.py @@ -5,3 +5,4 @@ class AzureCredentialType(str, Enum): ClientSecretCredential = "ClientSecretCredential" ManagedIdentityCredential = "ManagedIdentityCredential" CertificateCredential = "CertificateCredential" + DefaultAzureCredential = "DefaultAzureCredential" diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 88d58582d0c..acff566a3f6 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -2276,6 +2276,8 @@ class LlmProviders(str, Enum): DASHSCOPE = "dashscope" MOONSHOT = "moonshot" V0 = "v0" + MORPH = "morph" + LAMBDA_AI = "lambda_ai" DEEPSEEK = "deepseek" SAMBANOVA = "sambanova" MARITALK = "maritalk" @@ -2313,6 +2315,7 @@ class LlmProviders(str, Enum): LLAMA = "meta_llama" NSCALE = "nscale" PG_VECTOR = "pg_vector" + HYPERBOLIC = "hyperbolic" RECRAFT = "recraft" diff --git a/litellm/utils.py b/litellm/utils.py index f5d4022d62b..a8ed6c512fe 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -6684,6 +6684,8 @@ class ProviderConfigManager: return litellm.DatabricksConfig() elif litellm.LlmProviders.XAI == provider: return litellm.XAIChatConfig() + elif litellm.LlmProviders.LAMBDA_AI == provider: + return litellm.LambdaAIChatConfig() elif litellm.LlmProviders.LLAMA == provider: return litellm.LlamaAPIConfig() elif litellm.LlmProviders.TEXT_COMPLETION_OPENAI == provider: @@ -6832,6 +6834,8 @@ class ProviderConfigManager: return litellm.MoonshotChatConfig() elif litellm.LlmProviders.V0 == provider: return litellm.V0ChatConfig() + elif litellm.LlmProviders.MORPH == provider: + return litellm.MorphChatConfig() elif litellm.LlmProviders.BEDROCK == provider: bedrock_route = BedrockModelInfo.get_bedrock_route(model) bedrock_invoke_provider = litellm.BedrockLLM.get_bedrock_invoke_provider( @@ -6880,6 +6884,8 @@ class ProviderConfigManager: return litellm.OpenAIGPTConfig() elif litellm.LlmProviders.NSCALE == provider: return litellm.NscaleConfig() + elif litellm.LlmProviders.HYPERBOLIC == provider: + return litellm.HyperbolicChatConfig() return None @staticmethod diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 43ef9f798ab..4f22818792b 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -5417,12 +5417,12 @@ "supports_tool_choice": true, "deprecation_date": "2025-01-06" }, - "groq/qwen-qwq-32b": { - "max_tokens": 128000, - "max_input_tokens": 128000, - "max_output_tokens": 128000, + "groq/qwen/qwen3-32b": { + "max_tokens": 131000, + "max_input_tokens": 131000, + "max_output_tokens": 131000, "input_cost_per_token": 2.9e-07, - "output_cost_per_token": 3.9e-07, + "output_cost_per_token": 5.9e-07, "litellm_provider": "groq", "mode": "chat", "supports_function_calling": true, @@ -11039,6 +11039,16 @@ "mode": "chat", "supports_tool_choice": true }, + "openrouter/qwen/qwen-vl-plus": { + "max_tokens": 8192, + "max_input_tokens": 8192, + "max_output_tokens": 2048, + "input_cost_per_token": 0.00021, + "output_cost_per_token": 0.00063, + "litellm_provider": "openrouter", + "mode": "chat", + "supports_tool_choice": false + }, "openrouter/switchpoint/router": { "max_tokens": 131072, "max_input_tokens": 131072, @@ -14603,6 +14613,19 @@ "supports_tool_choice": false, "supports_response_schema": true }, + "fireworks_ai/accounts/fireworks/models/kimi-k2-instruct": { + "max_tokens": 131072, + "max_input_tokens": 131072, + "max_output_tokens": 16384, + "input_cost_per_token": 0.6e-06, + "output_cost_per_token": 2.5e-06, + "litellm_provider": "fireworks_ai", + "mode": "chat", + "supports_function_calling": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "source": "https://fireworks.ai/models/fireworks/kimi-k2-instruct" + }, "fireworks_ai/accounts/fireworks/models/llama-v3p1-405b-instruct": { "max_tokens": 16384, "max_input_tokens": 128000, @@ -14921,13 +14944,478 @@ "supports_system_messages": true, "supports_tool_choice": true }, - "voyage/voyage-01": { - "max_tokens": 4096, - "max_input_tokens": 4096, + "lambda_ai/deepseek-llama3.3-70b": { + "max_tokens": 131072, + "max_input_tokens": 131072, + "max_output_tokens": 131072, + "input_cost_per_token": 2e-07, + "output_cost_per_token": 6e-07, + "litellm_provider": "lambda_ai", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_reasoning": true + }, + "lambda_ai/deepseek-r1-0528": { + "max_tokens": 131072, + "max_input_tokens": 131072, + "max_output_tokens": 131072, + "input_cost_per_token": 2e-07, + "output_cost_per_token": 6e-07, + "litellm_provider": "lambda_ai", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_reasoning": true + }, + "lambda_ai/deepseek-r1-671b": { + "max_tokens": 131072, + "max_input_tokens": 131072, + "max_output_tokens": 131072, + "input_cost_per_token": 8e-07, + "output_cost_per_token": 8e-07, + "litellm_provider": "lambda_ai", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_reasoning": true + }, + "lambda_ai/deepseek-v3-0324": { + "max_tokens": 131072, + "max_input_tokens": 131072, + "max_output_tokens": 131072, + "input_cost_per_token": 2e-07, + "output_cost_per_token": 6e-07, + "litellm_provider": "lambda_ai", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, + "lambda_ai/hermes3-405b": { + "max_tokens": 131072, + "max_input_tokens": 131072, + "max_output_tokens": 131072, + "input_cost_per_token": 8e-07, + "output_cost_per_token": 8e-07, + "litellm_provider": "lambda_ai", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, + "lambda_ai/hermes3-70b": { + "max_tokens": 131072, + "max_input_tokens": 131072, + "max_output_tokens": 131072, + "input_cost_per_token": 1.2e-07, + "output_cost_per_token": 3e-07, + "litellm_provider": "lambda_ai", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, + "lambda_ai/hermes3-8b": { + "max_tokens": 131072, + "max_input_tokens": 131072, + "max_output_tokens": 131072, + "input_cost_per_token": 2.5e-08, + "output_cost_per_token": 4e-08, + "litellm_provider": "lambda_ai", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, + "lambda_ai/lfm-40b": { + "max_tokens": 131072, + "max_input_tokens": 131072, + "max_output_tokens": 131072, "input_cost_per_token": 1e-07, - "output_cost_per_token": 0.0, - "litellm_provider": "voyage", - "mode": "embedding" + "output_cost_per_token": 2e-07, + "litellm_provider": "lambda_ai", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, + "lambda_ai/lfm-7b": { + "max_tokens": 131072, + "max_input_tokens": 131072, + "max_output_tokens": 131072, + "input_cost_per_token": 2.5e-08, + "output_cost_per_token": 4e-08, + "litellm_provider": "lambda_ai", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, + "lambda_ai/llama-4-maverick-17b-128e-instruct-fp8": { + "max_tokens": 131072, + "max_input_tokens": 131072, + "max_output_tokens": 8192, + "input_cost_per_token": 5e-08, + "output_cost_per_token": 1e-07, + "litellm_provider": "lambda_ai", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, + "lambda_ai/llama-4-scout-17b-16e-instruct": { + "max_tokens": 16384, + "max_input_tokens": 16384, + "max_output_tokens": 8192, + "input_cost_per_token": 5e-08, + "output_cost_per_token": 1e-07, + "litellm_provider": "lambda_ai", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, + "lambda_ai/llama3.1-405b-instruct-fp8": { + "max_tokens": 131072, + "max_input_tokens": 131072, + "max_output_tokens": 131072, + "input_cost_per_token": 8e-07, + "output_cost_per_token": 8e-07, + "litellm_provider": "lambda_ai", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, + "lambda_ai/llama3.1-70b-instruct-fp8": { + "max_tokens": 131072, + "max_input_tokens": 131072, + "max_output_tokens": 131072, + "input_cost_per_token": 1.2e-07, + "output_cost_per_token": 3e-07, + "litellm_provider": "lambda_ai", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, + "lambda_ai/llama3.1-8b-instruct": { + "max_tokens": 131072, + "max_input_tokens": 131072, + "max_output_tokens": 131072, + "input_cost_per_token": 2.5e-08, + "output_cost_per_token": 4e-08, + "litellm_provider": "lambda_ai", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, + "lambda_ai/llama3.1-nemotron-70b-instruct-fp8": { + "max_tokens": 131072, + "max_input_tokens": 131072, + "max_output_tokens": 131072, + "input_cost_per_token": 1.2e-07, + "output_cost_per_token": 3e-07, + "litellm_provider": "lambda_ai", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, + "lambda_ai/llama3.2-11b-vision-instruct": { + "max_tokens": 131072, + "max_input_tokens": 131072, + "max_output_tokens": 131072, + "input_cost_per_token": 1.5e-08, + "output_cost_per_token": 2.5e-08, + "litellm_provider": "lambda_ai", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_vision": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, + "lambda_ai/llama3.2-3b-instruct": { + "max_tokens": 131072, + "max_input_tokens": 131072, + "max_output_tokens": 131072, + "input_cost_per_token": 1.5e-08, + "output_cost_per_token": 2.5e-08, + "litellm_provider": "lambda_ai", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, + "lambda_ai/llama3.3-70b-instruct-fp8": { + "max_tokens": 131072, + "max_input_tokens": 131072, + "max_output_tokens": 131072, + "input_cost_per_token": 1.2e-07, + "output_cost_per_token": 3e-07, + "litellm_provider": "lambda_ai", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, + "lambda_ai/qwen25-coder-32b-instruct": { + "max_tokens": 131072, + "max_input_tokens": 131072, + "max_output_tokens": 131072, + "input_cost_per_token": 5e-08, + "output_cost_per_token": 1e-07, + "litellm_provider": "lambda_ai", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, + "lambda_ai/qwen3-32b-fp8": { + "max_tokens": 131072, + "max_input_tokens": 131072, + "max_output_tokens": 131072, + "input_cost_per_token": 5e-08, + "output_cost_per_token": 1e-07, + "litellm_provider": "lambda_ai", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_reasoning": true + }, + "hyperbolic/moonshotai/Kimi-K2-Instruct": { + "max_tokens": 131072, + "max_input_tokens": 131072, + "max_output_tokens": 131072, + "input_cost_per_token": 2e-06, + "output_cost_per_token": 2e-06, + "litellm_provider": "hyperbolic", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, + "hyperbolic/deepseek-ai/DeepSeek-R1-0528": { + "max_tokens": 131072, + "max_input_tokens": 131072, + "max_output_tokens": 131072, + "input_cost_per_token": 2.5e-07, + "output_cost_per_token": 2.5e-07, + "litellm_provider": "hyperbolic", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, + "hyperbolic/Qwen/Qwen3-235B-A22B": { + "max_tokens": 131072, + "max_input_tokens": 131072, + "max_output_tokens": 131072, + "input_cost_per_token": 2e-06, + "output_cost_per_token": 2e-06, + "litellm_provider": "hyperbolic", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, + "hyperbolic/deepseek-ai/DeepSeek-V3-0324": { + "max_tokens": 32768, + "max_input_tokens": 32768, + "max_output_tokens": 32768, + "input_cost_per_token": 4e-07, + "output_cost_per_token": 4e-07, + "litellm_provider": "hyperbolic", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, + "hyperbolic/Qwen/QwQ-32B": { + "max_tokens": 131072, + "max_input_tokens": 131072, + "max_output_tokens": 131072, + "input_cost_per_token": 2e-07, + "output_cost_per_token": 2e-07, + "litellm_provider": "hyperbolic", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, + "hyperbolic/deepseek-ai/DeepSeek-R1": { + "max_tokens": 32768, + "max_input_tokens": 32768, + "max_output_tokens": 32768, + "input_cost_per_token": 4e-07, + "output_cost_per_token": 4e-07, + "litellm_provider": "hyperbolic", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, + "hyperbolic/deepseek-ai/DeepSeek-V3": { + "max_tokens": 32768, + "max_input_tokens": 32768, + "max_output_tokens": 32768, + "input_cost_per_token": 2e-07, + "output_cost_per_token": 2e-07, + "litellm_provider": "hyperbolic", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, + "hyperbolic/meta-llama/Llama-3.3-70B-Instruct": { + "max_tokens": 131072, + "max_input_tokens": 131072, + "max_output_tokens": 131072, + "input_cost_per_token": 1.2e-07, + "output_cost_per_token": 3e-07, + "litellm_provider": "hyperbolic", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, + "hyperbolic/Qwen/Qwen2.5-Coder-32B-Instruct": { + "max_tokens": 32768, + "max_input_tokens": 32768, + "max_output_tokens": 32768, + "input_cost_per_token": 1.2e-07, + "output_cost_per_token": 3e-07, + "litellm_provider": "hyperbolic", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, + "hyperbolic/meta-llama/Llama-3.2-3B-Instruct": { + "max_tokens": 32768, + "max_input_tokens": 32768, + "max_output_tokens": 32768, + "input_cost_per_token": 1.2e-07, + "output_cost_per_token": 3e-07, + "litellm_provider": "hyperbolic", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, + "hyperbolic/Qwen/Qwen2.5-72B-Instruct": { + "max_tokens": 131072, + "max_input_tokens": 131072, + "max_output_tokens": 131072, + "input_cost_per_token": 1.2e-07, + "output_cost_per_token": 3e-07, + "litellm_provider": "hyperbolic", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, + "hyperbolic/meta-llama/Meta-Llama-3-70B-Instruct": { + "max_tokens": 131072, + "max_input_tokens": 131072, + "max_output_tokens": 131072, + "input_cost_per_token": 1.2e-07, + "output_cost_per_token": 3e-07, + "litellm_provider": "hyperbolic", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, + "hyperbolic/NousResearch/Hermes-3-Llama-3.1-70B": { + "max_tokens": 32768, + "max_input_tokens": 32768, + "max_output_tokens": 32768, + "input_cost_per_token": 1.2e-07, + "output_cost_per_token": 3e-07, + "litellm_provider": "hyperbolic", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, + "hyperbolic/meta-llama/Meta-Llama-3.1-405B-Instruct": { + "max_tokens": 32768, + "max_input_tokens": 32768, + "max_output_tokens": 32768, + "input_cost_per_token": 1.2e-07, + "output_cost_per_token": 3e-07, + "litellm_provider": "hyperbolic", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, + "hyperbolic/meta-llama/Meta-Llama-3.1-8B-Instruct": { + "max_tokens": 32768, + "max_input_tokens": 32768, + "max_output_tokens": 32768, + "input_cost_per_token": 1.2e-07, + "output_cost_per_token": 3e-07, + "litellm_provider": "hyperbolic", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, + "hyperbolic/meta-llama/Meta-Llama-3.1-70B-Instruct": { + "max_tokens": 32768, + "max_input_tokens": 32768, + "max_output_tokens": 32768, + "input_cost_per_token": 1.2e-07, + "output_cost_per_token": 3e-07, + "litellm_provider": "hyperbolic", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true }, "voyage/voyage-lite-01": { "max_tokens": 4096, diff --git a/tests/llm_translation/test_hyperbolic.py b/tests/llm_translation/test_hyperbolic.py new file mode 100644 index 00000000000..38f4dea4367 --- /dev/null +++ b/tests/llm_translation/test_hyperbolic.py @@ -0,0 +1,119 @@ +import os +import sys +from datetime import datetime +from unittest.mock import MagicMock + +import pytest + +sys.path.insert( + 0, os.path.abspath("../..") +) # Adds the parent directory to the system path + +import litellm +from litellm import get_llm_provider + + +def test_get_llm_provider_hyperbolic(): + """Test that hyperbolic/ prefix returns the correct provider""" + model, provider, _, _ = get_llm_provider(model="hyperbolic/deepseek-v3") + assert provider == "hyperbolic" + assert model == "deepseek-v3" + + +def test_hyperbolic_completion_call(): + """Test basic completion call structure for Hyperbolic""" + # This is primarily a structure test since we don't have actual API keys + try: + litellm.set_verbose = True + response = litellm.completion( + model="hyperbolic/qwen-2.5-72b", + messages=[{"role": "user", "content": "Hello!"}], + mock_response="Hi there!", + ) + assert response is not None + except Exception as e: + # Expected to fail without valid API key, but should recognize the provider + assert "hyperbolic" in str(e).lower() or "api" in str(e).lower() + + +def test_hyperbolic_config_initialization(): + """Test that HyperbolicChatConfig initializes correctly""" + from litellm.llms.hyperbolic.chat.transformation import HyperbolicChatConfig + + config = HyperbolicChatConfig() + assert config.custom_llm_provider == "hyperbolic" + + +def test_hyperbolic_get_openai_compatible_provider_info(): + """Test API base and key handling""" + from litellm.llms.hyperbolic.chat.transformation import HyperbolicChatConfig + + config = HyperbolicChatConfig() + + # Test default API base + api_base, api_key = config._get_openai_compatible_provider_info(None, None) + assert api_base == "https://api.hyperbolic.xyz/v1" + # api_key may be set from environment, so we don't test for None + + # Test custom API base + custom_base = "https://custom.hyperbolic.com/v1" + api_base, api_key = config._get_openai_compatible_provider_info(custom_base, "test-key") + assert api_base == custom_base + assert api_key == "test-key" + + +def test_hyperbolic_in_provider_lists(): + """Test that hyperbolic is in all relevant provider lists""" + from litellm.constants import ( + openai_compatible_endpoints, + openai_compatible_providers, + openai_text_completion_compatible_providers, + ) + + assert "hyperbolic" in openai_compatible_providers + assert "hyperbolic" in openai_text_completion_compatible_providers + assert "https://api.hyperbolic.xyz/v1" in openai_compatible_endpoints + + +def test_hyperbolic_models_configuration(): + """Test that Hyperbolic models are properly configured""" + import json + import os + + # Load model configuration directly from the JSON file + json_path = os.path.join(os.path.dirname(__file__), "../../model_prices_and_context_window.json") + with open(json_path, 'r') as f: + model_data = json.load(f) + + # Test a few key models + test_models = [ + "hyperbolic/deepseek-ai/DeepSeek-V3", + "hyperbolic/Qwen/Qwen2.5-Coder-32B-Instruct", + "hyperbolic/deepseek-ai/DeepSeek-R1", + ] + + for model in test_models: + assert model in model_data + model_info = model_data[model] + assert model_info["litellm_provider"] == "hyperbolic" + assert model_info["mode"] == "chat" + assert "max_tokens" in model_info + assert "input_cost_per_token" in model_info + assert "output_cost_per_token" in model_info + + +def test_hyperbolic_supported_params(): + """Test that supported OpenAI parameters are correctly configured""" + from litellm.llms.hyperbolic.chat.transformation import HyperbolicChatConfig + + config = HyperbolicChatConfig() + supported_params = config.get_supported_openai_params("hyperbolic/deepseek-v3") + + # Check for essential parameters + assert "messages" in supported_params + assert "model" in supported_params + assert "stream" in supported_params + assert "temperature" in supported_params + assert "max_tokens" in supported_params + assert "tools" in supported_params + assert "tool_choice" in supported_params \ No newline at end of file diff --git a/tests/llm_translation/test_lambda_ai.py b/tests/llm_translation/test_lambda_ai.py new file mode 100644 index 00000000000..3b73629b4a7 --- /dev/null +++ b/tests/llm_translation/test_lambda_ai.py @@ -0,0 +1,153 @@ +""" +Tests for Lambda AI provider integration +""" +import os +from unittest import mock + +import pytest + +import litellm +from litellm import completion +from litellm.llms.lambda_ai.chat.transformation import LambdaAIChatConfig + + +def test_lambda_ai_config_initialization(): + """Test LambdaAIChatConfig initializes correctly""" + config = LambdaAIChatConfig() + assert config.custom_llm_provider == "lambda_ai" + + +def test_lambda_ai_get_openai_compatible_provider_info(): + """Test Lambda AI provider info retrieval""" + config = LambdaAIChatConfig() + + # Test with default values (no env vars set) + with mock.patch.dict(os.environ, {}, clear=True): + api_base, api_key = config._get_openai_compatible_provider_info(None, None) + assert api_base == "https://api.lambda.ai/v1" + assert api_key is None + + # Test with environment variables + with mock.patch.dict(os.environ, {"LAMBDA_API_KEY": "test-key", "LAMBDA_API_BASE": "https://custom.lambda.ai/v1"}): + api_base, api_key = config._get_openai_compatible_provider_info(None, None) + assert api_base == "https://custom.lambda.ai/v1" + assert api_key == "test-key" + + # Test with explicit parameters (should override env vars) + with mock.patch.dict(os.environ, {"LAMBDA_API_KEY": "env-key", "LAMBDA_API_BASE": "https://env.lambda.ai/v1"}): + api_base, api_key = config._get_openai_compatible_provider_info( + "https://param.lambda.ai/v1", "param-key" + ) + assert api_base == "https://param.lambda.ai/v1" + assert api_key == "param-key" + + +def test_get_llm_provider_lambda_ai(): + """Test that get_llm_provider correctly identifies Lambda AI""" + from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider + + # Test with lambda_ai/model-name format + model, provider, api_key, api_base = get_llm_provider("lambda_ai/llama3.1-8b-instruct") + assert model == "llama3.1-8b-instruct" + assert provider == "lambda_ai" + + # Test with api_base containing Lambda AI endpoint + model, provider, api_key, api_base = get_llm_provider( + "llama3.1-8b-instruct", api_base="https://api.lambda.ai/v1" + ) + assert model == "llama3.1-8b-instruct" + assert provider == "lambda_ai" + assert api_base == "https://api.lambda.ai/v1" + + +def test_lambda_ai_in_provider_lists(): + """Test that Lambda AI is registered in all necessary provider lists""" + assert "lambda_ai" in litellm.openai_compatible_providers + assert "lambda_ai" in litellm.provider_list + assert "https://api.lambda.ai/v1" in litellm.openai_compatible_endpoints + + +@pytest.mark.asyncio +async def test_lambda_ai_completion_call(): + """Test completion call with Lambda AI provider (requires LAMBDA_API_KEY)""" + # Skip if no API key is available + if not os.getenv("LAMBDA_API_KEY"): + pytest.skip("LAMBDA_API_KEY not set") + + try: + response = await litellm.acompletion( + model="lambda_ai/llama3.1-8b-instruct", + messages=[{"role": "user", "content": "Hello, this is a test"}], + max_tokens=10, + ) + assert response.choices[0].message.content + assert response.model + assert response.usage + except Exception as e: + # If the API key is invalid or there's a network issue, that's okay + # The important thing is that the provider was recognized + if "lambda_ai" not in str(e) and "provider" not in str(e).lower(): + # Re-raise if it's not a provider-related error + raise + + +def test_lambda_ai_models_configuration(): + """Test that Lambda AI models are configured correctly""" + from litellm import get_model_info + + # Reload model cost map to pick up local changes + os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" + litellm.model_cost = litellm.get_model_cost_map(url="") + + # Clear and repopulate lambda_ai_models list after reloading model_cost + litellm.lambda_ai_models = [] + litellm.add_known_models() + + # Some Lambda AI models to test + lambda_ai_models = [ + "lambda_ai/deepseek-llama3.3-70b", + "lambda_ai/hermes3-8b", + "lambda_ai/llama3.1-8b-instruct", + "lambda_ai/llama3.2-11b-vision-instruct", + "lambda_ai/qwen25-coder-32b-instruct", + ] + + for model in lambda_ai_models: + model_info = get_model_info(model) + assert model_info is not None, f"Model info not found for {model}" + assert model_info.get("litellm_provider") == "lambda_ai", f"{model} should have lambda_ai as provider" + assert model_info.get("mode") == "chat", f"{model} should be in chat mode" + assert model_info.get("supports_function_calling") is True, f"{model} should support function calling" + assert model_info.get("supports_system_messages") is True, f"{model} should support system messages" + + # Check vision support for vision models + if "vision" in model: + assert model_info.get("supports_vision") is True, f"{model} should support vision" + + +def test_lambda_ai_model_list_populated(): + """Test that lambda_ai_models list is populated correctly""" + # Ensure we're using local model cost map and repopulate models + os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" + litellm.model_cost = litellm.get_model_cost_map(url="") + + # Clear and repopulate all model lists after reloading model_cost + litellm.lambda_ai_models = [] + litellm.add_known_models() + + # This should be populated by the add_known_models function + assert len(litellm.lambda_ai_models) > 0, "lambda_ai_models list should not be empty" + + # Check that all models in the list are Lambda AI models + for model in litellm.lambda_ai_models: + assert model.startswith("lambda_ai/"), f"Model {model} should start with 'lambda_ai/'" + + # Check some expected models are in the list + expected_models = [ + "lambda_ai/llama3.1-8b-instruct", + "lambda_ai/hermes3-405b", + "lambda_ai/deepseek-v3-0324", + ] + + for model in expected_models: + assert model in litellm.lambda_ai_models, f"{model} should be in lambda_ai_models list" \ No newline at end of file diff --git a/tests/llm_translation/test_morph.py b/tests/llm_translation/test_morph.py new file mode 100644 index 00000000000..7d2568a0a7f --- /dev/null +++ b/tests/llm_translation/test_morph.py @@ -0,0 +1,108 @@ +"""Unit tests for Morph provider integration.""" + +import os +import sys +from unittest.mock import patch + +sys.path.insert( + 0, os.path.abspath("../..") +) # Adds the parent directory to the system path + +import litellm +from litellm import MorphChatConfig, get_llm_provider + +# Force model loading +litellm.add_known_models() + + +def test_morph_config_get_provider_info(): + """Test that MorphChatConfig returns correct provider info.""" + config = MorphChatConfig() + + # Test with environment variable + with patch.dict(os.environ, {"MORPH_API_KEY": "test-key-from-env"}): + api_base, api_key = config._get_openai_compatible_provider_info(None, None) + assert api_base == "https://api.morphllm.com/v1" + assert api_key == "test-key-from-env" + + # Test with passed api_key + api_base, api_key = config._get_openai_compatible_provider_info(None, "direct-key") + assert api_base == "https://api.morphllm.com/v1" + assert api_key == "direct-key" + + # Test with custom api_base + api_base, api_key = config._get_openai_compatible_provider_info("https://custom.morph.com", "key") + assert api_base == "https://custom.morph.com" + assert api_key == "key" + + +def test_morph_get_llm_provider(): + """Test that get_llm_provider correctly identifies morph models.""" + # Test with morph/model format + _, custom_llm_provider, _, _ = get_llm_provider("morph/morph-v3-large") + assert custom_llm_provider == "morph" + + _, custom_llm_provider, _, _ = get_llm_provider("morph/morph-v3-fast") + assert custom_llm_provider == "morph" + + +def test_morph_in_provider_lists(): + """Test that morph is included in all necessary provider lists.""" + import litellm + from litellm.constants import openai_compatible_providers, openai_compatible_endpoints + + # Check morph is in openai_compatible_providers + assert "morph" in openai_compatible_providers + + # Check morph endpoint is in openai_compatible_endpoints + assert "https://api.morphllm.com/v1" in openai_compatible_endpoints + + # Check morph is in provider_list + assert "morph" in litellm.provider_list + + # Check models are in model_list after initialization + assert all(model in litellm.model_list for model in ["morph/morph-v3-large", "morph/morph-v3-fast"]) + + +def test_morph_model_info(): + """Test that morph models have correct configuration.""" + import litellm + model_info = litellm.get_model_info("morph/morph-v3-large") + + assert model_info["litellm_provider"] == "morph" + assert model_info["mode"] == "chat" + assert model_info["max_tokens"] == 16000 + assert model_info["max_input_tokens"] == 16000 + assert model_info["max_output_tokens"] == 16000 + assert model_info["input_cost_per_token"] == 9e-07 # $0.9/1M tokens + assert model_info["output_cost_per_token"] == 1.9e-06 # $1.9/1M tokens + assert model_info["supports_function_calling"] is False + assert model_info["supports_vision"] is False + assert model_info["supports_system_messages"] is True + + +def test_morph_supported_params(): + """Test that MorphChatConfig returns correct supported parameters.""" + config = MorphChatConfig() + supported_params = config.get_supported_openai_params("morph/morph-v3-large") + + expected_params = [ + "messages", + "model", + "stream", + "temperature", + "max_tokens", + "tools", + "tool_choice", + "response_format", + ] + + assert all(param in supported_params for param in expected_params) + + +def test_morph_custom_llm_provider(): + """Test that morph models are correctly identified.""" + config = MorphChatConfig() + assert config.custom_llm_provider == "morph" + + diff --git a/tests/llm_translation/test_optional_params.py b/tests/llm_translation/test_optional_params.py index 79a1dec7449..3ccdfe485b8 100644 --- a/tests/llm_translation/test_optional_params.py +++ b/tests/llm_translation/test_optional_params.py @@ -985,7 +985,7 @@ def test_watsonx_tool_choice(): model="gemini-1.5-pro", custom_llm_provider="watsonx", tool_choice="auto" ) print(optional_params) - assert optional_params["tool_choice_options"] == "auto" + assert optional_params["tool_choice_option"] == "auto" def test_watsonx_text_top_k(): diff --git a/tests/local_testing/test_completion.py b/tests/local_testing/test_completion.py index 8b6ed7ae594..9c3ae1e1f48 100644 --- a/tests/local_testing/test_completion.py +++ b/tests/local_testing/test_completion.py @@ -760,7 +760,7 @@ def test_completion_base64(model): pytest.fail(f"An exception occurred - {str(e)}") -@pytest.mark.parametrize("model", ["claude-3-5-sonnet-latest"]) +@pytest.mark.parametrize("model", ["claude-3-sonnet-20240229"]) def test_completion_function_plus_image(model): litellm.set_verbose = True diff --git a/tests/local_testing/test_exceptions.py b/tests/local_testing/test_exceptions.py index 807ffc60821..6851325cc1f 100644 --- a/tests/local_testing/test_exceptions.py +++ b/tests/local_testing/test_exceptions.py @@ -422,7 +422,7 @@ def test_anthropic_openai_exception(): old_azure_key = os.environ["ANTHROPIC_API_KEY"] os.environ.pop("ANTHROPIC_API_KEY") response = completion( - model="anthropic/claude-3-5-sonnet-latest", + model="anthropic/claude-3-sonnet-20240229", messages=[{"role": "user", "content": "hello"}], ) print(f"response: {response}") @@ -495,7 +495,6 @@ def test_completion_bedrock_invalid_role_exception(): == "litellm.BadRequestError: Invalid Message passed in {'role': 'very-bad-role', 'content': 'hello'}" ) - @pytest.mark.skip(reason="OpenAI exception changed to a generic error") def test_content_policy_exceptionimage_generation_openai(): try: @@ -774,15 +773,7 @@ def test_litellm_predibase_exception(): @pytest.mark.parametrize( - "provider", - [ - "predibase", - "vertex_ai_beta", - "anthropic", - "databricks", - "watsonx", - "fireworks_ai", - ], + "provider", ["predibase", "vertex_ai_beta", "anthropic", "databricks", "watsonx", "fireworks_ai"] ) def test_exception_mapping(provider): """ @@ -835,14 +826,14 @@ def test_fireworks_ai_exception_mapping(): 2. Text-based rate limit detection (the main issue fixed) 3. Generic 400 errors that should NOT be rate limits 4. ExceptionCheckers utility function - + Related to: https://github.com/BerriAI/litellm/pull/11455 Based on Fireworks AI documentation: https://docs.fireworks.ai/tools-sdks/python-client/api-reference """ import litellm from litellm.llms.fireworks_ai.common_utils import FireworksAIException from litellm.litellm_core_utils.exception_mapping_utils import ExceptionCheckers - + # Test scenarios covering all important cases test_scenarios = [ { @@ -864,63 +855,57 @@ def test_fireworks_ai_exception_mapping(): "expected_exception": litellm.BadRequestError, }, ] - + # Test each scenario for scenario in test_scenarios: mock_exception = FireworksAIException( - status_code=scenario["status_code"], message=scenario["message"], headers={} + status_code=scenario["status_code"], + message=scenario["message"], + headers={} ) - + try: response = litellm.completion( model="fireworks_ai/llama-v3p1-70b-instruct", messages=[{"role": "user", "content": "Hello"}], mock_response=mock_exception, ) - pytest.fail( - f"Expected {scenario['expected_exception'].__name__} to be raised" - ) + pytest.fail(f"Expected {scenario['expected_exception'].__name__} to be raised") except scenario["expected_exception"] as e: if scenario["expected_exception"] == litellm.RateLimitError: assert "rate limit" in str(e).lower() or "429" in str(e) except Exception as e: - pytest.fail( - f"Expected {scenario['expected_exception'].__name__} but got {type(e).__name__}: {e}" - ) - + pytest.fail(f"Expected {scenario['expected_exception'].__name__} but got {type(e).__name__}: {e}") + # Test ExceptionCheckers.is_error_str_rate_limit() method directly - + # Test cases that should return True (rate limit detected) rate_limit_strings = [ "429 rate limit exceeded", - "Rate limit exceeded, please try again later", + "Rate limit exceeded, please try again later", "RATE LIMIT ERROR", "Error 429: rate limit", '{"error":{"type":"invalid_request_error","message":"rate limit exceeded, please try again later"}}', "HTTP 429 Too Many Requests", ] - + for error_str in rate_limit_strings: - assert ExceptionCheckers.is_error_str_rate_limit( - error_str - ), f"Should detect rate limit in: {error_str}" - + assert ExceptionCheckers.is_error_str_rate_limit(error_str), f"Should detect rate limit in: {error_str}" + # Test cases that should return False (not rate limit) non_rate_limit_strings = [ "400 Bad Request", - "Authentication failed", + "Authentication failed", "Invalid model specified", "Context window exceeded", "Internal server error", "", "Some other error message", ] - + for error_str in non_rate_limit_strings: - assert not ExceptionCheckers.is_error_str_rate_limit( - error_str - ), f"Should NOT detect rate limit in: {error_str}" - + assert not ExceptionCheckers.is_error_str_rate_limit(error_str), f"Should NOT detect rate limit in: {error_str}" + # Test edge cases assert not ExceptionCheckers.is_error_str_rate_limit(None) # type: ignore assert not ExceptionCheckers.is_error_str_rate_limit(42) # type: ignore @@ -1157,7 +1142,6 @@ def test_openai_gateway_timeout_error(): """ openai_client = OpenAI() mapped_target = openai_client.chat.completions.with_raw_response # type: ignore - def _return_exception(*args, **kwargs): import datetime @@ -1191,17 +1175,13 @@ def test_openai_gateway_timeout_error(): setattr(exception, k, v) raise exception - try: + try: with patch.object( mapped_target, "create", side_effect=_return_exception, ): - litellm.completion( - model="openai/gpt-3.5-turbo", - messages=[{"role": "user", "content": "Hello world"}], - client=openai_client, - ) + litellm.completion(model="openai/gpt-3.5-turbo", messages=[{"role": "user", "content": "Hello world"}], client=openai_client) pytest.fail("Expected to raise Timeout") except litellm.Timeout as e: assert e.status_code == 504 @@ -1397,3 +1377,6 @@ async def test_exception_bubbling_up(sync_mode, stream_mode, model): assert exc_info.value.code == "invalid_value" assert exc_info.value.param is not None assert exc_info.value.type == "invalid_request_error" + + + diff --git a/tests/local_testing/test_function_calling.py b/tests/local_testing/test_function_calling.py index 82ed0f2a08e..a3443499d0e 100644 --- a/tests/local_testing/test_function_calling.py +++ b/tests/local_testing/test_function_calling.py @@ -791,7 +791,7 @@ async def test_watsonx_tool_choice(sync_mode): mock_completion.assert_called_once() print(mock_completion.call_args.kwargs) json_data = json.loads(mock_completion.call_args.kwargs["data"]) - json_data["tool_choice_options"] == "auto" + json_data["tool_choice_option"] == "auto" except Exception as e: print(e) if "The read operation timed out" in str(e): diff --git a/tests/local_testing/test_router.py b/tests/local_testing/test_router.py index bdc37bc3cce..3445dd5a519 100644 --- a/tests/local_testing/test_router.py +++ b/tests/local_testing/test_router.py @@ -125,6 +125,12 @@ async def test_router_provider_wildcard_routing(): print("response 3 = ", response3) + response4 = await router.acompletion( + model="claude-3-5-sonnet-latest", + messages=[{"role": "user", "content": "hello"}], + ) + + @pytest.mark.asyncio() async def test_router_provider_wildcard_routing_regex(): """ @@ -1130,7 +1136,7 @@ async def test_aimg_gen_on_router(): "api_base": os.getenv("AZURE_SWEDEN_API_BASE"), "api_key": os.getenv("AZURE_SWEDEN_API_KEY"), }, - }, + } ] router = Router(model_list=model_list, num_retries=3) response = await router.aimage_generation( @@ -2781,4 +2787,4 @@ def test_router_get_model_group_info(): assert model_group_info is not None assert model_group_info.model_group == "gpt-4" assert model_group_info.input_cost_per_token > 0 - assert model_group_info.output_cost_per_token > 0 + assert model_group_info.output_cost_per_token > 0 \ No newline at end of file diff --git a/tests/local_testing/test_streaming.py b/tests/local_testing/test_streaming.py index 924dff707eb..5db40946772 100644 --- a/tests/local_testing/test_streaming.py +++ b/tests/local_testing/test_streaming.py @@ -642,7 +642,6 @@ def test_completion_ollama_hosted_stream(): "model", [ # "claude-3-5-haiku-20241022", - # "claude-2", # "mistral/mistral-small-latest", "openrouter/openai/gpt-4o-mini", ], @@ -673,6 +672,7 @@ def test_completion_model_stream(model): pytest.fail(f"Error occurred: {e}") + @pytest.mark.parametrize( "sync_mode", [True, False], @@ -889,7 +889,6 @@ async def test_completion_gemini_stream_accumulated_json(sync_mode): # return pytest.fail(f"Error occurred: {e}") - @pytest.mark.flaky(retries=3, delay=1) def test_completion_mistral_api_mistral_large_function_call_with_streaming(): litellm.set_verbose = True diff --git a/tests/test_litellm/llms/azure/test_azure_common_utils.py b/tests/test_litellm/llms/azure/test_azure_common_utils.py index e076b66b7d2..bcdcc71ee31 100644 --- a/tests/test_litellm/llms/azure/test_azure_common_utils.py +++ b/tests/test_litellm/llms/azure/test_azure_common_utils.py @@ -12,7 +12,13 @@ sys.path.insert( ) # Adds the parent directory to the system path import litellm from litellm.llms.azure.common_utils import BaseAzureLLM, get_azure_ad_token +from litellm.secret_managers.get_azure_ad_token_provider import ( + get_azure_ad_token_provider, +) from litellm.types.router import GenericLiteLLMParams +from litellm.types.secret_managers.get_azure_ad_token_provider import ( + AzureCredentialType, +) from litellm.types.utils import CallTypes @@ -1298,7 +1304,7 @@ def test_get_azure_ad_token_with_token_refresh(setup_mocks, monkeypatch): # Verify the debug message was logged setup_mocks["logger"].debug.assert_any_call( - "Using Azure AD token provider based on Service Principal with Secret workflow for Azure Auth" + "Using Azure AD token provider based on Service Principal with Secret workflow or DefaultAzureCredential for Azure Auth" ) # Verify get_azure_ad_token_provider was called @@ -1325,7 +1331,7 @@ def test_get_azure_ad_token_with_token_refresh_error(setup_mocks): # Verify the debug message was logged setup_mocks["logger"].debug.assert_any_call( - "Using Azure AD token provider based on Service Principal with Secret workflow for Azure Auth" + "Using Azure AD token provider based on Service Principal with Secret workflow or DefaultAzureCredential for Azure Auth" ) # Verify error was logged @@ -1333,8 +1339,8 @@ def test_get_azure_ad_token_with_token_refresh_error(setup_mocks): "Azure AD Token Provider could not be used." ) - # Verify get_azure_ad_token_provider was called - setup_mocks["token_provider"].assert_called_once() + # Verify get_azure_ad_token_provider was called twice (once for service principal, once for DefaultAzureCredential) + assert setup_mocks["token_provider"].call_count == 2 # Verify the token is None since the provider raised an error assert token is None @@ -1380,3 +1386,101 @@ def test_token_provider_raises_exception(setup_mocks): # Verify the error was logged setup_mocks["logger"].error.assert_called() + + +def test_get_azure_ad_token_provider_with_default_azure_credential(): + """ + Test that get_azure_ad_token_provider correctly uses DefaultAzureCredential + when explicitly specified as the credential type. This verifies that the function + can dynamically instantiate DefaultAzureCredential and return a working token provider. + """ + # Mock Azure identity classes + with patch('azure.identity.DefaultAzureCredential') as mock_default_cred, \ + patch('azure.identity.get_bearer_token_provider') as mock_token_provider: + + # Configure mocks + mock_credential_instance = MagicMock() + mock_default_cred.return_value = mock_credential_instance + mock_token_provider.return_value = lambda: "test-default-azure-token" + + # Test with DefaultAzureCredential specified explicitly + token_provider = get_azure_ad_token_provider( + azure_scope="https://cognitiveservices.azure.com/.default", + azure_credential=AzureCredentialType.DefaultAzureCredential + ) + + # Verify DefaultAzureCredential was instantiated + mock_default_cred.assert_called_once_with() + + # Verify get_bearer_token_provider was called with the right parameters + mock_token_provider.assert_called_once_with( + mock_credential_instance, + "https://cognitiveservices.azure.com/.default" + ) + + # Verify the returned token provider works + token = token_provider() + assert token == "test-default-azure-token" + + +def test_get_azure_ad_token_fallback_to_default_azure_credential(setup_mocks, monkeypatch): + """ + Test that get_azure_ad_token falls back to DefaultAzureCredential when the + service principal method fails but token refresh is enabled. This tests the + complete fallback flow from service principal to DefaultAzureCredential. + """ + # Clear environment variables that might interfere + monkeypatch.delenv("AZURE_USERNAME", raising=False) + monkeypatch.delenv("AZURE_PASSWORD", raising=False) + monkeypatch.delenv("AZURE_CLIENT_SECRET", raising=False) + monkeypatch.delenv("AZURE_CLIENT_ID", raising=False) + monkeypatch.delenv("AZURE_TENANT_ID", raising=False) + + # Reset mocks to ensure clean state + setup_mocks["token_provider"].reset_mock() + + # Enable token refresh + setup_mocks["litellm"].enable_azure_ad_token_refresh = True + + # Configure get_azure_ad_token_provider to fail first (service principal) + # but succeed on second call (DefaultAzureCredential) + def mock_token_provider_side_effect(*args, **kwargs): + # If called with azure_credential=DefaultAzureCredential, return a working provider + if kwargs.get("azure_credential") == AzureCredentialType.DefaultAzureCredential: + return lambda: "mock-default-azure-credential-token" + # Otherwise (service principal call), return None to simulate failure + return None + + setup_mocks["token_provider"].side_effect = mock_token_provider_side_effect + + # Create test parameters with no other auth methods available + litellm_params = GenericLiteLLMParams() + + # Call the function + token = get_azure_ad_token(litellm_params) + + # Verify the success debug message was logged + setup_mocks["logger"].debug.assert_any_call( + "Successfully obtained Azure AD token provider using DefaultAzureCredential" + ) + + # Verify get_azure_ad_token_provider was called twice: + # 1. First with just azure_scope (service principal attempt) + # 2. Second with azure_credential=DefaultAzureCredential (fallback) + assert setup_mocks["token_provider"].call_count == 2 + + # Verify the calls were made with expected parameters + calls = setup_mocks["token_provider"].call_args_list + + # First call should be service principal attempt (no azure_credential) + first_call_kwargs = calls[0][1] + assert "azure_scope" in first_call_kwargs + assert first_call_kwargs.get("azure_credential") is None + + # Second call should be DefaultAzureCredential attempt + second_call_kwargs = calls[1][1] + assert "azure_scope" in second_call_kwargs + assert second_call_kwargs.get("azure_credential") == AzureCredentialType.DefaultAzureCredential + + # Verify the token is what we expect from our DefaultAzureCredential mock + assert token == "mock-default-azure-credential-token" diff --git a/tests/test_litellm/llms/github_copilot/test_github_copilot_authenticator.py b/tests/test_litellm/llms/github_copilot/test_github_copilot_authenticator.py index 65413d46b92..c6ae2b9c4e1 100644 --- a/tests/test_litellm/llms/github_copilot/test_github_copilot_authenticator.py +++ b/tests/test_litellm/llms/github_copilot/test_github_copilot_authenticator.py @@ -178,3 +178,14 @@ class TestGitHubCopilotAuthenticator: authenticator._get_device_code.assert_called_once() authenticator._poll_for_access_token.assert_called_once_with("mock-device-code") mock_print.assert_called_once() + + def test_get_api_base_from_file(self, authenticator): + """Test retrieving the API base endpoint from a file.""" + mock_api_key_data = json.dumps({ + "token": "mock-api-key", + "expires_at": (datetime.now() + timedelta(hours=1)).timestamp(), + "endpoints": {"api": "https://api.enterprise.githubcopilot.com"} + }) + with patch("builtins.open", mock_open(read_data=mock_api_key_data)): + api_base = authenticator.get_api_base() + assert api_base == "https://api.enterprise.githubcopilot.com" diff --git a/tests/test_litellm/llms/github_copilot/test_github_copilot_transformation.py b/tests/test_litellm/llms/github_copilot/test_github_copilot_transformation.py index 9672f045df6..388a50243f4 100644 --- a/tests/test_litellm/llms/github_copilot/test_github_copilot_transformation.py +++ b/tests/test_litellm/llms/github_copilot/test_github_copilot_transformation.py @@ -40,6 +40,8 @@ def test_github_copilot_config_get_openai_compatible_provider_info(): mock_api_key = "gh.test-key-123456789" config.authenticator = MagicMock() config.authenticator.get_api_key.return_value = mock_api_key + # Test with dynamic endpoint + config.authenticator.get_api_base.return_value = "https://api.enterprise.githubcopilot.com" # Test with default values model = "github_copilot/gpt-4" @@ -54,10 +56,24 @@ def test_github_copilot_config_get_openai_compatible_provider_info(): custom_llm_provider="github_copilot", ) - assert api_base == "https://api.githubcopilot.com/" + assert api_base == "https://api.enterprise.githubcopilot.com" assert dynamic_api_key == mock_api_key assert custom_llm_provider == "github_copilot" + # Test fallback to default if no dynamic endpoint + config.authenticator.get_api_base.return_value = None + ( + api_base, + dynamic_api_key, + custom_llm_provider, + ) = config._get_openai_compatible_provider_info( + model=model, + api_base=None, + api_key=None, + custom_llm_provider="github_copilot", + ) + assert api_base == "https://api.githubcopilot.com/" + # Test with authentication failure config.authenticator.get_api_key.side_effect = GetAPIKeyError( message="Failed to get API key", @@ -128,8 +144,8 @@ def test_completion_github_copilot_mock_response(mock_completion, mock_get_api_k def test_transform_messages_disable_copilot_system_to_assistant(monkeypatch): """Test that system messages are converted to assistant unless disable_copilot_system_to_assistant is True.""" - from litellm.llms.github_copilot.chat.transformation import GithubCopilotConfig import litellm + from litellm.llms.github_copilot.chat.transformation import GithubCopilotConfig # Save original value original_flag = litellm.disable_copilot_system_to_assistant diff --git a/tests/test_litellm/test_main.py b/tests/test_litellm/test_main.py index b9a71e9621f..fd9c366a2f0 100644 --- a/tests/test_litellm/test_main.py +++ b/tests/test_litellm/test_main.py @@ -1097,28 +1097,3 @@ def test_stream_chunk_builder_thinking_blocks(): assert response is not None assert response.choices[0].message.content is not None assert response.choices[0].message.thinking_blocks is not None - - -from litellm.llms.openai.openai import OpenAIChatCompletion - - -def throw_retryable_error(*_, **__): - raise RuntimeError("BOOM") - - -@pytest.mark.asyncio -async def test_retrying() -> None: - litellm.num_retries = 10 - with ( - patch.object( - OpenAIChatCompletion, - "make_openai_chat_completion_request", - side_effect=throw_retryable_error, - ) as mock_request, - pytest.raises(litellm.InternalServerError, match="LiteLLM Retried: 10 times"), - ): - await litellm.acompletion( - model="gpt-4o-mini", - messages=[{"role": "user", "content": "Hello"}], - ) - assert mock_request.call_count >= 10, "Expected retrying to be used" diff --git a/tests/test_openai_endpoints.py b/tests/test_openai_endpoints.py index 0b5f1a94339..16b9838d80b 100644 --- a/tests/test_openai_endpoints.py +++ b/tests/test_openai_endpoints.py @@ -570,7 +570,7 @@ async def test_proxy_all_models(): await chat_completion( session=session, key=LITELLM_MASTER_KEY, - model="anthropic/claude-3-5-sonnet-latest", + model="anthropic/claude-3-sonnet-20240229", ) diff --git a/ui/litellm-dashboard/src/components/team/team_info.tsx b/ui/litellm-dashboard/src/components/team/team_info.tsx index 402f0b26e98..e72b6330c3a 100644 --- a/ui/litellm-dashboard/src/components/team/team_info.tsx +++ b/ui/litellm-dashboard/src/components/team/team_info.tsx @@ -30,6 +30,7 @@ import { teamMemberUpdateCall, Member, teamUpdateCall, + getGuardrailsList, } from "@/components/networking"; import { Button, Form, Input, Select, message, Tooltip } from "antd"; import { InfoCircleOutlined } from "@ant-design/icons"; @@ -147,6 +148,7 @@ const TeamInfoView: React.FC = ({ const [mcpAccessGroups, setMcpAccessGroups] = useState([]); const [mcpAccessGroupsLoaded, setMcpAccessGroupsLoaded] = useState(false); const [copiedStates, setCopiedStates] = useState>({}) + const [guardrailsList, setGuardrailsList] = useState([]); console.log("userModels in team info", userModels); @@ -182,6 +184,23 @@ const TeamInfoView: React.FC = ({ } }; + useEffect(() => { + const fetchGuardrails = async () => { + try { + if (!accessToken) return; + const response = await getGuardrailsList(accessToken); + const guardrailNames = response.guardrails.map( + (g: { guardrail_name: string }) => g.guardrail_name + ); + setGuardrailsList(guardrailNames); + } catch (error) { + console.error("Failed to fetch guardrails:", error); + } + }; + + fetchGuardrails(); + }, [accessToken]); + const handleMemberCreate = async (values: any) => { try { if (accessToken == null) return; @@ -673,13 +692,14 @@ const TeamInfoView: React.FC = ({