diff --git a/docker-compose.yml b/docker-compose.yml index c268f9ba0ff..8898aff62da 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -22,7 +22,9 @@ services: depends_on: - db # Indicates that this service depends on the 'db' service, ensuring 'db' starts first healthcheck: # Defines the health check configuration for the container - test: [ "CMD-SHELL", "wget --no-verbose --tries=1 http://localhost:4000/health/liveliness || exit 1" ] # Command to execute for health check + test: + - CMD-SHELL + - python3 -c "import urllib.request; urllib.request.urlopen('http://localhost:4000/health/liveliness')" # Command to execute for health check interval: 30s # Perform health check every 30 seconds timeout: 10s # Health check command times out after 10 seconds retries: 3 # Retry up to 3 times if health check fails diff --git a/docs/my-website/docs/assistants.md b/docs/my-website/docs/assistants.md index d262b492a70..2960d0fded8 100644 --- a/docs/my-website/docs/assistants.md +++ b/docs/my-website/docs/assistants.md @@ -3,6 +3,14 @@ import TabItem from '@theme/TabItem'; # /assistants +:::warning Deprecation Notice + +OpenAI has deprecated the Assistants API. It will shut down on **August 26, 2026**. + +Consider migrating to the [Responses API](/docs/response_api) instead. See [OpenAI's migration guide](https://platform.openai.com/docs/guides/responses-vs-assistants) for details. + +::: + Covers Threads, Messages, Assistants. LiteLLM currently covers: diff --git a/docs/my-website/docs/completion/drop_params.md b/docs/my-website/docs/completion/drop_params.md index 590d9a45955..a81fd897b4e 100644 --- a/docs/my-website/docs/completion/drop_params.md +++ b/docs/my-website/docs/completion/drop_params.md @@ -5,6 +5,14 @@ import TabItem from '@theme/TabItem'; Drop unsupported OpenAI params by your LLM Provider. +## Default Behavior + +**By default, LiteLLM raises an exception** if you send a parameter to a model that doesn't support it. + +For example, if you send `temperature=0.2` to a model that doesn't support the `temperature` parameter, LiteLLM will raise an exception. + +**When `drop_params=True` is set**, LiteLLM will drop the unsupported parameter instead of raising an exception. This allows your code to work seamlessly across different providers without having to customize parameters for each one. + ## Quick Start ```python diff --git a/docs/my-website/docs/completion/json_mode.md b/docs/my-website/docs/completion/json_mode.md index c86a1e59893..0122e202610 100644 --- a/docs/my-website/docs/completion/json_mode.md +++ b/docs/my-website/docs/completion/json_mode.md @@ -126,6 +126,8 @@ resp = completion( ) print("Received={}".format(resp)) + +events_list = EventsList.model_validate_json(resp.choices[0].message.content) ``` diff --git a/docs/my-website/docs/contribute_integration/custom_webhook_api.md b/docs/my-website/docs/contribute_integration/custom_webhook_api.md index 499c7fd51de..158937d2a43 100644 --- a/docs/my-website/docs/contribute_integration/custom_webhook_api.md +++ b/docs/my-website/docs/contribute_integration/custom_webhook_api.md @@ -95,11 +95,19 @@ curl -L -X POST 'http://0.0.0.0:4000/chat/completions' \ }' ``` -4. File a PR! +4. Add Documentation + +If you're adding a new integration, please add documentation for it under the `observability` folder: + +- Create a new file at `docs/my-website/docs/observability/_integration.md` +- Follow the format of existing integration docs, such as [Langsmith Integration](https://github.com/BerriAI/litellm/blob/main/docs/my-website/docs/observability/langsmith_integration.md) +- Include: Quick Start, SDK usage, Proxy usage, and any advanced configuration options + +5. File a PR! - Review our contribution guide [here](../../extras/contributing_code) -- push your fork to your GitHub repo -- submit a PR from there +- Push your fork to your GitHub repo +- Submit a PR from there ## What get's logged? diff --git a/docs/my-website/docs/embedding/supported_embedding.md b/docs/my-website/docs/embedding/supported_embedding.md index 0e8252b409b..11ca4da48a4 100644 --- a/docs/my-website/docs/embedding/supported_embedding.md +++ b/docs/my-website/docs/embedding/supported_embedding.md @@ -10,6 +10,26 @@ import os os.environ['OPENAI_API_KEY'] = "" response = embedding(model='text-embedding-ada-002', input=["good morning from litellm"]) ``` + +## Async Usage - `aembedding()` + +LiteLLM provides an asynchronous version of the `embedding` function called `aembedding`: + +```python +from litellm import aembedding +import asyncio + +async def get_embedding(): + response = await aembedding( + model='text-embedding-ada-002', + input=["good morning from litellm"] + ) + return response + +response = asyncio.run(get_embedding()) +print(response) +``` + ## Proxy Usage **NOTE** diff --git a/docs/my-website/docs/index.md b/docs/my-website/docs/index.md index 11d2963b7a3..c6e335e4cc3 100644 --- a/docs/my-website/docs/index.md +++ b/docs/my-website/docs/index.md @@ -7,8 +7,8 @@ https://github.com/BerriAI/litellm ## **Call 100+ LLMs using the OpenAI Input/Output Format** -- Translate inputs to provider's `completion`, `embedding`, and `image_generation` endpoints -- [Consistent output](https://docs.litellm.ai/docs/completion/output), text responses will always be available at `['choices'][0]['message']['content']` +- Translate inputs to provider's endpoints (`/chat/completions`, `/responses`, `/embeddings`, `/images`, `/audio`, `/batches`, and more) +- [Consistent output](https://docs.litellm.ai/docs/supported_endpoints) - same response format regardless of which provider you use - Retry/fallback logic across multiple deployments (e.g. Azure/OpenAI) - [Router](https://docs.litellm.ai/docs/routing) - Track spend & set budgets per project [LiteLLM Proxy Server](https://docs.litellm.ai/docs/simple_proxy) @@ -245,7 +245,7 @@ response = completion( -### Response Format (OpenAI Format) +### Response Format (OpenAI Chat Completions Format) ```json { @@ -514,15 +514,22 @@ response = completion( LiteLLM maps exceptions across all supported providers to the OpenAI exceptions. All our exceptions inherit from OpenAI's exception types, so any error-handling you have for that, should work out of the box with LiteLLM. ```python -from openai.error import OpenAIError +import litellm from litellm import completion +import os os.environ["ANTHROPIC_API_KEY"] = "bad-key" try: - # some code - completion(model="claude-instant-1", messages=[{"role": "user", "content": "Hey, how's it going?"}]) -except OpenAIError as e: - print(e) + completion(model="anthropic/claude-instant-1", messages=[{"role": "user", "content": "Hey, how's it going?"}]) +except litellm.AuthenticationError as e: + # Thrown when the API key is invalid + print(f"Authentication failed: {e}") +except litellm.RateLimitError as e: + # Thrown when you've exceeded your rate limit + print(f"Rate limited: {e}") +except litellm.APIError as e: + # Thrown for general API errors + print(f"API error: {e}") ``` ### See How LiteLLM Transforms Your Requests diff --git a/docs/my-website/docs/observability/helicone_integration.md b/docs/my-website/docs/observability/helicone_integration.md index 22ea051f7cd..92d0f5c3ebf 100644 --- a/docs/my-website/docs/observability/helicone_integration.md +++ b/docs/my-website/docs/observability/helicone_integration.md @@ -10,7 +10,7 @@ https://github.com/BerriAI/litellm ::: -[Helicone](https://helicone.ai/) is an open source observability platform that proxies your LLM requests and provides key insights into your usage, spend, latency and more. +[Helicone](https://helicone.ai/) is an open sourced observability platform providing key insights into your usage, spend, latency and more. ## Quick Start @@ -25,14 +25,10 @@ from litellm import completion ## Set env variables os.environ["HELICONE_API_KEY"] = "your-helicone-key" -os.environ["OPENAI_API_KEY"] = "your-openai-key" - -# Set callbacks -litellm.success_callback = ["helicone"] # OpenAI call response = completion( - model="gpt-4o", + model="helicone/gpt-4o-mini", messages=[{"role": "user", "content": "Hi πŸ‘‹ - I'm OpenAI"}], ) @@ -54,7 +50,7 @@ model_list: # Add Helicone callback litellm_settings: success_callback: ["helicone"] - + # Set Helicone API key environment_variables: HELICONE_API_KEY: "your-helicone-key" @@ -72,12 +68,12 @@ litellm --config config.yaml There are two main approaches to integrate Helicone with LiteLLM: -1. **Callbacks**: Log to Helicone while using any provider -2. **Proxy Mode**: Use Helicone as a proxy for advanced features +1. **As a Provider**: Use Helicone to log requests for [all models supported ](../providers/helicone) +2. **Callbacks**: Log to Helicone while using any provider ### Supported LLM Providers -Helicone can log requests across [various LLM providers](https://docs.helicone.ai/getting-started/quick-start), including: +Helicone can log requests across [all major LLM providers](https://helicone.ai/models), including: - OpenAI - Azure @@ -88,156 +84,149 @@ Helicone can log requests across [various LLM providers](https://docs.helicone.a - Replicate - And more -## Method 1: Using Callbacks +## Method 1: Using Helicone as a Provider + +Helicone's AI Gateway provides [advanced functionality](https://docs.helicone.ai) like caching, rate limiting, LLM security, and more. + + + + + Set Helicone as your base URL and pass authentication headers: + + ```python + import os + import litellm + from litellm import completion + + os.environ["HELICONE_API_KEY"] = "" # your Helicone API key + + messages = [{"content": "What is the capital of France?", "role": "user"}] + + # Helicone call - routes through Helicone gateway to any model + response = completion( + model="helicone/gpt-4o-mini", # or any 100+ models + messages=messages + ) + + print(response) + ``` + + ### Advanced Usage + + You can add custom metadata and properties to your requests using Helicone headers. Here are some examples: + + ```python + litellm.metadata = { + "Helicone-User-Id": "user-abc", # Specify the user making the request + "Helicone-Property-App": "web", # Custom property to add additional information + "Helicone-Property-Custom": "any-value", # Add any custom property + "Helicone-Prompt-Id": "prompt-supreme-court", # Assign an ID to associate this prompt with future versions + "Helicone-Cache-Enabled": "true", # Enable caching of responses + "Cache-Control": "max-age=3600", # Set cache limit to 1 hour + "Helicone-RateLimit-Policy": "10;w=60;s=user", # Set rate limit policy + "Helicone-Retry-Enabled": "true", # Enable retry mechanism + "helicone-retry-num": "3", # Set number of retries + "helicone-retry-factor": "2", # Set exponential backoff factor + "Helicone-Model-Override": "gpt-3.5-turbo-0613", # Override the model used for cost calculation + "Helicone-Session-Id": "session-abc-123", # Set session ID for tracking + "Helicone-Session-Path": "parent-trace/child-trace", # Set session path for hierarchical tracking + "Helicone-Omit-Response": "false", # Include response in logging (default behavior) + "Helicone-Omit-Request": "false", # Include request in logging (default behavior) + "Helicone-LLM-Security-Enabled": "true", # Enable LLM security features + "Helicone-Moderations-Enabled": "true", # Enable content moderation + } + ``` + + ### Caching and Rate Limiting + + Enable caching and set up rate limiting policies: + + ```python + litellm.metadata = { + "Helicone-Cache-Enabled": "true", # Enable caching of responses + "Cache-Control": "max-age=3600", # Set cache limit to 1 hour + "Helicone-RateLimit-Policy": "100;w=3600;s=user", # Set rate limit policy + } + ``` + + + + +## Method 2: Using Callbacks Log requests to Helicone while using any LLM provider directly. - + -```python -import os -import litellm -from litellm import completion + ```python + import os + import litellm + from litellm import completion -## Set env variables -os.environ["HELICONE_API_KEY"] = "your-helicone-key" -os.environ["OPENAI_API_KEY"] = "your-openai-key" -# os.environ["HELICONE_API_BASE"] = "" # [OPTIONAL] defaults to `https://api.helicone.ai` + ## Set env variables + os.environ["HELICONE_API_KEY"] = "your-helicone-key" + os.environ["OPENAI_API_KEY"] = "your-openai-key" + # os.environ["HELICONE_API_BASE"] = "" # [OPTIONAL] defaults to `https://api.helicone.ai` -# Set callbacks -litellm.success_callback = ["helicone"] + # Set callbacks + litellm.success_callback = ["helicone"] -# OpenAI call -response = completion( - model="gpt-4o", - messages=[{"role": "user", "content": "Hi πŸ‘‹ - I'm OpenAI"}], -) + # OpenAI call + response = completion( + model="gpt-4o", + messages=[{"role": "user", "content": "Hi πŸ‘‹ - I'm OpenAI"}], + ) -print(response) -``` + print(response) + ``` - - + + -```yaml title="config.yaml" -model_list: - - model_name: gpt-4 - litellm_params: - model: gpt-4 - api_key: os.environ/OPENAI_API_KEY - - model_name: claude-3 - litellm_params: - model: anthropic/claude-3-sonnet-20240229 - api_key: os.environ/ANTHROPIC_API_KEY + ```yaml title="config.yaml" + model_list: + - model_name: gpt-4 + litellm_params: + model: gpt-4 + api_key: os.environ/OPENAI_API_KEY + - model_name: claude-3 + litellm_params: + model: anthropic/claude-3-sonnet-20240229 + api_key: os.environ/ANTHROPIC_API_KEY -# Add Helicone logging -litellm_settings: - success_callback: ["helicone"] - -# Environment variables -environment_variables: - HELICONE_API_KEY: "your-helicone-key" - OPENAI_API_KEY: "your-openai-key" - ANTHROPIC_API_KEY: "your-anthropic-key" -``` + # Add Helicone logging + litellm_settings: + success_callback: ["helicone"] -Start the proxy: -```bash -litellm --config config.yaml -``` + # Environment variables + environment_variables: + HELICONE_API_KEY: "your-helicone-key" + OPENAI_API_KEY: "your-openai-key" + ANTHROPIC_API_KEY: "your-anthropic-key" + ``` -Make requests to your proxy: -```python -import openai + Start the proxy: + ```bash + litellm --config config.yaml + ``` -client = openai.OpenAI( - api_key="anything", # proxy doesn't require real API key - base_url="http://localhost:4000" -) + Make requests to your proxy: + ```python + import openai -response = client.chat.completions.create( - model="gpt-4", # This gets logged to Helicone - messages=[{"role": "user", "content": "Hello!"}] -) -``` + client = openai.OpenAI( + api_key="anything", # proxy doesn't require real API key + base_url="http://localhost:4000" + ) - - + response = client.chat.completions.create( + model="gpt-4", # This gets logged to Helicone + messages=[{"role": "user", "content": "Hello!"}] + ) + ``` -## Method 2: Using Helicone as a Proxy - -Helicone's proxy provides [advanced functionality](https://docs.helicone.ai/getting-started/proxy-vs-async) like caching, rate limiting, LLM security through [PromptArmor](https://promptarmor.com/) and more. - - - - -Set Helicone as your base URL and pass authentication headers: - -```python -import os -import litellm -from litellm import completion - -# Configure LiteLLM to use Helicone proxy -litellm.api_base = "https://oai.hconeai.com/v1" -litellm.headers = { - "Helicone-Auth": f"Bearer {os.getenv('HELICONE_API_KEY')}", -} - -# Set your OpenAI API key -os.environ["OPENAI_API_KEY"] = "your-openai-key" - -response = completion( - model="gpt-3.5-turbo", - messages=[{"role": "user", "content": "How does a court case get to the Supreme Court?"}] -) - -print(response) -``` - -### Advanced Usage - -You can add custom metadata and properties to your requests using Helicone headers. Here are some examples: - -```python -litellm.metadata = { - "Helicone-Auth": f"Bearer {os.getenv('HELICONE_API_KEY')}", # Authenticate to send requests to Helicone API - "Helicone-User-Id": "user-abc", # Specify the user making the request - "Helicone-Property-App": "web", # Custom property to add additional information - "Helicone-Property-Custom": "any-value", # Add any custom property - "Helicone-Prompt-Id": "prompt-supreme-court", # Assign an ID to associate this prompt with future versions - "Helicone-Cache-Enabled": "true", # Enable caching of responses - "Cache-Control": "max-age=3600", # Set cache limit to 1 hour - "Helicone-RateLimit-Policy": "10;w=60;s=user", # Set rate limit policy - "Helicone-Retry-Enabled": "true", # Enable retry mechanism - "helicone-retry-num": "3", # Set number of retries - "helicone-retry-factor": "2", # Set exponential backoff factor - "Helicone-Model-Override": "gpt-3.5-turbo-0613", # Override the model used for cost calculation - "Helicone-Session-Id": "session-abc-123", # Set session ID for tracking - "Helicone-Session-Path": "parent-trace/child-trace", # Set session path for hierarchical tracking - "Helicone-Omit-Response": "false", # Include response in logging (default behavior) - "Helicone-Omit-Request": "false", # Include request in logging (default behavior) - "Helicone-LLM-Security-Enabled": "true", # Enable LLM security features - "Helicone-Moderations-Enabled": "true", # Enable content moderation - "Helicone-Fallbacks": '["gpt-3.5-turbo", "gpt-4"]', # Set fallback models -} -``` - -### Caching and Rate Limiting - -Enable caching and set up rate limiting policies: - -```python -litellm.metadata = { - "Helicone-Auth": f"Bearer {os.getenv('HELICONE_API_KEY')}", # Authenticate to send requests to Helicone API - "Helicone-Cache-Enabled": "true", # Enable caching of responses - "Cache-Control": "max-age=3600", # Set cache limit to 1 hour - "Helicone-RateLimit-Policy": "100;w=3600;s=user", # Set rate limit policy -} -``` - - + ## Session Tracking and Tracing @@ -245,57 +234,62 @@ litellm.metadata = { Track multi-step and agentic LLM interactions using session IDs and paths: - + -```python -import litellm + ```python + import os + import litellm + from litellm import completion -litellm.api_base = "https://oai.hconeai.com/v1" -litellm.metadata = { - "Helicone-Auth": f"Bearer {os.getenv('HELICONE_API_KEY')}", - "Helicone-Session-Id": "session-abc-123", - "Helicone-Session-Path": "parent-trace/child-trace", -} + os.environ["HELICONE_API_KEY"] = "" # your Helicone API key -response = litellm.completion( - model="gpt-3.5-turbo", - messages=[{"role": "user", "content": "Start a conversation"}] -) -``` + messages = [{"content": "What is the capital of France?", "role": "user"}] - - + response = completion( + model="helicone/gpt-4", + messages=messages, + metadata={ + "Helicone-Session-Id": "session-abc-123", + "Helicone-Session-Path": "parent-trace/child-trace", + } + ) -```python -import openai + print(response) + ``` -client = openai.OpenAI( - api_key="anything", - base_url="http://localhost:4000" -) + + -# First request in session -response1 = client.chat.completions.create( - model="gpt-4", - messages=[{"role": "user", "content": "Hello"}], - extra_headers={ - "Helicone-Session-Id": "session-abc-123", - "Helicone-Session-Path": "conversation/greeting" - } -) + ```python + import openai -# Follow-up request in same session -response2 = client.chat.completions.create( - model="gpt-4", - messages=[{"role": "user", "content": "Tell me more"}], - extra_headers={ - "Helicone-Session-Id": "session-abc-123", - "Helicone-Session-Path": "conversation/follow-up" - } -) -``` + client = openai.OpenAI( + api_key="anything", + base_url="http://localhost:4000" + ) - + # First request in session + response1 = client.chat.completions.create( + model="gpt-4", + messages=[{"role": "user", "content": "Hello"}], + extra_headers={ + "Helicone-Session-Id": "session-abc-123", + "Helicone-Session-Path": "conversation/greeting" + } + ) + + # Follow-up request in same session + response2 = client.chat.completions.create( + model="gpt-4", + messages=[{"role": "user", "content": "Tell me more"}], + extra_headers={ + "Helicone-Session-Id": "session-abc-123", + "Helicone-Session-Path": "conversation/follow-up" + } + ) + ``` + + - `Helicone-Session-Id`: Unique identifier for the session to group related requests @@ -304,52 +298,50 @@ response2 = client.chat.completions.create( ## Retry and Fallback Mechanisms - + -```python -import litellm + ```python + import litellm -litellm.api_base = "https://oai.hconeai.com/v1" -litellm.metadata = { - "Helicone-Auth": f"Bearer {os.getenv('HELICONE_API_KEY')}", - "Helicone-Retry-Enabled": "true", - "helicone-retry-num": "3", - "helicone-retry-factor": "2", # Exponential backoff - "Helicone-Fallbacks": '["gpt-3.5-turbo", "gpt-4"]', -} + litellm.api_base = "https://ai-gateway.helicone.ai/" + litellm.metadata = { + "Helicone-Retry-Enabled": "true", + "helicone-retry-num": "3", + "helicone-retry-factor": "2", + } -response = litellm.completion( - model="gpt-4", - messages=[{"role": "user", "content": "Hello"}] -) -``` + response = litellm.completion( + model="helicone/gpt-4o-mini/openai,claude-3-5-sonnet-20241022/anthropic", # Try OpenAI first, then fallback to Anthropic, then continue with other models + messages=[{"role": "user", "content": "Hello"}] + ) + ``` - - + + -```yaml title="config.yaml" -model_list: - - model_name: gpt-4 - litellm_params: - model: gpt-4 - api_key: os.environ/OPENAI_API_KEY - api_base: "https://oai.hconeai.com/v1" + ```yaml title="config.yaml" + model_list: + - model_name: gpt-4 + litellm_params: + model: gpt-4 + api_key: os.environ/OPENAI_API_KEY + api_base: "https://oai.hconeai.com/v1" -default_litellm_params: - headers: - Helicone-Auth: "Bearer ${HELICONE_API_KEY}" - Helicone-Retry-Enabled: "true" - helicone-retry-num: "3" - helicone-retry-factor: "2" - Helicone-Fallbacks: '["gpt-3.5-turbo", "gpt-4"]' + default_litellm_params: + headers: + Helicone-Auth: "Bearer ${HELICONE_API_KEY}" + Helicone-Retry-Enabled: "true" + helicone-retry-num: "3" + helicone-retry-factor: "2" + Helicone-Fallbacks: '["gpt-3.5-turbo", "gpt-4"]' -environment_variables: - HELICONE_API_KEY: "your-helicone-key" - OPENAI_API_KEY: "your-openai-key" -``` + environment_variables: + HELICONE_API_KEY: "your-helicone-key" + OPENAI_API_KEY: "your-openai-key" + ``` - + -> **Supported Headers** - For a full list of supported Helicone headers and their descriptions, please refer to the [Helicone documentation](https://docs.helicone.ai/getting-started/quick-start). +> **Supported Headers** - For a full list of supported Helicone headers and their descriptions, please refer to the [Helicone documentation](https://docs.helicone.ai/features/advanced-usage/custom-properties). > By utilizing these headers and metadata options, you can gain deeper insights into your LLM usage, optimize performance, and better manage your AI workflows with Helicone and LiteLLM. diff --git a/docs/my-website/docs/observability/sumologic_integration.md b/docs/my-website/docs/observability/sumologic_integration.md new file mode 100644 index 00000000000..d0894146e4c --- /dev/null +++ b/docs/my-website/docs/observability/sumologic_integration.md @@ -0,0 +1,287 @@ +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +# Sumo Logic + +Send LiteLLM logs to Sumo Logic for observability, monitoring, and analysis. + +Sumo Logic is a cloud-native machine data analytics platform that provides real-time insights into your applications and infrastructure. +https://www.sumologic.com/ + +:::info +We want to learn how we can make the callbacks better! Meet the LiteLLM [founders](https://calendly.com/d/4mp-gd3-k5k/berriai-1-1-onboarding-litellm-hosted-version) or +join our [discord](https://discord.gg/wuPM9dRgDw) +::: + +## Pre-Requisites + +1. Create a Sumo Logic account at https://www.sumologic.com/ +2. Set up an HTTP Logs and Metrics Source in Sumo Logic: + - Go to **Manage Data** > **Collection** > **Collection** + - Click **Add Source** next to a Hosted Collector + - Select **HTTP Logs & Metrics** + - Copy the generated URL (it contains the authentication token) + +For more details, see the [HTTP Logs & Metrics Source](https://www.sumologic.com/help/docs/send-data/hosted-collectors/http-source/logs-metrics/) documentation. + +```shell +pip install litellm +``` + +## Quick Start + +Use just 2 lines of code to instantly log your LLM responses to Sumo Logic. + +The Sumo Logic HTTP Source URL includes the authentication token, so no separate API key is required. + + + + +```python +litellm.callbacks = ["sumologic"] +``` + +```python +import litellm +import os + +# Sumo Logic HTTP Source URL (includes auth token) +os.environ["SUMOLOGIC_WEBHOOK_URL"] = "https://collectors.sumologic.com/receiver/v1/http/your-token-here" + +# LLM API Keys +os.environ['OPENAI_API_KEY'] = "" + +# Set sumologic as a callback +litellm.callbacks = ["sumologic"] + +# OpenAI call +response = litellm.completion( + model="gpt-3.5-turbo", + messages=[ + {"role": "user", "content": "Hi πŸ‘‹ - I'm testing Sumo Logic integration"} + ] +) +``` + + + + +1. Setup config.yaml + +```yaml +model_list: + - model_name: gpt-3.5-turbo + litellm_params: + model: openai/gpt-3.5-turbo + api_key: os.environ/OPENAI_API_KEY + +litellm_settings: + callbacks: ["sumologic"] + +environment_variables: + SUMOLOGIC_WEBHOOK_URL: os.environ/SUMOLOGIC_WEBHOOK_URL +``` + +2. Start LiteLLM Proxy + +```bash +litellm --config /path/to/config.yaml +``` + +3. Test it! + +```bash +curl -L -X POST 'http://0.0.0.0:4000/chat/completions' \ +-H 'Content-Type: application/json' \ +-H 'Authorization: Bearer sk-1234' \ +-d '{ + "model": "gpt-3.5-turbo", + "messages": [ + { + "role": "user", + "content": "Hey, how are you?" + } + ] +}' +``` + + + + +## What Data is Logged? + +LiteLLM sends the [Standard Logging Payload](https://docs.litellm.ai/docs/proxy/logging_spec) to Sumo Logic, which includes: + +- **Request details**: Model, messages, parameters +- **Response details**: Completion text, token usage, latency +- **Metadata**: User ID, custom metadata, timestamps +- **Cost tracking**: Response cost based on token usage + +Example payload: + +```json +{ + "id": "chatcmpl-123", + "call_type": "litellm.completion", + "model": "gpt-3.5-turbo", + "messages": [ + {"role": "user", "content": "Hello"} + ], + "response": { + "choices": [{ + "message": { + "role": "assistant", + "content": "Hi there!" + } + }] + }, + "usage": { + "prompt_tokens": 10, + "completion_tokens": 5, + "total_tokens": 15 + }, + "response_cost": 0.0001, + "start_time": "2024-01-01T00:00:00", + "end_time": "2024-01-01T00:00:01" +} +``` + +## Advanced Configuration + +### Batching Settings + +Control how LiteLLM batches logs before sending to Sumo Logic: + + + + +```python +import litellm + +os.environ["SUMOLOGIC_WEBHOOK_URL"] = "https://collectors.sumologic.com/receiver/v1/http/your-token" + +litellm.callbacks = ["sumologic"] + +# Configure batch settings (optional) +# These are inherited from CustomBatchLogger +# Default batch_size: 100 +# Default flush_interval: 60 seconds +``` + + + + +```yaml +litellm_settings: + callbacks: ["sumologic"] + +environment_variables: + SUMOLOGIC_WEBHOOK_URL: os.environ/SUMOLOGIC_WEBHOOK_URL +``` + + + + +### Compressed Data + +Sumo Logic supports compressed data (gzip or deflate). LiteLLM automatically handles compression when beneficial. + +Benefits: +- Reduced network usage +- Faster message delivery +- Lower data transfer costs + +### Query Logs in Sumo Logic + +Once logs are flowing to Sumo Logic, you can query them using the Sumo Logic Query Language: + +```sql +_sourceCategory=litellm +| json "model", "response_cost", "usage.total_tokens" as model, cost, tokens +| sum(cost) by model +``` + +Example queries: + +**Total cost by model:** +```sql +_sourceCategory=litellm +| json "model", "response_cost" as model, cost +| sum(cost) as total_cost by model +| sort by total_cost desc +``` + +**Average response time:** +```sql +_sourceCategory=litellm +| json "start_time", "end_time" as start, end +| parse regex field=start "(?\d+)" +| parse regex field=end "(?\d+)" +| (end_ms - start_ms) as response_time_ms +| avg(response_time_ms) as avg_response_time +``` + +**Requests per user:** +```sql +_sourceCategory=litellm +| json "model_parameters.user" as user +| count by user +``` + +## Authentication + +The Sumo Logic HTTP Source URL includes the authentication token, so you only need to set the `SUMOLOGIC_WEBHOOK_URL` environment variable. + +**Security Best Practices:** +- Keep your HTTP Source URL private (it contains the auth token) +- Store it in environment variables or secrets management +- Regenerate the URL if it's compromised (in Sumo Logic UI) +- Use separate HTTP Sources for different environments (dev, staging, prod) + +## Getting Your Sumo Logic URL + +1. Log in to [Sumo Logic](https://www.sumologic.com/) +2. Go to **Manage Data** > **Collection** > **Collection** +3. Click **Add Source** next to a Hosted Collector +4. Select **HTTP Logs & Metrics** +5. Configure the source: + - **Name**: LiteLLM Logs + - **Source Category**: litellm (optional, but helps with queries) +6. Click **Save** +7. Copy the displayed URL - it will look like: + ``` + https://collectors.sumologic.com/receiver/v1/http/ZaVnC4dhaV39Tn37... + ``` + +## Troubleshooting + +### Logs not appearing in Sumo Logic + +1. **Verify the URL**: Make sure `SUMOLOGIC_WEBHOOK_URL` is set correctly +2. **Check the HTTP Source**: Ensure it's active in Sumo Logic UI +3. **Wait for batching**: Logs are sent in batches, wait 60 seconds +4. **Check for errors**: Enable debug logging in LiteLLM: + ```python + litellm.set_verbose = True + ``` + +### URL Format + +The URL must be the complete HTTP Source URL from Sumo Logic: +- βœ… Correct: `https://collectors.sumologic.com/receiver/v1/http/ZaVnC4dhaV39Tn37...` + +### No authentication errors + +If you get authentication errors, regenerate the HTTP Source URL in Sumo Logic: +1. Go to your HTTP Source in Sumo Logic +2. Click the settings icon +3. Click **Show URL** +4. Click **Regenerate URL** +5. Update your `SUMOLOGIC_WEBHOOK_URL` environment variable + +## Support & Talk to Founders + +- [Schedule Demo πŸ‘‹](https://calendly.com/d/4mp-gd3-k5k/berriai-1-1-onboarding-litellm-hosted-version) +- [Community Discord πŸ’­](https://discord.gg/wuPM9dRgDw) +- Our numbers πŸ“ž +1 (770) 8783-106 / ‭+1 (412) 618-6238‬ +- Our emails βœ‰οΈ ishaan@berri.ai / krrish@berri.ai diff --git a/docs/my-website/docs/providers/bedrock_writer.md b/docs/my-website/docs/providers/bedrock_writer.md new file mode 100644 index 00000000000..00d77a37f44 --- /dev/null +++ b/docs/my-website/docs/providers/bedrock_writer.md @@ -0,0 +1,316 @@ +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +# Bedrock - Writer Palmyra + +## Overview + +| Property | Details | +|-------|-------| +| Description | Writer Palmyra X5 and X4 foundation models on Amazon Bedrock, offering advanced reasoning, tool calling, and document processing capabilities | +| Provider Route on LiteLLM | `bedrock/` | +| Supported Operations | `/chat/completions` | +| Link to Provider Doc | [Writer on AWS Bedrock β†—](https://aws.amazon.com/bedrock/writer/) | + +## Quick Start + +### LiteLLM SDK + +```python showLineNumbers title="SDK Usage" +import litellm +import os + +os.environ["AWS_ACCESS_KEY_ID"] = "" +os.environ["AWS_SECRET_ACCESS_KEY"] = "" +os.environ["AWS_REGION_NAME"] = "us-west-2" + +response = litellm.completion( + model="bedrock/us.writer.palmyra-x5-v1:0", + messages=[{"role": "user", "content": "Hello, how are you?"}] +) + +print(response.choices[0].message.content) +``` + +### LiteLLM Proxy + +**1. Setup config.yaml** + +```yaml showLineNumbers title="proxy_config.yaml" +model_list: + - model_name: writer-palmyra-x5 + litellm_params: + model: bedrock/us.writer.palmyra-x5-v1:0 + aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID + aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY + aws_region_name: us-west-2 +``` + +**2. Start the proxy** + +```bash showLineNumbers title="Start Proxy" +litellm --config config.yaml +``` + +**3. Call the proxy** + + + + +```bash showLineNumbers title="curl Request" +curl -X POST http://localhost:4000/v1/chat/completions \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer sk-1234" \ + -d '{ + "model": "writer-palmyra-x5", + "messages": [{"role": "user", "content": "Hello, how are you?"}] + }' +``` + + + + +```python showLineNumbers title="OpenAI SDK" +from openai import OpenAI + +client = OpenAI( + api_key="sk-1234", + base_url="http://localhost:4000/v1" +) + +response = client.chat.completions.create( + model="writer-palmyra-x5", + messages=[{"role": "user", "content": "Hello, how are you?"}] +) + +print(response.choices[0].message.content) +``` + + + + +## Tool Calling + +Writer Palmyra models support multi-step tool calling for complex workflows. + +### LiteLLM SDK + +```python showLineNumbers title="Tool Calling - SDK" +import litellm + +tools = [ + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get the current weather in a location", + "parameters": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "The city and state" + } + }, + "required": ["location"] + } + } + } +] + +response = litellm.completion( + model="bedrock/us.writer.palmyra-x5-v1:0", + messages=[{"role": "user", "content": "What's the weather in Boston?"}], + tools=tools +) +``` + +### LiteLLM Proxy + + + + +```bash showLineNumbers title="Tool Calling - curl" +curl -X POST http://localhost:4000/v1/chat/completions \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer sk-1234" \ + -d '{ + "model": "writer-palmyra-x5", + "messages": [{"role": "user", "content": "What'\''s the weather in Boston?"}], + "tools": [{ + "type": "function", + "function": { + "name": "get_weather", + "description": "Get the current weather in a location", + "parameters": { + "type": "object", + "properties": { + "location": {"type": "string", "description": "The city and state"} + }, + "required": ["location"] + } + } + }] + }' +``` + + + + +```python showLineNumbers title="Tool Calling - OpenAI SDK" +from openai import OpenAI + +client = OpenAI( + api_key="sk-1234", + base_url="http://localhost:4000/v1" +) + +tools = [ + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get the current weather in a location", + "parameters": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "The city and state" + } + }, + "required": ["location"] + } + } + } +] + +response = client.chat.completions.create( + model="writer-palmyra-x5", + messages=[{"role": "user", "content": "What's the weather in Boston?"}], + tools=tools +) +``` + + + + +## Document Input + +Writer Palmyra models support document inputs including PDFs. + +### LiteLLM SDK + +```python showLineNumbers title="PDF Document Input - SDK" +import litellm +import base64 + +# Read and encode PDF +with open("document.pdf", "rb") as f: + pdf_base64 = base64.b64encode(f.read()).decode("utf-8") + +response = litellm.completion( + model="bedrock/us.writer.palmyra-x5-v1:0", + messages=[ + { + "role": "user", + "content": [ + { + "type": "image_url", + "image_url": { + "url": f"data:application/pdf;base64,{pdf_base64}" + } + }, + { + "type": "text", + "text": "Summarize this document" + } + ] + } + ] +) +``` + +### LiteLLM Proxy + + + + +```bash showLineNumbers title="PDF Document Input - curl" +# First, base64 encode your PDF +PDF_BASE64=$(base64 -i document.pdf) + +curl -X POST http://localhost:4000/v1/chat/completions \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer sk-1234" \ + -d '{ + "model": "writer-palmyra-x5", + "messages": [{ + "role": "user", + "content": [ + { + "type": "image_url", + "image_url": {"url": "data:application/pdf;base64,'$PDF_BASE64'"} + }, + { + "type": "text", + "text": "Summarize this document" + } + ] + }] + }' +``` + + + + +```python showLineNumbers title="PDF Document Input - OpenAI SDK" +from openai import OpenAI +import base64 + +client = OpenAI( + api_key="sk-1234", + base_url="http://localhost:4000/v1" +) + +# Read and encode PDF +with open("document.pdf", "rb") as f: + pdf_base64 = base64.b64encode(f.read()).decode("utf-8") + +response = client.chat.completions.create( + model="writer-palmyra-x5", + messages=[ + { + "role": "user", + "content": [ + { + "type": "image_url", + "image_url": { + "url": f"data:application/pdf;base64,{pdf_base64}" + } + }, + { + "type": "text", + "text": "Summarize this document" + } + ] + } + ] +) +``` + + + + +## Supported Models + +| Model ID | Context Window | Input Cost (per 1K tokens) | Output Cost (per 1K tokens) | +|----------|---------------|---------------------------|----------------------------| +| `bedrock/us.writer.palmyra-x5-v1:0` | 1M tokens | $0.0006 | $0.006 | +| `bedrock/us.writer.palmyra-x4-v1:0` | 128K tokens | $0.0025 | $0.010 | +| `bedrock/writer.palmyra-x5-v1:0` | 1M tokens | $0.0006 | $0.006 | +| `bedrock/writer.palmyra-x4-v1:0` | 128K tokens | $0.0025 | $0.010 | + +:::info Cross-Region Inference +The `us.writer.*` model IDs use cross-region inference profiles. Use these for production workloads. +::: diff --git a/docs/my-website/docs/providers/helicone.md b/docs/my-website/docs/providers/helicone.md new file mode 100644 index 00000000000..3f0cfcbcb28 --- /dev/null +++ b/docs/my-website/docs/providers/helicone.md @@ -0,0 +1,268 @@ +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +# Helicone + +## Overview + +| Property | Details | +|-------|-------| +| Description | Helicone is an AI gateway and observability platform that provides OpenAI-compatible endpoints with advanced monitoring, caching, and analytics capabilities. | +| Provider Route on LiteLLM | `helicone/` | +| Link to Provider Doc | [Helicone Documentation β†—](https://docs.helicone.ai) | +| Base URL | `https://ai-gateway.helicone.ai/` | +| Supported Operations | [`/chat/completions`](#sample-usage), [`/completions`](#text-completion), [`/embeddings`](#embeddings) | + +
+ +**We support [ALL models available](https://helicone.ai/models) through Helicone's AI Gateway. Use `helicone/` as a prefix when sending requests.** + +## What is Helicone? + +Helicone is an open-source observability platform for LLM applications that provides: +- **Request Monitoring**: Track all LLM requests with detailed metrics +- **Caching**: Reduce costs and latency with intelligent caching +- **Rate Limiting**: Control request rates per user/key +- **Cost Tracking**: Monitor spend across models and users +- **Custom Properties**: Tag requests with metadata for filtering and analysis +- **Prompt Management**: Version control for prompts + +## Required Variables + +```python showLineNumbers title="Environment Variables" +os.environ["HELICONE_API_KEY"] = "" # your Helicone API key +``` + +Get your Helicone API key from your [Helicone dashboard](https://helicone.ai). + +## Usage - LiteLLM Python SDK + +### Non-streaming + +```python showLineNumbers title="Helicone Non-streaming Completion" +import os +import litellm +from litellm import completion + +os.environ["HELICONE_API_KEY"] = "" # your Helicone API key + +messages = [{"content": "What is the capital of France?", "role": "user"}] + +# Helicone call - routes through Helicone gateway to OpenAI +response = completion( + model="helicone/gpt-4", + messages=messages +) + +print(response) +``` + +### Streaming + +```python showLineNumbers title="Helicone Streaming Completion" +import os +import litellm +from litellm import completion + +os.environ["HELICONE_API_KEY"] = "" # your Helicone API key + +messages = [{"content": "Write a short poem about AI", "role": "user"}] + +# Helicone call with streaming +response = completion( + model="helicone/gpt-4", + messages=messages, + stream=True +) + +for chunk in response: + print(chunk) +``` + +### With Metadata (Helicone Custom Properties) + +```python showLineNumbers title="Helicone with Custom Properties" +import os +import litellm +from litellm import completion + +os.environ["HELICONE_API_KEY"] = "" # your Helicone API key + +response = completion( + model="helicone/gpt-4o-mini", + messages=[{"role": "user", "content": "What's the weather like?"}], + metadata={ + "Helicone-Property-Environment": "production", + "Helicone-Property-User-Id": "user_123", + "Helicone-Property-Session-Id": "session_abc" + } +) + +print(response) +``` + +### Text Completion + +```python showLineNumbers title="Helicone Text Completion" +import os +import litellm + +os.environ["HELICONE_API_KEY"] = "" # your Helicone API key + +response = litellm.completion( + model="helicone/gpt-4o-mini", # text completion model + prompt="Once upon a time" +) + +print(response) +``` + + +## Retry and Fallback Mechanisms + +```python +import litellm + +litellm.api_base = "https://ai-gateway.helicone.ai/" +litellm.metadata = { + "Helicone-Retry-Enabled": "true", + "helicone-retry-num": "3", + "helicone-retry-factor": "2", +} + +response = litellm.completion( + model="helicone/gpt-4o-mini/openai,claude-3-5-sonnet-20241022/anthropic", # Try OpenAI first, then fallback to Anthropic, then continue with other models, + messages=[{"role": "user", "content": "Hello"}] +) +``` + +## Supported OpenAI Parameters + +Helicone supports all standard OpenAI-compatible parameters: + +| Parameter | Type | Description | +|-----------|------|-------------| +| `messages` | array | **Required**. Array of message objects with 'role' and 'content' | +| `model` | string | **Required**. Model ID (e.g., gpt-4, claude-3-opus, etc.) | +| `stream` | boolean | Optional. Enable streaming responses | +| `temperature` | float | Optional. Sampling temperature | +| `top_p` | float | Optional. Nucleus sampling parameter | +| `max_tokens` | integer | Optional. Maximum tokens to generate | +| `frequency_penalty` | float | Optional. Penalize frequent tokens | +| `presence_penalty` | float | Optional. Penalize tokens based on presence | +| `stop` | string/array | Optional. Stop sequences | +| `n` | integer | Optional. Number of completions to generate | +| `tools` | array | Optional. List of available tools/functions | +| `tool_choice` | string/object | Optional. Control tool/function calling | +| `response_format` | object | Optional. Response format specification | +| `user` | string | Optional. User identifier | + +## Helicone-Specific Headers + +Pass these as metadata to leverage Helicone features: + +| Header | Description | +|--------|-------------| +| `Helicone-Property-*` | Custom properties for filtering (e.g., `Helicone-Property-User-Id`) | +| `Helicone-Cache-Enabled` | Enable caching for this request | +| `Helicone-User-Id` | User identifier for tracking | +| `Helicone-Session-Id` | Session identifier for grouping requests | +| `Helicone-Prompt-Id` | Prompt identifier for versioning | +| `Helicone-Rate-Limit-Policy` | Rate limiting policy name | + +Example with headers: + +```python showLineNumbers title="Helicone with Custom Headers" +import litellm + +response = litellm.completion( + model="helicone/gpt-4", + messages=[{"role": "user", "content": "Hello"}], + metadata={ + "Helicone-Cache-Enabled": "true", + "Helicone-Property-Environment": "production", + "Helicone-Property-User-Id": "user_123", + "Helicone-Session-Id": "session_abc", + "Helicone-Prompt-Id": "prompt_v1" + } +) +``` + +## Advanced Usage + +### Using with Different Providers + +Helicone acts as a gateway and supports multiple providers: + +```python showLineNumbers title="Helicone with Anthropic" +import litellm + +# Set both Helicone and Anthropic keys +os.environ["HELICONE_API_KEY"] = "your-helicone-key" + +response = litellm.completion( + model="helicone/claude-3.5-haiku/anthropic", + messages=[{"role": "user", "content": "Hello"}] +) +``` + +### Caching + +Enable caching to reduce costs and latency: + +```python showLineNumbers title="Helicone Caching" +import litellm + +response = litellm.completion( + model="helicone/gpt-4", + messages=[{"role": "user", "content": "What is 2+2?"}], + metadata={ + "Helicone-Cache-Enabled": "true" + } +) + +# Subsequent identical requests will be served from cache +response2 = litellm.completion( + model="helicone/gpt-4", + messages=[{"role": "user", "content": "What is 2+2?"}], + metadata={ + "Helicone-Cache-Enabled": "true" + } +) +``` + +## Features + +### Request Monitoring +- Track all requests with detailed metrics +- View request/response pairs +- Monitor latency and errors +- Filter by custom properties + +### Cost Tracking +- Per-model cost tracking +- Per-user cost tracking +- Cost alerts and budgets +- Historical cost analysis + +### Rate Limiting +- Per-user rate limits +- Per-API key rate limits +- Custom rate limit policies +- Automatic enforcement + +### Analytics +- Request volume trends +- Cost trends +- Latency percentiles +- Error rates + +Visit [Helicone Pricing](https://helicone.ai/pricing) for details. + +## Additional Resources + +- [Helicone Official Documentation](https://docs.helicone.ai) +- [Helicone Dashboard](https://helicone.ai) +- [Helicone GitHub](https://github.com/Helicone/helicone) +- [API Reference](https://docs.helicone.ai/rest/ai-gateway/post-v1-chat-completions) + diff --git a/docs/my-website/docs/providers/nvidia_nim_rerank.md b/docs/my-website/docs/providers/nvidia_nim_rerank.md index 7373014a960..d28f056c24b 100644 --- a/docs/my-website/docs/providers/nvidia_nim_rerank.md +++ b/docs/my-website/docs/providers/nvidia_nim_rerank.md @@ -141,6 +141,111 @@ curl -X POST http://0.0.0.0:4000/rerank \ }' ``` +## `/v1/ranking` Models (llama-3.2-nv-rerankqa-1b-v2) + +Some Nvidia NIM rerank models use the `/v1/ranking` endpoint instead of the default `/v1/retrieval/{model}/reranking` endpoint. + +Use the `ranking/` prefix to force requests to the `/v1/ranking` endpoint: + +### LiteLLM Python SDK + +```python showLineNumbers title="Force /v1/ranking endpoint with ranking/ prefix" +import litellm +import os + +os.environ['NVIDIA_NIM_API_KEY'] = "nvapi-..." + +# Use "ranking/" prefix to force /v1/ranking endpoint +response = litellm.rerank( + model="nvidia_nim/ranking/nvidia/llama-3.2-nv-rerankqa-1b-v2", + query="which way did the traveler go?", + documents=[ + "two roads diverged in a yellow wood...", + "then took the other, as just as fair...", + "i shall be telling this with a sigh somewhere ages and ages hence..." + ], + top_n=3, + truncate="END", # Optional: truncate long text from the end +) + +print(response) +``` + +### LiteLLM Proxy + +```yaml showLineNumbers title="config.yaml" +model_list: + - model_name: nvidia-ranking + litellm_params: + model: nvidia_nim/ranking/nvidia/llama-3.2-nv-rerankqa-1b-v2 + api_key: os.environ/NVIDIA_NIM_API_KEY +``` + +```bash title="Request to LiteLLM Proxy" +curl -X POST http://0.0.0.0:4000/rerank \ + -H "Authorization: Bearer sk-1234" \ + -H "Content-Type: application/json" \ + -d '{ + "model": "nvidia-ranking", + "query": "which way did the traveler go?", + "documents": [ + "two roads diverged in a yellow wood...", + "then took the other, as just as fair..." + ], + "top_n": 2 + }' +``` + +### Understanding Model Resolution + +**Ranking Endpoint (`/v1/ranking`):** + +``` +model: nvidia_nim/ranking/nvidia/llama-3.2-nv-rerankqa-1b-v2 + β””β”€β”€β”€β”€β”¬β”€β”€β”€β”€β”˜ β””β”€β”€β”¬β”€β”€β”˜ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”¬β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ + β”‚ β”‚ β”‚ + β”‚ β”‚ └────▢ Model name sent to provider + β”‚ β”‚ + β”‚ └────────────────────────▢ Tells LiteLLM the request/response and url should be sent to Nvidia NIM /v1/ranking endpoint + β”‚ + └─────────────────────────────────▢ Provider prefix + +API URL: https://ai.api.nvidia.com/v1/ranking +``` + +**Visual Flow:** + +``` +Client Request LiteLLM Provider API +────────────── ──────────── ───────────── + +# Default reranking endpoint +model: "nvidia_nim/nvidia/model-name" + 1. Extracts model: nvidia/model-name + 2. Routes to default endpoint ──────▢ POST /v1/retrieval/nvidia/model-name/reranking + + +# Forced ranking endpoint +model: "nvidia_nim/ranking/nvidia/model-name" + 1. Detects "ranking/" prefix + 2. Extracts model: nvidia/model-name + 3. Routes to ranking endpoint ──────▢ POST /v1/ranking + Body: {"model": "nvidia/model-name", ...} +``` + +**When to use each endpoint:** + +| Endpoint | Model Prefix | Use Case | +|----------|--------------|----------| +| `/v1/retrieval/{model}/reranking` | `nvidia_nim/` | Default for most rerank models | +| `/v1/ranking` | `nvidia_nim/ranking/` | For models like `nvidia/llama-3.2-nv-rerankqa-1b-v2` that require this endpoint | + +:::tip + +Check the [Nvidia NIM model deployment page](https://build.nvidia.com/nvidia/llama-3_2-nv-rerankqa-1b-v2/deploy) to see which endpoint your model requires. + +::: + ## API Parameters ### Required Parameters @@ -203,16 +308,7 @@ response = litellm.rerank( -## API Endpoint - -The rerank endpoint uses a different base URL than chat/embeddings: - -- **Chat/Embeddings:** `https://integrate.api.nvidia.com/v1/` -- **Rerank:** `https://ai.api.nvidia.com/v1/` - -LiteLLM automatically uses the correct endpoint for rerank requests. - -### Custom API Base URL +## Custom API Base URL You can override the default base URL in several ways: @@ -258,4 +354,3 @@ Get your Nvidia NIM API key from [Nvidia's website](https://developer.nvidia.com - [Nvidia NIM Chat Completions](./nvidia_nim#sample-usage) - [LiteLLM Rerank Endpoint](../rerank) - [Nvidia NIM Official Docs β†—](https://docs.api.nvidia.com/nim/reference/) - diff --git a/docs/my-website/docs/providers/sap.md b/docs/my-website/docs/providers/sap.md new file mode 100644 index 00000000000..a9183b9c0df --- /dev/null +++ b/docs/my-website/docs/providers/sap.md @@ -0,0 +1,121 @@ +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +# SAP Generative AI Hub + +LiteLLM supports SAP Generative AI Hub's Orchestration Service. + +| Property | Details | +|-------|-------| +| Description | SAP's Generative AI Hub provides access to foundation models through the AI Core orchestration service. | +| Provider Route on LiteLLM | `sap/` | +| Supported Endpoints | `/chat/completions` | +| API Reference | [SAP AI Core Documentation](https://help.sap.com/docs/sap-ai-core) | + +## Authentication + +SAP Generative AI Hub uses service key authentication. You can provide credentials via: + +1. **Environment variable** - Set `AICORE_SERVICE_KEY` with your service key JSON +2. **Direct parameter** - Pass `api_key` with the service key JSON string + +```python showLineNumbers title="Environment Variable" +import os +os.environ["AICORE_SERVICE_KEY"] = '{"clientid": "...", "clientsecret": "...", ...}' +``` + +## Usage - LiteLLM Python SDK + +```python showLineNumbers title="SAP Chat Completion" +from litellm import completion +import os + +os.environ["AICORE_SERVICE_KEY"] = '{"clientid": "...", "clientsecret": "...", ...}' + +response = completion( + model="sap/gpt-4", + messages=[{"role": "user", "content": "Hello from LiteLLM"}] +) +print(response) +``` + +```python showLineNumbers title="SAP Chat Completion - Streaming" +from litellm import completion +import os + +os.environ["AICORE_SERVICE_KEY"] = '{"clientid": "...", "clientsecret": "...", ...}' + +response = completion( + model="sap/gpt-4", + messages=[{"role": "user", "content": "Hello from LiteLLM"}], + stream=True +) + +for chunk in response: + print(chunk.choices[0].delta.content or "", end="") +``` + +## Usage - LiteLLM Proxy + +Add to your LiteLLM Proxy config: + +```yaml showLineNumbers title="config.yaml" +model_list: + - model_name: sap-gpt4 + litellm_params: + model: sap/gpt-4 + api_key: os.environ/AICORE_SERVICE_KEY +``` + +Start the proxy: + +```bash showLineNumbers title="Start Proxy" +litellm --config config.yaml +``` + + + + +```bash showLineNumbers title="Test Request" +curl http://localhost:4000/v1/chat/completions \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer your-proxy-api-key" \ + -d '{ + "model": "sap-gpt4", + "messages": [{"role": "user", "content": "Hello"}] + }' +``` + + + + +```python showLineNumbers title="OpenAI SDK" +from openai import OpenAI + +client = OpenAI( + base_url="http://localhost:4000", + api_key="your-proxy-api-key" +) + +response = client.chat.completions.create( + model="sap-gpt4", + messages=[{"role": "user", "content": "Hello"}] +) +print(response.choices[0].message.content) +``` + + + + +## Supported Parameters + +| Parameter | Description | +|-----------|-------------| +| `temperature` | Controls randomness | +| `max_tokens` | Maximum tokens in response | +| `top_p` | Nucleus sampling | +| `tools` | Function calling tools | +| `tool_choice` | Tool selection behavior | +| `response_format` | Output format (json_object, json_schema) | +| `stream` | Enable streaming | + diff --git a/docs/my-website/docs/proxy/dynamic_rate_limit.md b/docs/my-website/docs/proxy/dynamic_rate_limit.md index f5438b5a6f5..3c3500f8a6c 100644 --- a/docs/my-website/docs/proxy/dynamic_rate_limit.md +++ b/docs/my-website/docs/proxy/dynamic_rate_limit.md @@ -149,6 +149,7 @@ litellm_settings: priority_reservation_settings: default_priority: 0 # Weight (0%) assigned to keys without explicit priority metadata saturation_threshold: 0.50 # A model is saturated if it has hit 50% of its RPM limit + saturation_check_cache_ttl: 60 # How long (seconds) saturation values are cached locally general_settings: master_key: sk-1234 # OR set `LITELLM_MASTER_KEY=".."` in your .env @@ -168,6 +169,8 @@ general_settings: - **default_priority (float)**: Weight/percentage (0.0 to 1.0) assigned to API keys that have no priority metadata set (defaults to 0.5) - **saturation_threshold (float)**: Saturation level (0.0 to 1.0) at which strict priority enforcement begins for a model. Saturation is calculated as `max(current_rpm/max_rpm, current_tpm/max_tpm)`. Below this threshold, generous mode allows priority borrowing from unused capacity. Above this threshold, strict mode enforces normalized priority limits. - Example: When model usage is low, keys can use more than their allocated share. When model usage is high, keys are strictly limited to their allocated share. +- **saturation_check_cache_ttl (int)**: TTL in seconds for local cache when reading saturation values from Redis (defaults to 60). In multi-node deployments, this controls how quickly nodes converge on the same saturation state. Lower values mean faster convergence but more Redis reads. + - Example: Set to `5` for faster multi-node consistency, or `0` to always read directly from Redis. **Start Proxy** diff --git a/docs/my-website/docs/proxy/enterprise.md b/docs/my-website/docs/proxy/enterprise.md index cfd6ab31015..3c6d77cc7a2 100644 --- a/docs/my-website/docs/proxy/enterprise.md +++ b/docs/my-website/docs/proxy/enterprise.md @@ -15,8 +15,7 @@ Features: - βœ… [SSO for Admin UI](./ui.md#✨-enterprise-features) - βœ… [Audit Logs with retention policy](#audit-logs) - βœ… [JWT-Auth](./token_auth.md) - - βœ… [Control available public, private routes (Restrict certain endpoints on proxy)](#control-available-public-private-routes) - - βœ… [Control available public, private routes](#control-available-public-private-routes) + - βœ… [Control available public, private routes](./public_routes.md) - βœ… [Secret Managers - AWS Key Manager, Google Secret Manager, Azure Key, Hashicorp Vault](../secret) - βœ… [[BETA] AWS Key Manager v2 - Key Decryption](#beta-aws-key-manager---key-decryption) - βœ… IP address‑based access control lists @@ -181,148 +180,7 @@ Expected Response ### Control available public, private routes -**Restrict certain endpoints of proxy** - -:::info - -❓ Use this when you want to: -- make an existing private route -> public -- set certain routes as admin_only routes - -::: - -#### Usage - Define public, admin only routes - -**Step 1** - Set on config.yaml - - -| Route Type | Optional | Requires Virtual Key Auth | Admin Can Access | All Roles Can Access | Description | -|------------|----------|---------------------------|-------------------|----------------------|-------------| -| `public_routes` | βœ… | ❌ | βœ… | βœ… | Routes that can be accessed without any authentication | -| `admin_only_routes` | βœ… | βœ… | βœ… | ❌ | Routes that can only be accessed by [Proxy Admin](./self_serve#available-roles) | -| `allowed_routes` | βœ… | βœ… | βœ… | βœ… | Routes are exposed on the proxy. If not set then all routes exposed. | - -`LiteLLMRoutes.public_routes` is an ENUM corresponding to the default public routes on LiteLLM. [You can see this here](https://github.com/BerriAI/litellm/blob/main/litellm/proxy/_types.py) - -```yaml -general_settings: - master_key: sk-1234 - public_routes: ["LiteLLMRoutes.public_routes", "/spend/calculate"] # routes that can be accessed without any auth - admin_only_routes: ["/key/generate"] # Optional - routes that can only be accessed by Proxy Admin - allowed_routes: ["/chat/completions", "/spend/calculate", "LiteLLMRoutes.public_routes"] # Optional - routes that can be accessed by anyone after Authentication -``` - -**Step 2** - start proxy - -```shell -litellm --config config.yaml -``` - -**Step 3** - Test it - - - - - -```shell -curl --request POST \ - --url 'http://localhost:4000/spend/calculate' \ - --header 'Content-Type: application/json' \ - --data '{ - "model": "gpt-4", - "messages": [{"role": "user", "content": "Hey, how'\''s it going?"}] - }' -``` - -πŸŽ‰ Expect this endpoint to work without an `Authorization / Bearer Token` - - - - - - -**Successful Request** - -```shell -curl --location 'http://0.0.0.0:4000/key/generate' \ ---header 'Authorization: Bearer ' \ ---header 'Content-Type: application/json' \ ---data '{}' -``` - - -**Un-successfull Request** - -```shell - curl --location 'http://0.0.0.0:4000/key/generate' \ ---header 'Authorization: Bearer ' \ ---header 'Content-Type: application/json' \ ---data '{"user_role": "internal_user"}' -``` - -**Expected Response** - -```json -{ - "error": { - "message": "user not allowed to access this route. Route=/key/generate is an admin only route", - "type": "auth_error", - "param": "None", - "code": "403" - } -} -``` - - - - - - - -**Successful Request** - -```shell -curl http://localhost:4000/chat/completions \ --H "Content-Type: application/json" \ --H "Authorization: Bearer sk-1234" \ --d '{ -"model": "fake-openai-endpoint", -"messages": [ - {"role": "user", "content": "Hello, Claude"} -] -}' -``` - - -**Un-successfull Request** - -```shell -curl --location 'http://0.0.0.0:4000/embeddings' \ ---header 'Content-Type: application/json' \ --H "Authorization: Bearer sk-1234" \ ---data ' { -"model": "text-embedding-ada-002", -"input": ["write a litellm poem"] -}' -``` - -**Expected Response** - -```json -{ - "error": { - "message": "Route /embeddings not allowed", - "type": "auth_error", - "param": "None", - "code": "403" - } -} -``` - - - - - +See [Control Public & Private Routes](./public_routes.md) for detailed documentation on configuring public routes, admin-only routes, allowed routes, and wildcard patterns. ## Spend Tracking diff --git a/docs/my-website/docs/proxy/public_routes.md b/docs/my-website/docs/proxy/public_routes.md new file mode 100644 index 00000000000..21a92a00be5 --- /dev/null +++ b/docs/my-website/docs/proxy/public_routes.md @@ -0,0 +1,223 @@ +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +# Control Public & Private Routes + +:::info + +Requires a LiteLLM Enterprise License. [Get a free trial](https://calendly.com/d/4mp-gd3-k5k/litellm-1-1-onboarding-chat). + +::: + +Control which routes require authentication and which routes are publicly accessible. + +## Route Types + +| Route Type | Requires Auth | Description | +|------------|---------------|-------------| +| `public_routes` | No | Routes accessible without any authentication | +| `admin_only_routes` | Yes (Admin only) | Routes only accessible by [Proxy Admin](./self_serve#available-roles) | +| `allowed_routes` | Yes | Routes exposed on the proxy. If not set, all routes are exposed | + +## Quick Start + +### Make Routes Public + +Allow specific routes to be accessed without authentication: + +```yaml +general_settings: + master_key: sk-1234 + public_routes: ["LiteLLMRoutes.public_routes", "/spend/calculate"] +``` + +### Restrict Routes to Admin Only + +Restrict certain routes to only be accessible by Proxy Admin: + +```yaml +general_settings: + master_key: sk-1234 + admin_only_routes: ["/key/generate", "/key/delete"] +``` + +### Limit Available Routes + +Only expose specific routes on the proxy: + +```yaml +general_settings: + master_key: sk-1234 + allowed_routes: ["/chat/completions", "/embeddings", "LiteLLMRoutes.public_routes"] +``` + +## Usage Examples + +### Define Public, Admin Only, and Allowed Routes + +```yaml +general_settings: + master_key: sk-1234 + public_routes: ["LiteLLMRoutes.public_routes", "/spend/calculate"] + admin_only_routes: ["/key/generate"] + allowed_routes: ["/chat/completions", "/spend/calculate", "LiteLLMRoutes.public_routes"] +``` + +`LiteLLMRoutes.public_routes` is an ENUM corresponding to the default public routes on LiteLLM. [View the source](https://github.com/BerriAI/litellm/blob/main/litellm/proxy/_types.py). + +### Testing + + + + + +```shell +curl --request POST \ + --url 'http://localhost:4000/spend/calculate' \ + --header 'Content-Type: application/json' \ + --data '{ + "model": "gpt-4", + "messages": [{"role": "user", "content": "Hey, how'\''s it going?"}] + }' +``` + +This endpoint works without an `Authorization` header. + + + + + +**Successful Request (Admin)** + +```shell +curl --location 'http://0.0.0.0:4000/key/generate' \ +--header 'Authorization: Bearer ' \ +--header 'Content-Type: application/json' \ +--data '{}' +``` + +**Unsuccessful Request (Non-Admin)** + +```shell +curl --location 'http://0.0.0.0:4000/key/generate' \ +--header 'Authorization: Bearer ' \ +--header 'Content-Type: application/json' \ +--data '{"user_role": "internal_user"}' +``` + +**Expected Response** + +```json +{ + "error": { + "message": "user not allowed to access this route. Route=/key/generate is an admin only route", + "type": "auth_error", + "param": "None", + "code": "403" + } +} +``` + + + + + +**Successful Request** + +```shell +curl http://localhost:4000/chat/completions \ +-H "Content-Type: application/json" \ +-H "Authorization: Bearer sk-1234" \ +-d '{ +"model": "fake-openai-endpoint", +"messages": [ + {"role": "user", "content": "Hello, Claude"} +] +}' +``` + +**Unsuccessful Request (Route Not Allowed)** + +```shell +curl --location 'http://0.0.0.0:4000/embeddings' \ +--header 'Content-Type: application/json' \ +-H "Authorization: Bearer sk-1234" \ +--data '{ +"model": "text-embedding-ada-002", +"input": ["write a litellm poem"] +}' +``` + +**Expected Response** + +```json +{ + "error": { + "message": "Route /embeddings not allowed", + "type": "auth_error", + "param": "None", + "code": "403" + } +} +``` + + + + + +## Advanced: Wildcard Patterns + +Use wildcard patterns to match multiple routes at once. + +### Syntax + +| Pattern | Description | Example | +|---------|-------------|---------| +| `/path/*` | Matches any route starting with `/path/` | `/api/*` matches `/api/users`, `/api/users/123` | + + +### Examples + +#### Make All Routes Under a Path Public + +```yaml +general_settings: + master_key: sk-1234 + public_routes: + - "LiteLLMRoutes.public_routes" + - "/api/v1/*" # All routes under /api/v1/ + - "/health/*" # All health check routes +``` + +#### Restrict Admin Routes with Wildcards + +```yaml +general_settings: + master_key: sk-1234 + admin_only_routes: + - "/admin/*" # All admin routes + - "/internal/*" # All internal routes +``` + +### Testing Wildcard Routes + +**Config:** +```yaml +general_settings: + master_key: sk-1234 + public_routes: + - "/public/*" +``` + +**Test:** +```shell +# This works without auth (matches /public/*) +curl http://localhost:4000/public/status + +# This also works without auth (matches /public/*) +curl http://localhost:4000/public/health/detailed + +# This requires auth (doesn't match /public/*) +curl http://localhost:4000/private/data +``` + diff --git a/docs/my-website/docs/response_api.md b/docs/my-website/docs/response_api.md index 52e4c1e26f5..4e828c6c580 100644 --- a/docs/my-website/docs/response_api.md +++ b/docs/my-website/docs/response_api.md @@ -43,6 +43,38 @@ response = litellm.responses( print(response) ``` +#### Response Format (OpenAI Responses API Format) + +```json +{ + "id": "resp_abc123", + "object": "response", + "created_at": 1734366691, + "status": "completed", + "model": "o1-pro-2025-01-30", + "output": [ + { + "type": "message", + "id": "msg_abc123", + "status": "completed", + "role": "assistant", + "content": [ + { + "type": "output_text", + "text": "Once upon a time, a little unicorn named Stardust lived in a magical meadow where flowers sang lullabies. One night, she discovered that her horn could paint dreams across the sky, and she spent the evening creating the most beautiful aurora for all the forest creatures to enjoy. As the animals drifted off to sleep beneath her shimmering lights, Stardust curled up on a cloud of moonbeams, happy to have shared her magic with her friends.", + "annotations": [] + } + ] + } + ], + "usage": { + "input_tokens": 18, + "output_tokens": 98, + "total_tokens": 116 + } +} +``` + #### Streaming ```python showLineNumbers title="OpenAI Streaming Response" import litellm diff --git a/docs/my-website/sidebars.js b/docs/my-website/sidebars.js index 20b94963cf0..3efad0a9ab1 100644 --- a/docs/my-website/sidebars.js +++ b/docs/my-website/sidebars.js @@ -118,11 +118,83 @@ const sidebars = { ], // But you can create a sidebar manually tutorialSidebar: [ - { type: "doc", id: "index" }, // NEW + { type: "doc", id: "index", label: "Getting Started" }, { type: "category", - label: "LiteLLM AI Gateway", + label: "LiteLLM Python SDK", + items: [ + { + type: "link", + label: "Quick Start", + href: "/docs/#litellm-python-sdk", + }, + { + type: "category", + label: "SDK Functions", + items: [ + { + type: "doc", + id: "completion/input", + label: "completion()", + }, + { + type: "doc", + id: "embedding/supported_embedding", + label: "embedding()", + }, + { + type: "doc", + id: "response_api", + label: "responses()", + }, + { + type: "doc", + id: "text_completion", + label: "text_completion()", + }, + { + type: "doc", + id: "image_generation", + label: "image_generation()", + }, + { + type: "doc", + id: "audio_transcription", + label: "transcription()", + }, + { + type: "doc", + id: "text_to_speech", + label: "speech()", + }, + { + type: "link", + label: "All Supported Endpoints β†’", + href: "/docs/supported_endpoints", + }, + ], + }, + { + type: "category", + label: "Configuration", + items: [ + "set_keys", + "caching/all_caches", + ], + }, + "completion/token_usage", + "exception_mapping", + { + type: "category", + label: "LangChain, LlamaIndex, Instructor", + items: ["langchain/langchain", "tutorials/instructor"], + } + ], + }, + { + type: "category", + label: "LiteLLM AI Gateway (Proxy)", link: { type: "generated-index", title: "LiteLLM AI Gateway (LLM Proxy)", @@ -226,6 +298,7 @@ const sidebars = { "proxy/custom_auth", "proxy/ip_address", "proxy/multiple_admins", + "proxy/public_routes", ], }, { @@ -578,6 +651,7 @@ const sidebars = { "providers/bedrock_rerank", "providers/bedrock_agentcore", "providers/bedrock_agents", + "providers/bedrock_writer", "providers/bedrock_batches", "providers/bedrock_vector_store", ] @@ -614,6 +688,7 @@ const sidebars = { "providers/github_copilot", "providers/gradient_ai", "providers/groq", + "providers/helicone", "providers/heroku", { type: "category", @@ -667,6 +742,7 @@ const sidebars = { ] }, "providers/sambanova", + "providers/sap", "providers/snowflake", "providers/togetherai", "providers/topaz", @@ -694,6 +770,7 @@ const sidebars = { type: "category", label: "Guides", items: [ + "budget_manager", "completion/computer_use", "completion/web_search", "completion/web_fetch", @@ -746,27 +823,6 @@ const sidebars = { "wildcard_routing" ], }, - { - type: "category", - label: "LiteLLM Python SDK", - items: [ - "set_keys", - "budget_manager", - "caching/all_caches", - "completion/token_usage", - "sdk_custom_pricing", - "embedding/async_embedding", - "embedding/moderation", - "migration", - "sdk_custom_pricing", - { - type: "category", - label: "LangChain, LlamaIndex, Instructor Integration", - items: ["langchain/langchain", "tutorials/instructor"], - } - ], - }, - { type: "category", label: "Load Testing", @@ -836,6 +892,8 @@ const sidebars = { type: "category", label: "Extras", items: [ + "sdk_custom_pricing", + "migration", "data_security", "data_retention", "proxy/security_encryption_faq", @@ -850,7 +908,7 @@ const sidebars = { "Learn how to deploy + call models from different providers on LiteLLM", slug: "/project", }, - items: [ + items: [ "projects/smolagents", "projects/mini-swe-agent", "projects/openai-agents", diff --git a/litellm/__init__.py b/litellm/__init__.py index 5a441b99f6f..34bfc778982 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -265,6 +265,7 @@ heroku_key: Optional[str] = None cometapi_key: Optional[str] = None ovhcloud_key: Optional[str] = None lemonade_key: Optional[str] = None +sap_service_key: Optional[str] = None amazon_nova_api_key: Optional[str] = None common_cloud_provider_auth_params: dict = { "params": ["project", "region_name", "token"], @@ -1069,7 +1070,7 @@ from litellm.litellm_core_utils.core_helpers import remove_index_from_tool_calls from litellm.litellm_core_utils.token_counter import get_modified_max_tokens # client must be imported immediately as it's used as a decorator at function definition time from .utils import client -# Note: Most other utils imports are lazy-loaded via __getattr__ to avoid loading utils.py +# Note: Most other utils imports are lazy-loaded via __getattr__ to avoid loading utils.py # (which imports tiktoken) at import time from .llms.bytez.chat.transformation import BytezChatConfig @@ -1110,6 +1111,7 @@ from .llms.jina_ai.rerank.transformation import JinaAIRerankConfig from .llms.deepinfra.rerank.transformation import DeepinfraRerankConfig from .llms.hosted_vllm.rerank.transformation import HostedVLLMRerankConfig from .llms.nvidia_nim.rerank.transformation import NvidiaNimRerankConfig +from .llms.nvidia_nim.rerank.ranking_transformation import NvidiaNimRankingConfig from .llms.vertex_ai.rerank.transformation import VertexAIRerankConfig from .llms.fireworks_ai.rerank.transformation import FireworksAIRerankConfig from .llms.clarifai.chat.transformation import ClarifaiConfig @@ -1241,6 +1243,7 @@ from .llms.topaz.common_utils import TopazModelInfo from .llms.topaz.image_variations.transformation import TopazImageVariationConfig from litellm.llms.openai.completion.transformation import OpenAITextCompletionConfig from .llms.groq.chat.transformation import GroqChatConfig +from .llms.sap.chat.transformation import GenAIHubOrchestrationConfig from .llms.voyage.embedding.transformation import VoyageEmbeddingConfig from .llms.voyage.embedding.transformation_contextual import ( VoyageContextualEmbeddingConfig, @@ -1339,6 +1342,7 @@ from .llms.azure.chat.o_series_transformation import AzureOpenAIO1Config from .llms.watsonx.completion.transformation import IBMWatsonXAIConfig from .llms.watsonx.chat.transformation import IBMWatsonXChatConfig from .llms.watsonx.embed.transformation import IBMWatsonXEmbeddingConfig +from .llms.sap.embed.transformation import GenAIHubEmbeddingConfig from .llms.watsonx.audio_transcription.transformation import ( IBMWatsonXAudioTranscriptionConfig, ) @@ -1511,13 +1515,13 @@ def set_global_gitlab_config(config: Dict[str, Any]) -> None: if TYPE_CHECKING: from litellm.types.utils import ModelInfo as _ModelInfoType - + # Cost calculator functions cost_per_token: Callable[..., Tuple[float, float]] completion_cost: Callable[..., float] response_cost_calculator: Any modify_integration: Any - + # Utils functions - type stubs for truly lazy loaded functions only # (functions NOT imported via "from .main import *") get_response_string: Callable[..., str] @@ -1547,7 +1551,7 @@ if TYPE_CHECKING: get_first_chars_messages: Callable[..., str] get_provider_fields: Callable[..., List] get_valid_models: Callable[..., list] - + # Response types - truly lazy loaded only (not in main.py or elsewhere) ModelResponseListIterator: Type[Any] @@ -1563,7 +1567,7 @@ def __getattr__(name: str) -> Any: if name in _cost_calculator_names: from ._lazy_imports import _lazy_import_cost_calculator return _lazy_import_cost_calculator(name) - + # Lazy load litellm_logging functions _litellm_logging_names = ( "Logging", @@ -1572,7 +1576,7 @@ def __getattr__(name: str) -> Any: if name in _litellm_logging_names: from ._lazy_imports import _lazy_import_litellm_logging return _lazy_import_litellm_logging(name) - + # Lazy load utils functions _utils_names = ( "exception_type", "get_optional_params", "get_response_string", "token_counter", diff --git a/litellm/completion_extras/litellm_responses_transformation/transformation.py b/litellm/completion_extras/litellm_responses_transformation/transformation.py index 8ee730defb9..37170c6010d 100644 --- a/litellm/completion_extras/litellm_responses_transformation/transformation.py +++ b/litellm/completion_extras/litellm_responses_transformation/transformation.py @@ -873,8 +873,10 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator): usage=None, ) elif output_item.get("type") == "message": + # Don't emit is_finished=True here - there may be more output items + # (e.g., tool_calls) coming after the message. Wait for response.completed. return GenericStreamingChunk( - finish_reason="stop", is_finished=True, usage=None, text="" + finish_reason="", is_finished=False, usage=None, text="" ) elif event_type == "response.output_text.delta": @@ -907,6 +909,12 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator): ) ] ) + elif event_type == "response.completed": + # Response is fully complete - now we can signal is_finished=True + # This ensures we don't prematurely end the stream before tool_calls arrive + return GenericStreamingChunk( + text="", tool_use=None, is_finished=True, finish_reason="stop", usage=None + ) else: pass # For any unhandled event types, create a minimal valid chunk or skip diff --git a/litellm/constants.py b/litellm/constants.py index 1c0f800eccd..2c982ee41c4 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -345,6 +345,7 @@ LITELLM_CHAT_PROVIDERS = [ "huggingface", "together_ai", "datarobot", + "helicone", "openrouter", "cometapi", "vertex_ai", @@ -553,6 +554,7 @@ openai_compatible_endpoints: List = [ "https://api.morphllm.com/v1", "https://api.lambda.ai/v1", "https://api.hyperbolic.xyz/v1", + "https://ai-gateway.helicone.ai/", "https://ai-gateway.vercel.sh/v1", "https://api.inference.wandb.ai/v1", "https://api.clarifai.com/v2/ext/openai/v1", @@ -598,6 +600,7 @@ openai_compatible_providers: List = [ "moonshot", "publicai", "v0", + "helicone", "morph", "lambda_ai", "hyperbolic", @@ -935,6 +938,8 @@ BEDROCK_CONVERSE_MODELS = [ "amazon.nova-lite-v1:0", "amazon.nova-2-lite-v1:0", "amazon.nova-pro-v1:0", + "writer.palmyra-x4-v1:0", + "writer.palmyra-x5-v1:0", ] diff --git a/litellm/litellm_core_utils/exception_mapping_utils.py b/litellm/litellm_core_utils/exception_mapping_utils.py index 056522e2a64..97f62fed81a 100644 --- a/litellm/litellm_core_utils/exception_mapping_utils.py +++ b/litellm/litellm_core_utils/exception_mapping_utils.py @@ -82,6 +82,14 @@ class ExceptionCheckers: for substring in known_exception_substrings: if substring in _error_str_lowercase: return True + + # Cerebras pattern: "Current length is X while limit is Y" + if ( + "current length is" in _error_str_lowercase + and "while limit is" in _error_str_lowercase + ): + return True + return False @staticmethod diff --git a/litellm/litellm_core_utils/get_llm_provider_logic.py b/litellm/litellm_core_utils/get_llm_provider_logic.py index 168c837f1e5..677dac3d313 100644 --- a/litellm/litellm_core_utils/get_llm_provider_logic.py +++ b/litellm/litellm_core_utils/get_llm_provider_logic.py @@ -406,6 +406,8 @@ def get_llm_provider( # noqa: PLR0915 custom_llm_provider = "clarifai" elif model.startswith("amazon_nova"): custom_llm_provider = "amazon_nova" + elif model.startswith("sap/"): + custom_llm_provider = "sap" if not custom_llm_provider: if litellm.suppress_debug_info is False: print() # noqa diff --git a/litellm/litellm_core_utils/get_supported_openai_params.py b/litellm/litellm_core_utils/get_supported_openai_params.py index 19b52d2dace..4b40f44cbc4 100644 --- a/litellm/litellm_core_utils/get_supported_openai_params.py +++ b/litellm/litellm_core_utils/get_supported_openai_params.py @@ -116,6 +116,11 @@ def get_supported_openai_params( # noqa: PLR0915 f"Unsupported provider config: {transcription_provider_config} for model: {model}" ) return litellm.OpenAIConfig().get_supported_openai_params(model=model) + elif custom_llm_provider == "sap": + if request_type == "chat_completion": + return litellm.GenAIHubOrchestrationConfig().get_supported_openai_params(model=model) + elif request_type == "embeddings": + return litellm.GenAIHubEmbeddingConfig().get_supported_openai_params(model=model) elif custom_llm_provider == "azure": if litellm.AzureOpenAIO1Config().is_o_series_model(model=model): return litellm.AzureOpenAIO1Config().get_supported_openai_params( diff --git a/litellm/litellm_core_utils/streaming_handler.py b/litellm/litellm_core_utils/streaming_handler.py index 4ffb7ace5b0..d92af417175 100644 --- a/litellm/litellm_core_utils/streaming_handler.py +++ b/litellm/litellm_core_utils/streaming_handler.py @@ -441,7 +441,6 @@ class CustomStreamWrapper: finish_reason = None logprobs = None usage = None - if str_line and str_line.choices and len(str_line.choices) > 0: if ( str_line.choices[0].delta is not None diff --git a/litellm/llms/azure_ai/anthropic/messages_transformation.py b/litellm/llms/azure_ai/anthropic/messages_transformation.py index 55818cc07d6..73dc84167ab 100644 --- a/litellm/llms/azure_ai/anthropic/messages_transformation.py +++ b/litellm/llms/azure_ai/anthropic/messages_transformation.py @@ -48,12 +48,7 @@ class AzureAnthropicMessagesConfig(AnthropicMessagesConfig): headers = BaseAzureLLM._base_validate_azure_environment( headers=headers, litellm_params=litellm_params_obj ) - - # Azure Anthropic uses x-api-key header (not api-key) - # Convert api-key to x-api-key if present - if "api-key" in headers and "x-api-key" not in headers: - headers["x-api-key"] = headers.pop("api-key") - + # Set anthropic-version header if "anthropic-version" not in headers: headers["anthropic-version"] = "2023-06-01" diff --git a/litellm/llms/azure_ai/anthropic/transformation.py b/litellm/llms/azure_ai/anthropic/transformation.py index 150ad0a48b2..ebefbd3bf7f 100644 --- a/litellm/llms/azure_ai/anthropic/transformation.py +++ b/litellm/llms/azure_ai/anthropic/transformation.py @@ -55,11 +55,6 @@ class AzureAnthropicConfig(AnthropicConfig): headers = BaseAzureLLM._base_validate_azure_environment( headers=headers, litellm_params=litellm_params_obj ) - - # Azure Anthropic uses x-api-key header (not api-key) - # Convert api-key to x-api-key if present - if "api-key" in headers and "x-api-key" not in headers: - headers["x-api-key"] = headers.pop("api-key") # Get tools and other anthropic-specific setup tools = optional_params.get("tools") diff --git a/litellm/llms/dashscope/chat/transformation.py b/litellm/llms/dashscope/chat/transformation.py index 0edcc2a0c34..155d8c9ec27 100644 --- a/litellm/llms/dashscope/chat/transformation.py +++ b/litellm/llms/dashscope/chat/transformation.py @@ -51,7 +51,7 @@ class DashScopeChatConfig(OpenAIGPTConfig): api_base = ( api_base or get_secret_str("DASHSCOPE_API_BASE") - or "https://dashscope-intl.aliyuncs.com/compatible-mode/v1" + or "https://dashscope.aliyuncs.com/compatible-mode/v1" ) # type: ignore dynamic_api_key = api_key or get_secret_str("DASHSCOPE_API_KEY") return api_base, dynamic_api_key diff --git a/litellm/llms/nvidia_nim/rerank/common_utils.py b/litellm/llms/nvidia_nim/rerank/common_utils.py new file mode 100644 index 00000000000..2bd8c123c90 --- /dev/null +++ b/litellm/llms/nvidia_nim/rerank/common_utils.py @@ -0,0 +1,28 @@ +""" +Common utilities for NVIDIA NIM rerank provider. +""" + + +def get_nvidia_nim_rerank_config(model: str): + """ + Get the appropriate NVIDIA NIM rerank config based on the model. + + Args: + model: The model string (e.g., "nvidia/llama-3.2-nv-rerankqa-1b-v2" or "ranking/nvidia/llama-3.2-nv-rerankqa-1b-v2") + + Returns: + NvidiaNimRankingConfig if model starts with "ranking/", else NvidiaNimRerankConfig + + Example: + - "ranking/nvidia/llama-3.2-nv-rerankqa-1b-v2" -> NvidiaNimRankingConfig + - "nvidia/llama-3.2-nv-rerankqa-1b-v2" -> NvidiaNimRerankConfig + """ + from litellm.llms.nvidia_nim.rerank.ranking_transformation import ( + NvidiaNimRankingConfig, + ) + from litellm.llms.nvidia_nim.rerank.transformation import NvidiaNimRerankConfig + + if model.startswith("ranking/"): + return NvidiaNimRankingConfig() + return NvidiaNimRerankConfig() + diff --git a/litellm/llms/nvidia_nim/rerank/ranking_transformation.py b/litellm/llms/nvidia_nim/rerank/ranking_transformation.py new file mode 100644 index 00000000000..72e3c039d4c --- /dev/null +++ b/litellm/llms/nvidia_nim/rerank/ranking_transformation.py @@ -0,0 +1,75 @@ +""" +Transformation for NVIDIA NIM Ranking models that use /v1/ranking endpoint. + +Use this by passing "nvidia_nim/ranking/" to force the /v1/ranking endpoint. + +Reference: https://build.nvidia.com/nvidia/llama-3_2-nv-rerankqa-1b-v2/deploy +""" + +from typing import Dict, Optional + +from litellm.llms.nvidia_nim.rerank.transformation import NvidiaNimRerankConfig + + +class NvidiaNimRankingConfig(NvidiaNimRerankConfig): + """ + Configuration for NVIDIA NIM models that use the /v1/ranking endpoint. + + Example: + curl -X "POST" 'https://ai.api.nvidia.com/v1/ranking' \ + -H 'Accept: application/json' \ + -H 'Content-Type: application/json' \ + -d '{ + "model": "nvidia/llama-3.2-nv-rerankqa-1b-v2", + "query": {"text": "which way did the traveler go?"}, + "passages": [{"text": "..."}, {"text": "..."}], + "truncate": "END" + }' + """ + + def _get_clean_model_name(self, model: str) -> str: + """Strip 'ranking/' prefix from model name.""" + if model.startswith("ranking/"): + return model[len("ranking/"):] + return model + + def get_complete_url( + self, + api_base: Optional[str], + model: str, + optional_params: Optional[dict] = None, + ) -> str: + """ + Construct the Nvidia NIM ranking URL. + + Format: {api_base}/v1/ranking + """ + if not api_base: + api_base = self.DEFAULT_NIM_RERANK_API_BASE + + api_base = api_base.rstrip("/") + + if api_base.endswith("/ranking"): + return api_base + + if api_base.endswith("/v1"): + api_base = api_base[:-3] + + return f"{api_base}/v1/ranking" + + def transform_rerank_request( + self, + model: str, + optional_rerank_params: Dict, + headers: dict, + ) -> dict: + """ + Transform request, using clean model name without 'ranking/' prefix. + """ + clean_model = self._get_clean_model_name(model) + return super().transform_rerank_request( + model=clean_model, + optional_rerank_params=optional_rerank_params, + headers=headers, + ) + diff --git a/litellm/llms/openai/chat/gpt_transformation.py b/litellm/llms/openai/chat/gpt_transformation.py index 4e553a3da5c..034ccae94ad 100644 --- a/litellm/llms/openai/chat/gpt_transformation.py +++ b/litellm/llms/openai/chat/gpt_transformation.py @@ -168,9 +168,11 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig): ): # gpt-4 does not support 'response_format' model_specific_params.append("response_format") + # Normalize model name for responses API (e.g., "responses/gpt-4.1" -> "gpt-4.1") + model_for_check = model.split("responses/", 1)[1] if "responses/" in model else model if ( - model in litellm.open_ai_chat_completion_models - ) or model in litellm.open_ai_text_completion_models: + model_for_check in litellm.open_ai_chat_completion_models + ) or model_for_check in litellm.open_ai_text_completion_models: model_specific_params.append( "user" ) # user is not a param supported by all openai-compatible endpoints - e.g. azure ai diff --git a/litellm/llms/openai_like/providers.json b/litellm/llms/openai_like/providers.json index 3fb20b2dfc1..a6c19222619 100644 --- a/litellm/llms/openai_like/providers.json +++ b/litellm/llms/openai_like/providers.json @@ -10,5 +10,9 @@ "special_handling": { "convert_content_list_to_string": true } + }, + "helicone": { + "base_url": "https://ai-gateway.helicone.ai/", + "api_key_env": "HELICONE_API_KEY" } } diff --git a/litellm/llms/sap/chat/__init__.py b/litellm/llms/sap/chat/__init__.py new file mode 100755 index 00000000000..8b137891791 --- /dev/null +++ b/litellm/llms/sap/chat/__init__.py @@ -0,0 +1 @@ + diff --git a/litellm/llms/sap/chat/handler.py b/litellm/llms/sap/chat/handler.py new file mode 100755 index 00000000000..beabe255130 --- /dev/null +++ b/litellm/llms/sap/chat/handler.py @@ -0,0 +1,262 @@ +from __future__ import annotations + +import json +import time +import httpx + +from typing import Iterator, Optional, AsyncIterator + +from litellm.llms.base_llm.chat.transformation import BaseConfig +from litellm.types.llms.openai import OpenAIChatCompletionChunk +from ...custom_httpx.llm_http_handler import BaseLLMHTTPHandler + + +# ------------------------------- +# Errors +# ------------------------------- +class GenAIHubOrchestrationError(Exception): + def __init__(self, status_code: int, message: str): + super().__init__(message) + self.status_code = status_code + self.message = message + + +# ------------------------------- +# Stream parsing helpers +# ------------------------------- + + +def _now_ts() -> int: + return int(time.time()) + + +def _is_terminal_chunk(chunk: OpenAIChatCompletionChunk) -> bool: + """OpenAI-shaped chunk is terminal if any choice has a non-None finish_reason.""" + try: + for ch in chunk.choices or []: + if ch.finish_reason is not None: + return True + except Exception: + pass + return False + + +class _StreamParser: + """Normalize orchestration streaming events into OpenAI-like chunks.""" + + @staticmethod + def _from_orchestration_result(evt: dict) -> Optional[OpenAIChatCompletionChunk]: + """ + Accepts orchestration_result shape and maps it to an OpenAI-like *chunk*. + """ + orc = evt.get("orchestration_result") or {} + if not orc: + return None + + return OpenAIChatCompletionChunk.model_validate( + { + "id": orc.get("id") or evt.get("request_id") or "stream-chunk", + "object": orc.get("object") or "chat.completion.chunk", + "created": orc.get("created") or evt.get("created") or _now_ts(), + "model": orc.get("model") or "unknown", + "choices": [ + { + "index": c.get("index", 0), + "delta": c.get("delta") or {}, + "finish_reason": c.get("finish_reason"), + } + for c in (orc.get("choices") or []) + ], + } + ) + + @staticmethod + def to_openai_chunk(event_obj: dict) -> Optional[OpenAIChatCompletionChunk]: + """ + Accepts: + - {"final_result": } (IMPORTANT: this is just another chunk, NOT terminal) + - {"orchestration_result": {...}} (map to chunk) + - already-openai-shaped chunks + - other events (ignored) + Raises: + - ValueError for in-stream error objects + """ + # In-stream error per spec (surface as exception) + if "code" in event_obj or "error" in event_obj: + raise ValueError(json.dumps(event_obj)) + + # FINAL RESULT IS *NOT* TERMINAL: treat it as the next chunk + if "final_result" in event_obj: + fr = event_obj["final_result"] or {} + # ensure it looks like an OpenAI chunk + if "object" not in fr: + fr["object"] = "chat.completion.chunk" + return OpenAIChatCompletionChunk.model_validate(fr) + + # Orchestration incremental delta + if "orchestration_result" in event_obj: + return _StreamParser._from_orchestration_result(event_obj) + + # Already an OpenAI-like chunk + if "choices" in event_obj and "object" in event_obj: + return OpenAIChatCompletionChunk.model_validate(event_obj) + + # Unknown / heartbeat / metrics + return None + + +# ------------------------------- +# Iterators +# ------------------------------- +class SAPStreamIterator: + """ + Sync iterator over an httpx streaming response that yields OpenAIChatCompletionChunk. + Accepts both SSE `data: ...` and raw JSON lines. Closes on terminal chunk or [DONE]. + """ + + def __init__( + self, + response: Iterator, + event_prefix: str = "data: ", + final_msg: str = "[DONE]", + ): + self._resp = response + self._iter = response + self._prefix = event_prefix + self._final = final_msg + self._done = False + + def __iter__(self) -> Iterator[OpenAIChatCompletionChunk]: + return self + + def __next__(self) -> OpenAIChatCompletionChunk: + if self._done: + raise StopIteration + + for raw in self._iter: + line = (raw or "").strip() + if not line: + continue + + payload = ( + line[len(self._prefix) :] if line.startswith(self._prefix) else line + ) + if payload == self._final: + self._safe_close() + raise StopIteration + + try: + obj = json.loads(payload) + except Exception: + continue + + try: + chunk = _StreamParser.to_openai_chunk(obj) + except ValueError as e: + self._safe_close() + raise e + + if chunk is None: + continue + + # Close on terminal + if _is_terminal_chunk(chunk): + self._safe_close() + + return chunk + + self._safe_close() + raise StopIteration + + def _safe_close(self) -> None: + if self._done: + return + else: + self._done = True + + +class AsyncSAPStreamIterator: + sync_stream = False + + def __init__( + self, + response:AsyncIterator, + event_prefix: str = "data: ", + final_msg: str = "[DONE]", + ): + self._resp = response + self._prefix = event_prefix + self._final = final_msg + self._line_iter = None + self._done = False + + def __aiter__(self): + return self + + async def __anext__(self): + if self._done: + raise StopAsyncIteration + + if self._line_iter is None: + self._line_iter = self._resp + + while True: + try: + raw = await self._line_iter.__anext__() + except (StopAsyncIteration, httpx.ReadError, OSError): + await self._aclose() + raise StopAsyncIteration + + line = (raw or "").strip() + if not line: + continue + + # now = lambda: int(time.time() * 1000) + payload = ( + line[len(self._prefix) :] if line.startswith(self._prefix) else line + ) + if payload == self._final: + await self._aclose() + raise StopAsyncIteration + try: + obj = json.loads(payload) + except Exception: + continue + + try: + chunk = _StreamParser.to_openai_chunk(obj) + except ValueError as e: + await self._aclose() + raise GenAIHubOrchestrationError(502, str(e)) + + if chunk is None: + continue + + # If terminal, close BEFORE returning. Next __anext__() will stop immediately. + if any(c.finish_reason is not None for c in (chunk.choices or [])): + await self._aclose() + + return chunk + + async def _aclose(self): + if self._done: + return + else: + self._done = True + + +# ------------------------------- +# LLM handler +# ------------------------------- +class GenAIHubOrchestration(BaseLLMHTTPHandler): + def _add_stream_param_to_request_body( + self, + data: dict, + provider_config: BaseConfig, + fake_stream: bool + ): + if data.get("config", {}).get("stream", None) is not None: + data["config"]["stream"]["enabled"] = True + else: + data["config"]["stream"] = {"enabled": True} + return data diff --git a/litellm/llms/sap/chat/models.py b/litellm/llms/sap/chat/models.py new file mode 100644 index 00000000000..d8039ff5618 --- /dev/null +++ b/litellm/llms/sap/chat/models.py @@ -0,0 +1,112 @@ +from typing import Union, Literal + +from pydantic import BaseModel, Field, field_validator + + +def validate_different_content(v: Union[str, dict, list]) -> str: + if v in ((), {}, []): + return "" + elif isinstance(v, dict) and "text" in v: + return v['text'] + elif isinstance(v, list): + new_v = [] + for item in v: + if isinstance(item, dict) and "text" in item: + if item['text']: + new_v.append(item['text']) + elif isinstance(item, str): + new_v.append(item) + return '\n'.join(new_v) + elif isinstance(v, str): + return v + raise ValueError("Content must be a string") + return v + +class TextContent(BaseModel): + type_: Literal["text"] = Field(default="text", alias="type") + text: str + + +class ImageURLContent(BaseModel): + url: str + detail: str = "auto" + + +class ImageContent(BaseModel): + type_: Literal["image_url"] = Field(default="image_url", alias="type") + image_url: ImageURLContent + + +class FunctionObj(BaseModel): + name: str + arguments: str + + +class FunctionTool(BaseModel): + description: str = "" + name: str + parameters: dict = {} + strict: bool = False + + +class ChatCompletionTool(BaseModel): + type_: Literal["function"] = Field(default="function", alias="type") + function: FunctionTool + + +class MessageToolCall(BaseModel): + id: str + type_: Literal["function"] = Field(default="function", alias="type") + function: FunctionObj + + +class SAPMessage(BaseModel): + """ + Model for SystemChatMessage and DeveloperChatMessage + """ + + role: Literal["system", "developer"] = "system" + content: str + + _content_validator = field_validator("content", mode="before")(validate_different_content) + + +class SAPUserMessage(BaseModel): + role: Literal["user"] = "user" + content: Union[ + str, TextContent, ImageContent, list[Union[TextContent, ImageContent]] + ] + + +class SAPAssistantMessage(BaseModel): + role: Literal["assistant"] = "assistant" + content: str = "" + refusal: str = "" + tool_calls: list[MessageToolCall] = [] + + _content_validator = field_validator("content", mode="before")(validate_different_content) + + + +class SAPToolChatMessage(BaseModel): + role: Literal["tool"] = "tool" + tool_call_id: str + content: str + + _content_validator = field_validator("content", mode="before")(validate_different_content) + + +class ResponseFormat(BaseModel): + type_: Literal["text", "json_object"] = Field(default="text", alias="type") + + +class JSONResponseSchema(BaseModel): + description: str = "" + name: str + schema_: dict = Field(default_factory=dict, alias="schema") + strict: bool = False + + +class ResponseFormatJSONSchema(BaseModel): + type_: Literal["json_schema"] = Field(default="json_schema", alias="type") + json_schema: JSONResponseSchema diff --git a/litellm/llms/sap/chat/transformation.py b/litellm/llms/sap/chat/transformation.py new file mode 100755 index 00000000000..01ceb72c0de --- /dev/null +++ b/litellm/llms/sap/chat/transformation.py @@ -0,0 +1,299 @@ +""" +Translate from OpenAI's `/v1/chat/completions` to SAP Generative AI Hub's Orchestration Service`v2/completion` +""" +from typing import List, Optional, Union, Dict, Tuple, Any, TYPE_CHECKING, Iterator, AsyncIterator +from functools import cached_property +import litellm +import httpx + + +from litellm.types.llms.openai import AllMessageValues +from litellm.types.utils import ModelResponse + +from ...openai.chat.gpt_transformation import OpenAIGPTConfig + +if TYPE_CHECKING: + from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + + LiteLLMLoggingObj = _LiteLLMLoggingObj +else: + LiteLLMLoggingObj = Any + +from ..credentials import get_token_creator +from .models import ( + SAPMessage, + SAPAssistantMessage, + SAPToolChatMessage, + ChatCompletionTool, + ResponseFormatJSONSchema, + ResponseFormat, + SAPUserMessage, +) +from .handler import GenAIHubOrchestrationError, AsyncSAPStreamIterator, SAPStreamIterator + +def validate_dict(data: dict, model) -> dict: + return model(**data).model_dump(by_alias=True) + + +class GenAIHubOrchestrationConfig(OpenAIGPTConfig): + frequency_penalty: Optional[int] = None + function_call: Optional[Union[str, dict]] = None + functions: Optional[list] = None + logit_bias: Optional[dict] = None + max_tokens: Optional[int] = None + n: Optional[int] = None + presence_penalty: Optional[int] = None + stop: Optional[Union[str, list]] = None + temperature: Optional[int] = None + top_p: Optional[int] = None + response_format: Optional[dict] = None + tools: Optional[list] = None + tool_choice: Optional[Union[str, dict]] = None # + model_version: str = "latest" + + def __init__( + self, + frequency_penalty: Optional[int] = None, + function_call: Optional[Union[str, dict]] = None, + functions: Optional[list] = None, + logit_bias: Optional[dict] = None, + max_tokens: Optional[int] = None, + n: Optional[int] = None, + presence_penalty: Optional[int] = None, + stop: Optional[Union[str, list]] = None, + temperature: Optional[int] = None, + top_p: Optional[int] = None, + response_format: Optional[dict] = None, + tools: Optional[list] = None, + tool_choice: Optional[Union[str, dict]] = None, + ) -> None: + locals_ = locals().copy() + for key, value in locals_.items(): + if key != "self" and value is not None: + setattr(self.__class__, key, value) + self.token_creator = None + self._base_url = None + self._resource_group = None + + def run_env_setup(self, service_key: Optional[str] = None) -> None: + try: + self.token_creator, self._base_url, self._resource_group = get_token_creator(service_key) # type: ignore + except ValueError as err: + raise GenAIHubOrchestrationError(status_code=400, message=err.args[0]) + + + @property + def headers(self) -> Dict[str, str]: + if self.token_creator is None: + self.run_env_setup() + access_token = self.token_creator() # type: ignore + return { + "Authorization": access_token, + "AI-Resource-Group": self.resource_group, + "Content-Type": "application/json", + } + + @property + def base_url(self) -> str: + if self._base_url is None: + self.run_env_setup() + return self._base_url # type: ignore + + + @property + def resource_group(self) -> str: + if self._resource_group is None: + self.run_env_setup() + return self._resource_group # type: ignore + + @cached_property + def deployment_url(self) -> str: + # Keep a short, tight client lifecycle here to avoid fd leaks + client = litellm.module_level_client + # with httpx.Client(timeout=30) as client: + deployments = client.get( + f"{self.base_url}/lm/deployments", headers=self.headers + ).json() + valid: List[Tuple[str, str]] = [] + for dep in deployments.get("resources", []): + if dep.get("scenarioId") == "orchestration": + cfg = client.get( + f'{self.base_url}/lm/configurations/{dep["configurationId"]}', + headers=self.headers, + ).json() + if cfg.get("executableId") == "orchestration": + valid.append((dep["deploymentUrl"], dep["createdAt"])) + # newest first + return sorted(valid, key=lambda x: x[1], reverse=True)[0][0] + + @classmethod + def get_config(cls): + return super().get_config() + + def get_supported_openai_params(self, model): + params = [ + "frequency_penalty", + "logit_bias", + "logprobs", + "top_logprobs", + "max_tokens", + "max_completion_tokens", + "prediction", + "n", + "presence_penalty", + "seed", + "stop", + "stream", + "stream_options", + "temperature", + "top_p", + "tools", + "tool_choice", + "function_call", + "functions", + "extra_headers", + "parallel_tool_calls", + "response_format", + "timeout", + ] + if ( + model.startswith('anthropic') + or model.startswith("amazon") + or model.startswith("cohere") + or model.startswith("alephalpha") + or model == "gpt-4" + ): + params.remove("response_format") + if model.startswith("gemini") or model.startswith("amazon"): + params.remove("tool_choice") + return params + + def validate_environment( + self, + headers: dict, + model: str, + messages: List[AllMessageValues], + optional_params: dict, + litellm_params: dict, + api_key: Optional[str] = None, + api_base: Optional[str] = None, + ) -> dict: + if api_key: + self.run_env_setup(api_key) + return self.headers + + def get_complete_url( + self, + api_base: Optional[str], + api_key: Optional[str], + model: str, + optional_params: dict, + litellm_params: dict, + stream: Optional[bool] = None, + ): + api_base_ = f"{self.deployment_url}/v2/completion" + return api_base_ + + def transform_request( + self, + model: str, + messages: List[Dict[str, str]], # type: ignore + optional_params: dict, + litellm_params: dict, + headers: dict, + ) -> dict: + supported_params = self.get_supported_openai_params(model) + model_params = { + k: v for k, v in optional_params.items() if k in supported_params + } + model_version = optional_params.pop("model_version", "latest") + template = [] + for message in messages: + if message["role"] == "user": + template.append(validate_dict(message, SAPUserMessage)) + elif message["role"] == "assistant": + template.append(validate_dict(message, SAPAssistantMessage)) + elif message["role"] == "tool": + template.append(validate_dict(message, SAPToolChatMessage)) + else: + template.append(validate_dict(message, SAPMessage)) + + tools_ = optional_params.pop("tools", []) + tools_ = [validate_dict(tool, ChatCompletionTool) for tool in tools_] + if tools_ != []: + tools = {"tools": tools_} + else: + tools = {} + + response_format = model_params.pop("response_format", {}) + resp_type = response_format.get("type", None) + if resp_type: + if resp_type== "json_schema": + response_format = validate_dict(response_format, ResponseFormatJSONSchema) + else: + response_format = validate_dict(response_format, ResponseFormat) + response_format = {"response_format": response_format} + model_params.pop("stream", False) + stream_config = {} + if "stream_options" in model_params: + # stream_config["enabled"] = True + stream_options = model_params.pop("stream_options", {}) + stream_config["chunk_size"] = stream_options.get("chunk_size", 100) + if "delimiters" in stream_options: + stream_config["delimiters"] = stream_options.get("delimiters") + # else: + # stream_config["enabled"] = False + config = { + "config": { + "modules": { + "prompt_templating": { + "prompt": { + "template": template, + **tools, + **response_format + }, + "model": { + "name": model, + "params": model_params, + "version": model_version, + }, + }, + }, + "stream": stream_config, + } + } + + return config + + def transform_response( + self, + model: str, + raw_response: httpx.Response, + model_response: ModelResponse, + logging_obj: LiteLLMLoggingObj, + request_data: dict, + messages: List[AllMessageValues], + optional_params: dict, + litellm_params: dict, + encoding: Any, + api_key: Optional[str] = None, + json_mode: Optional[bool] = None, + ) -> ModelResponse: + logging_obj.post_call( + input=messages, + api_key=api_key, + original_response=raw_response.text, + additional_args={"complete_input_dict": request_data}, + ) + return ModelResponse.model_validate(raw_response.json()["final_result"]) + + def get_model_response_iterator( + self, + streaming_response: Union[Iterator[str], AsyncIterator[str], "ModelResponse"], + sync_stream: bool, + json_mode: Optional[bool] = False, + ): + if sync_stream: + return SAPStreamIterator(response=streaming_response) # type: ignore + else: + return AsyncSAPStreamIterator(response=streaming_response) # type: ignore diff --git a/litellm/llms/sap/credentials.py b/litellm/llms/sap/credentials.py new file mode 100644 index 00000000000..e10bcbf7eae --- /dev/null +++ b/litellm/llms/sap/credentials.py @@ -0,0 +1,325 @@ +from __future__ import annotations +from typing import Any, Callable, Dict, Final, List, Optional, Sequence, Tuple +from datetime import datetime, timedelta, timezone +from threading import Lock +from pathlib import Path +from dataclasses import dataclass +import json +import os +import tempfile + +from litellm import sap_service_key +from litellm.llms.custom_httpx.http_handler import _get_httpx_client + +AUTH_ENDPOINT_SUFFIX = "/oauth/token" + +CONFIG_FILE_ENV_VAR = "AICORE_CONFIG" +HOME_PATH_ENV_VAR = "AICORE_HOME" +PROFILE_ENV_VAR = "AICORE_PROFILE" + +VCAP_SERVICES_ENV_VAR = "VCAP_SERVICES" +VCAP_AICORE_SERVICE_NAME = "aicore" +SERVICE_KEY_ENV_VAR = "AICORE_SERVICE_KEY" + +DEFAULT_HOME_PATH = os.path.join(os.path.expanduser("~"), ".aicore") + + +def _get_home() -> str: + return os.getenv(HOME_PATH_ENV_VAR, DEFAULT_HOME_PATH) + + +def _get_nested(d: Dict[str, Any], path: Sequence[str]) -> Any: + cur: Any = d + for k in path: + if not isinstance(cur, dict) or k not in cur: + raise KeyError(".".join(path)) + cur = cur[k] + return cur + + +def _load_json_env(var_name: str) -> Optional[Dict[str, Any]]: + raw = os.environ.get(var_name) + if not raw: + return None + try: + return json.loads(raw) + except json.JSONDecodeError: + return None + + +def _load_vcap() -> Dict[str, Any]: + return _load_json_env(VCAP_SERVICES_ENV_VAR) or {} + + +def _get_vcap_service(label: str) -> Optional[Dict[str, Any]]: + for services in _load_vcap().values(): + for svc in services: + if svc.get("label") == label: + return svc + return None + + +@dataclass(frozen=True) +class CredentialsValue: + name: str + vcap_key: Optional[Tuple[str, ...]] = None + default: Optional[str] = None + transform_fn: Optional[Callable[[str], str]] = None + + +CREDENTIAL_VALUES: Final[List[CredentialsValue]] = [ + CredentialsValue("client_id", ("clientid",)), + CredentialsValue("client_secret", ("clientsecret",)), + CredentialsValue( + "auth_url", + ("url",), + transform_fn=lambda url: url.rstrip("/") + + ("" if url.endswith(AUTH_ENDPOINT_SUFFIX) else AUTH_ENDPOINT_SUFFIX), + ), + CredentialsValue( + "base_url", + ("serviceurls", "AI_API_URL"), + transform_fn=lambda url: url.rstrip("/") + + ("" if url.endswith("/v2") else "/v2"), + ), + CredentialsValue("resource_group", default="default"), + CredentialsValue( + "cert_url", + ("certurl",), + transform_fn=lambda url: url.rstrip("/") + + ("" if url.endswith(AUTH_ENDPOINT_SUFFIX) else AUTH_ENDPOINT_SUFFIX), + ), + # file paths (kept for config compatibility) + CredentialsValue("cert_file_path"), + CredentialsValue("key_file_path"), + # inline PEMs from VCAP + CredentialsValue( + "cert_str", ("certificate",), transform_fn=lambda s: s.replace("\\n", "\n") + ), + CredentialsValue( + "key_str", ("key",), transform_fn=lambda s: s.replace("\\n", "\n") + ), +] + + +def init_conf(profile: Optional[str] = None) -> Dict[str, Any]: + """ + Loads config JSON from: + 1) $AICORE_CONFIG if set, otherwise + 2) $AICORE_HOME/config.json (or config_.json when profile is given/not default) + Returns {} when nothing is found. + """ + home = Path(_get_home()) + profile = profile or os.environ.get(PROFILE_ENV_VAR) + cfg_env = os.getenv(CONFIG_FILE_ENV_VAR) + cfg_path = ( + Path(cfg_env) + if cfg_env + else ( + home + / ( + "config.json" + if profile in (None, "", "default") + else f"config_{profile}.json" + ) + ) + ) + + if cfg_path and cfg_path.exists(): + try: + with cfg_path.open(encoding="utf-8") as f: + return json.load(f) + except json.JSONDecodeError: + raise KeyError(f"{cfg_path} is not valid JSON. Please fix or remove it!") + + # If an explicit non-default profile was requested but not found, raise. + if cfg_env or (profile not in (None, "", "default")): + raise FileNotFoundError( + f"Unable to locate profile config file at '{cfg_path}' in AICORE_HOME '{home}'" + ) + + return {} + + +def _env_name(name: str) -> str: + return f"AICORE_{name.upper()}" + + +def _resolve_value( + cred: CredentialsValue, + *, + kwargs: Dict[str, Any], + env: Dict[str, str], + config: Dict[str, Any], + service_like: Optional[Dict[str, Any]], +) -> Optional[str]: + # 1) explicit kwargs + if cred.name in kwargs and kwargs[cred.name] is not None: + return kwargs[cred.name] + + # 2) environment variables (primary name) + env_key = _env_name(cred.name) + if env_key in env and env[env_key] is not None: + return env[env_key] + + # 3) config file (accept both prefixed and plain keys) + for key in (env_key, cred.name): + if key in config and config[key] is not None: + return config[key] + + # 4) service-like source (AICORE_SERVICE_KEY first, else VCAP) + if service_like and cred.vcap_key: + try: + val = _get_nested(service_like, ("credentials",) + cred.vcap_key) + if val is not None: + return val + except KeyError: + pass + + # 5) default + return cred.default + + +def fetch_credentials(service_key: Optional[str] = None, profile: Optional[str] = None, **kwargs) -> Dict[str, str]: + """ + Resolution order per key: + kwargs + > env (AICORE_) + > config (AICORE_ or plain ) + > service-like source from JSON in $AICORE_SERVICE_KEY (same structure as a VCAP service object) + falling back to service entry in $VCAP_SERVICES with label 'aicore' + > default + """ + config = init_conf(profile) + env = os.environ # snapshot for testability + service_like = None + + if not config: + # Prefer AICORE_SERVICE_KEY if present; otherwise fall back to the VCAP service. + service_like = service_key or sap_service_key or _load_json_env(SERVICE_KEY_ENV_VAR) or _get_vcap_service( + VCAP_AICORE_SERVICE_NAME + ) + + out: Dict[str, str] = {} + for cred in CREDENTIAL_VALUES: + value = _resolve_value(cred, kwargs=kwargs, env=env, config=config, service_like=service_like) # type: ignore + if value is None: + continue + if cred.transform_fn: + value = cred.transform_fn(value) + out[cred.name] = value + if "cert_url" in out.keys(): + out["auth_url"] = out.pop("cert_url") + return out + + +def get_token_creator( + service_key: Optional[str] = None, + profile: Optional[str] = None, + *, + timeout: float = 30.0, + expiry_buffer_minutes: int = 60, + **overrides, +) -> Tuple[Callable[[], str], str, str]: + """ + Creates a callable that fetches and caches an OAuth2 bearer token + using credentials from `fetch_credentials()`. + + The callable: + - Automatically loads credentials via fetch_credentials(profile, **overrides) + - Fetches a new token only if expired or near expiry + - Caches token thread-safely with a configurable refresh buffer + + Args: + profile: Optional AICore profile name + timeout: HTTP request timeout in seconds (default 30s) + expiry_buffer_minutes: Refresh the token this many minutes before expiry + overrides: Any explicit credential overrides (client_id, client_secret, etc.) + + Returns: + Callable[[], str]: function returning a valid "Bearer " string. + """ + + # Resolve credentials using your helper + credentials: Dict[str, str] = fetch_credentials(service_key=service_key, profile=profile, **overrides) + + auth_url = credentials.get("auth_url") + client_id = credentials.get("client_id") + client_secret = credentials.get("client_secret") + cert_str = credentials.get("cert_str") + key_str = credentials.get("key_str") + cert_file_path = credentials.get("cert_file_path") + key_file_path = credentials.get("key_file_path") + + # Sanity check + if not auth_url or not client_id: + raise ValueError( + "fetch_credentials did not return valid 'auth_url' or 'client_id'" + ) + + modes = [ + client_secret is not None, + (cert_str is not None and key_str is not None), + (cert_file_path is not None and key_file_path is not None), + ] + if sum(bool(m) for m in modes) != 1: + raise ValueError( + "Invalid credentials: provide exactly one of client_secret, " + "(cert_str & key_str), or (cert_file_path & key_file_path)." + ) + + lock = Lock() + token: Optional[str] = None + token_expiry: Optional[datetime] = None + + def _request_token(cert_pair=None) -> tuple[str, datetime]: + data = {"grant_type": "client_credentials", "client_id": client_id} + if client_secret: + data["client_secret"] = client_secret + + client = _get_httpx_client() + # with httpx.Client(cert=cert_pair, timeout=timeout) as client: + resp = client.post(auth_url, data=data) + try: + resp.raise_for_status() + payload = resp.json() + access_token = payload["access_token"] + expires_in = int(payload.get("expires_in", 3600)) + expiry_date = datetime.now(timezone.utc) + timedelta(seconds=expires_in) + return f"Bearer {access_token}", expiry_date + except Exception as e: + msg = getattr(resp, "text", str(e)) + raise RuntimeError(f"Token request failed: {msg}") from e + + def _fetch_token() -> tuple[str, datetime]: + # Case 1: secret-based auth + if client_secret: + return _request_token() + # Case 2: cert/key strings + if cert_str and key_str: + cert_str_fixed = cert_str.replace("\\n", "\n") + key_str_fixed = key_str.replace("\\n", "\n") + with tempfile.TemporaryDirectory() as tmp: + cert_path = os.path.join(tmp, "cert.pem") + key_path = os.path.join(tmp, "key.pem") + with open(cert_path, "w") as f: + f.write(cert_str_fixed) + with open(key_path, "w") as f: + f.write(key_str_fixed) + return _request_token(cert_pair=(cert_path, key_path)) + # Case 3: file-based cert/key + return _request_token(cert_pair=(cert_file_path, key_file_path)) + + def get_token() -> str: + nonlocal token, token_expiry + with lock: + now = datetime.now(timezone.utc) + if ( + token is None + or token_expiry is None + or token_expiry - now < timedelta(minutes=expiry_buffer_minutes) + ): + token, token_expiry = _fetch_token() + return token + + return get_token, credentials["base_url"], credentials["resource_group"] diff --git a/litellm/llms/sap/embed/transformation.py b/litellm/llms/sap/embed/transformation.py new file mode 100644 index 00000000000..93f32c00abd --- /dev/null +++ b/litellm/llms/sap/embed/transformation.py @@ -0,0 +1,176 @@ +""" +Translates from OpenAI's `/v1/embeddings` to IBM's `/text/embeddings` route. +""" + +from typing import Optional, List, Dict, Literal +from pydantic import BaseModel, Field +from functools import cached_property + +import httpx + +from litellm.llms.base_llm.embedding.transformation import ( + BaseEmbeddingConfig, + LiteLLMLoggingObj, +) +from litellm.types.llms.openai import AllEmbeddingInputValues +from litellm.types.utils import EmbeddingResponse + +from ..chat.handler import GenAIHubOrchestrationError +from ..credentials import get_token_creator + + +class Usage(BaseModel): + prompt_tokens: int + total_tokens: int + + +class EmbeddingItem(BaseModel): + object: Literal["embedding"] + embedding: List[float] = Field( + ..., description="Vector of floats (length varies by model)." + ) + index: int + + +class FinalResult(BaseModel): + object: Literal["list"] + data: List[EmbeddingItem] + model: str + usage: Usage + + +class EmbeddingsResponse(BaseModel): + request_id: str + final_result: FinalResult + + +class EmbeddingModel(BaseModel): + name: str + version: str = "latest" + params: dict = Field(default_factory=dict, validation_alias="parameters") + + +class EmbeddingsModules(BaseModel): + embeddings: EmbeddingModel + + +class EmbeddingInput(BaseModel): + text: str | List[str] + type: Literal["text", "document", "query"] = "text" + + +class EmbeddingRequest(BaseModel): + config: EmbeddingsModules + input: EmbeddingInput + + +def validate_dict(data: dict, model) -> dict: + return model(**data).model_dump() + + +class GenAIHubEmbeddingConfig(BaseEmbeddingConfig): + def __init__(self): + super().__init__() + self._access_token_data = {} + self.token_creator, self.base_url, self.resource_group = get_token_creator() + + @property + def headers(self) -> Dict: + access_token = self.token_creator() + # headers for completions and embeddings requests + headers = { + "Authorization": access_token, + "AI-Resource-Group": self.resource_group, + "Content-Type": "application/json", + } + return headers + + @cached_property + def deployment_url(self) -> str: + with httpx.Client(timeout=30) as client: + valid_deployments = [] + deployments = client.get( + self.base_url + "/lm/deployments", headers=self.headers + ).json() + for deployment in deployments.get("resources", []): + if deployment["scenarioId"] == "orchestration": + config_details = client.get( + self.base_url + + f'/lm/configurations/{deployment["configurationId"]}', + headers=self.headers, + ).json() + if config_details["executableId"] == "orchestration": + valid_deployments.append( + (deployment["deploymentUrl"], deployment["createdAt"]) + ) + return sorted(valid_deployments, key=lambda x: x[1], reverse=True)[0][0] + + def get_error_class(self, error_message, status_code, headers): + return GenAIHubOrchestrationError(status_code, error_message) + + def get_supported_openai_params(self, model: str) -> list: + if "text-embedding-3" in model: + return ["encoding_format", "dimensions"] + else: + return [ + "encoding_format", + ] + + def map_openai_params( + self, + non_default_params: dict, + optional_params: dict, + model: str, + drop_params: bool, + ) -> dict: + return optional_params + + def validate_environment(self, headers: dict, *args, **kwargs) -> dict: + return self.headers + + def get_complete_url( + self, + api_base: Optional[str], + api_key: Optional[str], + model: str, + optional_params: dict, + litellm_params: dict, + stream: Optional[bool] = None, + ) -> str: + url = self.deployment_url.rstrip("/") + "/v2/embeddings" + return url + + def transform_embedding_request( + self, + model: str, + input: AllEmbeddingInputValues, + optional_params: dict, + headers: dict, + ) -> dict: + model_dict = {} + model_dict["name"] = model + model_dict["version"] = optional_params.get("version", "latest") + model_dict["params"] = optional_params.get("parameters", {}) + input_dict = {"text": input} + body = { + "config": { + "modules": { + "embeddings": {"model": validate_dict(model_dict, EmbeddingModel)} + } + }, + "input": validate_dict(input_dict, EmbeddingInput), + } + return body + + def transform_embedding_response( + self, + model: str, + raw_response: httpx.Response, + model_response: EmbeddingResponse, + logging_obj: LiteLLMLoggingObj, + api_key: Optional[str], + request_data: dict, + optional_params: dict, + litellm_params: dict, + ) -> EmbeddingResponse: + return EmbeddingResponse.model_validate(raw_response.json()["final_result"]) diff --git a/litellm/main.py b/litellm/main.py index 59838c2a035..20b2cbb7db8 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -176,6 +176,7 @@ from .llms.databricks.embed.handler import DatabricksEmbeddingHandler from .llms.deprecated_providers import aleph_alpha, palm from .llms.gemini.common_utils import get_api_key_from_env from .llms.groq.chat.handler import GroqChatCompletion +from .llms.sap.chat.handler import GenAIHubOrchestration from .llms.heroku.chat.transformation import HerokuChatConfig from .llms.huggingface.embedding.handler import HuggingFaceEmbedding from .llms.lemonade.chat.transformation import LemonadeChatConfig @@ -255,6 +256,8 @@ openai_text_completions = OpenAITextCompletion() openai_audio_transcriptions = OpenAIAudioTranscription() openai_image_variations = OpenAIImageVariationsHandler() groq_chat_completions = GroqChatCompletion() +sap_gen_ai_hub_chat_completions = GenAIHubOrchestration() +sap_gen_ai_hub_emb = GenAIHubOrchestration() azure_ai_embedding = AzureAIEmbedding() anthropic_chat_completions = AnthropicChatCompletion() azure_anthropic_chat_completions = AzureAnthropicChatCompletion() @@ -2093,6 +2096,34 @@ def completion( # type: ignore # noqa: PLR0915 logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements client=client, ) + elif custom_llm_provider == "sap": + headers = headers or litellm.headers + ## LOAD CONFIG - if set + config = litellm.GenAIHubOrchestrationConfig.get_config() + for k, v in config.items(): + if ( + k not in optional_params + ): # completion(top_k=3) > openai_config(top_k=3) <- allows for dynamic variables to be passed in + optional_params[k] = v + + response = sap_gen_ai_hub_chat_completions.completion( + model=model, + messages=messages, + headers=headers, + model_response=model_response, + acompletion=acompletion, + logging_obj=logging, + optional_params=optional_params, + litellm_params=litellm_params, + timeout=timeout, # type: ignore + shared_session=shared_session, + client=client, + custom_llm_provider=custom_llm_provider, + encoding=encoding, + api_key=api_key, + api_base=api_base, + stream=stream, + ) elif custom_llm_provider == "aiohttp_openai": # NEW aiohttp provider for 10-100x higher RPS api_base = ( @@ -4858,6 +4889,21 @@ def embedding( # noqa: PLR0915 client=client, aembedding=aembedding, ) + elif custom_llm_provider == "sap": + response = base_llm_http_handler.embedding( + model=model, + input=input, + custom_llm_provider=custom_llm_provider, + api_base=api_base, + api_key=api_key, + logging_obj=logging, + timeout=timeout, + model_response=EmbeddingResponse(), + optional_params=optional_params, + litellm_params={}, + client=client, + aembedding=aembedding, + ) elif custom_llm_provider == "azure_ai": api_base = ( api_base # for deepinfra/perplexity/anyscale/groq/friendliai we check in get_llm_provider and pass in the api base from there diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index fde60a92370..ca438b5fd24 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -255,6 +255,50 @@ "mode": "image_generation", "output_cost_per_image": 0.06 }, + "us.writer.palmyra-x4-v1:0": { + "input_cost_per_token": 2.5e-06, + "litellm_provider": "bedrock_converse", + "max_input_tokens": 128000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 1e-05, + "supports_function_calling": true, + "supports_pdf_input": true + }, + "us.writer.palmyra-x5-v1:0": { + "input_cost_per_token": 6e-07, + "litellm_provider": "bedrock_converse", + "max_input_tokens": 1000000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 6e-06, + "supports_function_calling": true, + "supports_pdf_input": true + }, + "writer.palmyra-x4-v1:0": { + "input_cost_per_token": 2.5e-06, + "litellm_provider": "bedrock_converse", + "max_input_tokens": 128000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 1e-05, + "supports_function_calling": true, + "supports_pdf_input": true + }, + "writer.palmyra-x5-v1:0": { + "input_cost_per_token": 6e-07, + "litellm_provider": "bedrock_converse", + "max_input_tokens": 1000000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 6e-06, + "supports_function_calling": true, + "supports_pdf_input": true + }, "amazon.nova-lite-v1:0": { "input_cost_per_token": 6e-08, "litellm_provider": "bedrock_converse", @@ -6206,6 +6250,19 @@ "supports_function_calling": true, "supports_tool_choice": true }, + "cerebras/zai-glm-4.6": { + "input_cost_per_token": 2.25e-06, + "litellm_provider": "cerebras", + "max_input_tokens": 128000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 2.75e-06, + "source": "https://www.cerebras.ai/pricing", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_tool_choice": true + }, "chat-bison": { "input_cost_per_character": 2.5e-07, "input_cost_per_token": 1.25e-07, @@ -22865,6 +22922,13 @@ "mode": "rerank", "output_cost_per_token": 0.0 }, + "nvidia_nim/ranking/nvidia/llama-3.2-nv-rerankqa-1b-v2": { + "input_cost_per_query": 0.0, + "input_cost_per_token": 0.0, + "litellm_provider": "nvidia_nim", + "mode": "rerank", + "output_cost_per_token": 0.0 + }, "sagemaker/meta-textgeneration-llama-2-13b": { "input_cost_per_token": 0.0, "litellm_provider": "sagemaker", diff --git a/litellm/proxy/_experimental/mcp_server/ui_session_utils.py b/litellm/proxy/_experimental/mcp_server/ui_session_utils.py new file mode 100644 index 00000000000..6572b831a27 --- /dev/null +++ b/litellm/proxy/_experimental/mcp_server/ui_session_utils.py @@ -0,0 +1,85 @@ +"""Helpers to resolve real team contexts for UI session tokens.""" + +from __future__ import annotations + +from typing import List + +from litellm._logging import verbose_logger +from litellm.constants import UI_SESSION_TOKEN_TEAM_ID +from litellm.proxy._types import UserAPIKeyAuth + + +def clone_user_api_key_auth_with_team( + user_api_key_auth: UserAPIKeyAuth, + team_id: str, +) -> UserAPIKeyAuth: + """Return a deep copy of the auth context with a different team id.""" + + try: + cloned_auth = user_api_key_auth.model_copy(deep=True) + except AttributeError: + cloned_auth = user_api_key_auth.copy(deep=True) # type: ignore[attr-defined] + cloned_auth.team_id = team_id + return cloned_auth + + +async def resolve_ui_session_team_ids( + user_api_key_auth: UserAPIKeyAuth, +) -> List[str]: + """Resolve the real team ids backing a UI session token.""" + + if ( + user_api_key_auth.team_id != UI_SESSION_TOKEN_TEAM_ID + or not user_api_key_auth.user_id + ): + return [] + + from litellm.proxy.auth.auth_checks import get_user_object + from litellm.proxy.proxy_server import ( + prisma_client, + proxy_logging_obj, + user_api_key_cache, + ) + + if prisma_client is None: + verbose_logger.debug("Cannot resolve UI session team ids without DB access") + return [] + + try: + user_obj = await get_user_object( + user_id=user_api_key_auth.user_id, + prisma_client=prisma_client, + user_api_key_cache=user_api_key_cache, + user_id_upsert=False, + parent_otel_span=user_api_key_auth.parent_otel_span, + proxy_logging_obj=proxy_logging_obj, + ) + except Exception as exc: # pragma: no cover - defensive logging + verbose_logger.warning( + "Failed to load teams for UI session token user.", + exc, + ) + return [] + + if user_obj is None or not user_obj.teams: + return [] + + resolved_team_ids: List[str] = [] + for team_id in user_obj.teams: + if team_id and team_id not in resolved_team_ids: + resolved_team_ids.append(team_id) + return resolved_team_ids + + +async def build_effective_auth_contexts( + user_api_key_auth: UserAPIKeyAuth, +) -> List[UserAPIKeyAuth]: + """Return auth contexts that reflect the actual teams for UI session tokens.""" + + resolved_team_ids = await resolve_ui_session_team_ids(user_api_key_auth) + if resolved_team_ids: + return [ + clone_user_api_key_auth_with_team(user_api_key_auth, team_id) + for team_id in resolved_team_ids + ] + return [user_api_key_auth] diff --git a/litellm/proxy/auth/auth_checks.py b/litellm/proxy/auth/auth_checks.py index fc79a4d3591..309bd577606 100644 --- a/litellm/proxy/auth/auth_checks.py +++ b/litellm/proxy/auth/auth_checks.py @@ -402,13 +402,14 @@ def _allowed_routes_check(user_route: str, allowed_routes: list) -> bool: - user_route: str - the route the user is trying to call - allowed_routes: List[str|LiteLLMRoutes] - the list of allowed routes for the user. """ + from starlette.routing import compile_path for allowed_route in allowed_routes: - if ( - allowed_route in LiteLLMRoutes.__members__ - and user_route in LiteLLMRoutes[allowed_route].value - ): - return True + if allowed_route in LiteLLMRoutes.__members__: + for template in LiteLLMRoutes[allowed_route].value: + regex, _, _ = compile_path(template) + if regex.match(user_route): + return True elif allowed_route == user_route: return True return False diff --git a/litellm/proxy/auth/auth_utils.py b/litellm/proxy/auth/auth_utils.py index e6de65da2bd..c4d0d2f8f1c 100644 --- a/litellm/proxy/auth/auth_utils.py +++ b/litellm/proxy/auth/auth_utils.py @@ -251,30 +251,38 @@ def route_in_additonal_public_routes(current_route: str): - bool - True if the route is defined in public_routes - bool - False if the route is not defined in public_routes + Supports wildcard patterns (e.g., "/api/*" matches "/api/users", "/api/users/123") In order to use this the litellm config.yaml should have the following in general_settings: ```yaml general_settings: master_key: sk-1234 - public_routes: ["LiteLLMRoutes.public_routes", "/spend/calculate"] + public_routes: ["LiteLLMRoutes.public_routes", "/spend/calculate", "/api/*"] ``` """ - - # check if user is premium_user - if not do nothing + from litellm.proxy.auth.route_checks import RouteChecks from litellm.proxy.proxy_server import general_settings, premium_user try: if premium_user is not True: return False - # check if this is defined on the config if general_settings is None: return False routes_defined = general_settings.get("public_routes", []) + + # Check exact match first if current_route in routes_defined: return True + # Check wildcard patterns + for route_pattern in routes_defined: + if RouteChecks._route_matches_wildcard_pattern( + route=current_route, pattern=route_pattern + ): + return True + return False except Exception as e: verbose_proxy_logger.error(f"route_in_additonal_public_routes: {str(e)}") diff --git a/litellm/proxy/common_utils/html_forms/ui_login.py b/litellm/proxy/common_utils/html_forms/ui_login.py index 8478d41e475..42cfb592a78 100644 --- a/litellm/proxy/common_utils/html_forms/ui_login.py +++ b/litellm/proxy/common_utils/html_forms/ui_login.py @@ -8,7 +8,22 @@ if server_root_path != "": url_to_redirect_to += server_root_path url_to_redirect_to += "/login" new_ui_login_url = get_custom_url("", "ui/login") -html_form = f""" + + +def build_ui_login_form(show_deprecation_banner: bool = False) -> str: + banner_html = ( + f""" +
+ Deprecated: Logging in with username and password on this page is deprecated. + Please use the new login page instead. + This page will be dedicated to signing in via SSO in the future. +
+ """ + if show_deprecation_banner + else "" + ) + + return f""" @@ -209,11 +224,7 @@ html_form = f"""
-
- Deprecated: Logging in with username and password on this page is deprecated. - Please use the new login page instead. - This page will be dedicated to signing in via SSO in the future. -
+ {banner_html}