Merge remote-tracking branch 'origin/main' into fix_vertex_expired_tokens

.circleci/config.yml

@@ -1291,6 +1291,7 @@ jobs:
             pip install jinja2
             pip install "tokenizers==0.20.0"
             pip install "uvloop==0.21.0"
+            pip install "fastuuid==0.12.0"
             pip install jsonschema
       - setup_litellm_enterprise_pip
       - run:
@@ -1476,6 +1477,7 @@ jobs:
             docker run -d \
               -p 4000:4000 \
               -e DATABASE_URL=$PROXY_DATABASE_URL \
+              -e DEFAULT_NUM_WORKERS_LITELLM_PROXY=1 \
               -e DISABLE_SCHEMA_UPDATE="True" \
               -v $(pwd)/litellm/proxy/example_config_yaml/bad_schema.prisma:/app/schema.prisma \
               -v $(pwd)/litellm/proxy/example_config_yaml/bad_schema.prisma:/app/litellm/proxy/schema.prisma \
@@ -2961,6 +2963,7 @@ jobs:
           command: |
             docker run --name my-app \
               -p 4000:4000 \
+              -e DEFAULT_NUM_WORKERS_LITELLM_PROXY=1 \
               -e DATABASE_URL="postgresql://wrong:wrong@wrong:5432/wrong" \
               myapp:latest \
               --port 4000 > docker_output.log 2>&1 || true

.circleci/requirements.txt

@@ -14,4 +14,5 @@ google-cloud-iam==2.19.1
 fastapi-sso==0.16.0 
 uvloop==0.21.0
 mcp==1.10.1    # for MCP server  
-semantic_router==0.1.10 # for auto-routing with litellm
+semantic_router==0.1.10 # for auto-routing with litellm
+fastuuid==0.12.0

.github/workflows/auto_update_price_and_context_window_file.py

@@ -43,8 +43,8 @@ def write_to_file(file_path, data):
         # Print an error message if writing to file fails
         print("Error updating JSON file:", e)
 
-# Update the existing models and add the missing models
-def transform_remote_data(data):
+# Update the existing models and add the missing models for OpenRouter
+def transform_openrouter_data(data):
     transformed = {}
     for row in data:
         # Add the fields 'max_tokens' and 'input_cost_per_token'
@@ -81,6 +81,34 @@ def transform_remote_data(data):
 
     return transformed
 
+# Update the existing models and add the missing models for Vercel AI Gateway
+def transform_vercel_ai_gateway_data(data):
+    transformed = {}
+    for row in data:
+        obj = {
+            "max_tokens": row["context_window"],
+            "input_cost_per_token": float(row["pricing"]["input"]),
+            "output_cost_per_token": float(row["pricing"]["output"]),
+            'max_output_tokens': row['max_tokens'],
+            'max_input_tokens': row["context_window"],
+        }
+
+        # Handle cache pricing if available
+        if "pricing" in row:
+            if "input_cache_read" in row["pricing"] and row["pricing"]["input_cache_read"] is not None:
+                obj['cache_read_input_token_cost'] = float(f"{float(row['pricing']['input_cache_read']):e}")
+            
+            if "input_cache_write" in row["pricing"] and row["pricing"]["input_cache_write"] is not None:
+                obj['cache_creation_input_token_cost'] = float(f"{float(row['pricing']['input_cache_write']):e}")
+
+        mode = "embedding" if "embedding" in row["id"].lower() else "chat"
+        
+        obj.update({"litellm_provider": "vercel_ai_gateway", "mode": mode})
+
+        transformed[f'vercel_ai_gateway/{row["id"]}'] = obj
+
+    return transformed
+
 
 # Load local data from a specified file
 def load_local_data(file_path):
@@ -100,22 +128,32 @@ def load_local_data(file_path):
 
 def main():
     local_file_path = "model_prices_and_context_window.json"  # Path to the local data file
-    url = "https://openrouter.ai/api/v1/models"  # URL to fetch remote data
+    openrouter_url = "https://openrouter.ai/api/v1/models"  # URL to fetch OpenRouter data
+    vercel_ai_gateway_url = "https://ai-gateway.vercel.sh/v1/models"  # URL to fetch Vercel AI Gateway data
 
     # Load local data from file
     local_data = load_local_data(local_file_path)
-    # Fetch remote data asynchronously
-    remote_data = asyncio.run(fetch_data(url))
-    # Transform the fetched remote data
-    remote_data = transform_remote_data(remote_data)
+    
+    # Fetch OpenRouter data
+    openrouter_data = asyncio.run(fetch_data(openrouter_url))
+    # Transform the fetched OpenRouter data
+    openrouter_data = transform_openrouter_data(openrouter_data)
+    
+    # Fetch Vercel AI Gateway data
+    vercel_data = asyncio.run(fetch_data(vercel_ai_gateway_url))
+    # Transform the fetched Vercel AI Gateway data
+    vercel_data = transform_vercel_ai_gateway_data(vercel_data)
+    
+    # Combine both datasets
+    all_remote_data = {**openrouter_data, **vercel_data}
 
-    # If both local and remote data are available, synchronize and save
-    if local_data and remote_data:
-        sync_local_data_with_remote(local_data, remote_data)
+    # If both local and openrouter data are available, synchronize and save
+    if local_data and all_remote_data:
+        sync_local_data_with_remote(local_data, all_remote_data)
         write_to_file(local_file_path, local_data)
     else:
         print("Failed to fetch model data from either local file or URL.")
 
 # Entry point of the script
 if __name__ == "__main__":
-    main()
+    main()

deploy/charts/litellm-helm/Chart.yaml

@@ -18,7 +18,7 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.4.5
+version: 0.4.6
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to

deploy/charts/litellm-helm/README.md

@@ -36,11 +36,50 @@ If `db.useStackgresOperator` is used (not yet implemented):
 | `service.port`                                             | TCP port that the Kubernetes Service will listen on.  Also the TCP port within the Pod that the proxy will listen on.                                                                 | `4000`  |
 | `service.loadBalancerClass`                                | Optional LoadBalancer implementation class (only used when `service.type` is `LoadBalancer`)                                                                                                          | `""`  |
 | `ingress.*`                                                | See [values.yaml](./values.yaml) for example settings                                                                                                                                 | N/A  |
-| `proxy_config.*`                                           | See [values.yaml](./values.yaml) for default settings.  See [example_config_yaml](../../../litellm/proxy/example_config_yaml/) for configuration examples.                            | N/A  |
-| `extraContainers[]`                                        | An array of additional containers to be deployed as sidecars alongside the LiteLLM Proxy.                                                                                             | `[]`  |
+| `proxyConfigMap.create`     | When `true`, render a ConfigMap from `.Values.proxy_config` and mount it.                                                                                                                                      | `true` |
+| `proxyConfigMap.name`       | When `create=false`, name of the existing ConfigMap to mount.                                                                                                                                                  | `""`  |
+| `proxyConfigMap.key`        | Key in the ConfigMap that contains the proxy config file.                                                                                                                                                      | `"config.yaml"` |
+| `proxy_config.*`            | See [values.yaml](./values.yaml) for default settings. Rendered into the ConfigMap’s `config.yaml` only when `proxyConfigMap.create=true`. See [example_config_yaml](../../../litellm/proxy/example_config_yaml/) for configuration examples. | `N/A` |
+| `extraContainers[]`         | An array of additional containers to be deployed as sidecars alongside the LiteLLM Proxy.
+| `pdb.enabled`                   | Enable a PodDisruptionBudget for the LiteLLM proxy Deployment                                                                 | `false` |
+| `pdb.minAvailable`             | Minimum number/percentage of pods that must be available during **voluntary** disruptions (choose **one** of minAvailable/maxUnavailable) | `null`  |
+| `pdb.maxUnavailable`           | Maximum number/percentage of pods that can be unavailable during **voluntary** disruptions (choose **one** of minAvailable/maxUnavailable) | `null`  |
+| `pdb.annotations`              | Extra metadata annotations to add to the PDB                                                                                   | `{}`    |
+| `pdb.labels`                   | Extra metadata labels to add to the PDB                                                                                        | `{}`    |
+
+#### Example `proxy_config` ConfigMap from values (default):
+
+
+```
+proxyConfigMap:
+  create: true
+  key: "config.yaml"
+
+proxy_config:
+  general_settings:
+    master_key: os.environ/PROXY_MASTER_KEY
+  model_list:
+    - model_name: gpt-3.5-turbo
+      litellm_params:
+        model: gpt-3.5-turbo
+        api_key: eXaMpLeOnLy
+```
+
+#### Example using existing `proxyConfigMap` instead of creating it:
+
+
+```
+proxyConfigMap:
+  create: false
+  name: my-litellm-config
+  key: config.yaml
+
+# proxy_config is ignored in this mode
+```
 
 #### Example `environmentSecrets` Secret 
 
+
 ```
 apiVersion: v1
 kind: Secret

deploy/charts/litellm-helm/templates/NOTES.txt

@@ -20,3 +20,4 @@
   echo "Visit http://127.0.0.1:8080 to use your application"
   kubectl --namespace {{ .Release.Namespace }} port-forward $POD_NAME 8080:$CONTAINER_PORT
 {{- end }}
+PDB: {{ if .Values.pdb.enabled }}enabled{{ else }}disabled{{ end }}. Configure via .Values.pdb.*

deploy/charts/litellm-helm/templates/configmap-litellm.yaml

@@ -1,7 +1,9 @@
+{{- if .Values.proxyConfigMap.create }}
 apiVersion: v1
 kind: ConfigMap
 metadata:
   name: {{ include "litellm.fullname" . }}-config
 data:
   config.yaml: |
-{{ .Values.proxy_config | toYaml | indent 6 }}
+{{ .Values.proxy_config | toYaml | indent 6 }}
+{{- end }}

deploy/charts/litellm-helm/templates/deployment.yaml

@@ -16,7 +16,9 @@ spec:
   template:
     metadata:
       annotations:
+        {{- if .Values.proxyConfigMap.create }}
         checksum/config: {{ include (print $.Template.BasePath "/configmap-litellm.yaml") . | sha256sum }}
+        {{- end }}
         {{- with .Values.podAnnotations }}
         {{- toYaml . | nindent 8 }}
         {{- end }}
@@ -183,9 +185,13 @@ spec:
         {{- end }}
         - name: litellm-config
           configMap:
+            {{- if .Values.proxyConfigMap.create }}
             name: {{ include "litellm.fullname" . }}-config
+            {{- else }}
+            name: {{ .Values.proxyConfigMap.name }}
+            {{- end }}
             items:
-              - key: "config.yaml"
+              - key: {{ .Values.proxyConfigMap.key | default "config.yaml" }}
                 path: "config.yaml"
       {{- with .Values.volumes }}
         {{- toYaml . | nindent 8 }}

deploy/charts/litellm-helm/templates/poddisruptionbudget.yaml

@@ -0,0 +1,33 @@
+{{- /*
+PodDisruptionBudget for LiteLLM proxy
+Controlled via .Values.pdb.enabled and .Values.pdb.{minAvailable|maxUnavailable}
+Only one of minAvailable / maxUnavailable should be set. If both are set, minAvailable wins.
+*/ -}}
+{{- if .Values.pdb.enabled }}
+apiVersion: policy/v1
+kind: PodDisruptionBudget
+metadata:
+  name: {{ include "litellm.fullname" . }}
+  labels:
+    {{- include "litellm.labels" . | nindent 4 }}
+  {{- with .Values.pdb.labels }}
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
+  {{- with .Values.pdb.annotations }}
+  annotations:
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
+spec:
+  selector:
+    matchLabels:
+      {{- /* Match the Deployment selector to target the same pod set */ -}}
+      {{- include "litellm.selectorLabels" . | nindent 6 }}
+  {{- if .Values.pdb.minAvailable }}
+  minAvailable: {{ .Values.pdb.minAvailable }}
+  {{- else if .Values.pdb.maxUnavailable }}
+  maxUnavailable: {{ .Values.pdb.maxUnavailable }}
+  {{- else }}
+  # Safe default if enabled but not configured
+  maxUnavailable: 1
+  {{- end }}
+{{- end }}

deploy/charts/litellm-helm/tests/deployment_tests.yaml

@@ -115,3 +115,25 @@ tests:
           content:
             name: EXTRA_ENV_VAR
             value: EXTRA_ENV_VAR_VALUE
+  - it: should mount existing configmap when create=false
+    template: deployment.yaml
+    set:
+      proxyConfigMap:
+        create: false
+        name: my-litellm-config
+        key: custom.yaml
+    asserts:
+      - contains:
+          path: spec.template.spec.volumes
+          content:
+            name: litellm-config
+            configMap:
+              name: my-litellm-config
+              items:
+                - key: custom.yaml
+                  path: config.yaml
+      - contains:
+          path: spec.template.spec.containers[0].volumeMounts
+          content:
+            name: litellm-config
+            mountPath: /etc/litellm/

deploy/charts/litellm-helm/tests/pdb_tests.yaml

@@ -0,0 +1,45 @@
+suite: "pdb enabled"
+templates:
+  - poddisruptionbudget.yaml
+tests:
+  - it: "renders a PDB with maxUnavailable=1"
+    set:
+      pdb.enabled: true
+      pdb.maxUnavailable: 1
+    asserts:
+      - hasDocuments: { count: 1 }
+      - isKind: { of: PodDisruptionBudget }
+      - equal: { path: apiVersion, value: policy/v1 }
+      - equal: { path: spec.maxUnavailable, value: 1 }
+      - equal:
+          path: spec.selector.matchLabels
+          value:
+            app.kubernetes.io/name: litellm
+            app.kubernetes.io/instance: RELEASE-NAME
+
+---
+suite: "pdb disabled"
+templates:
+  - poddisruptionbudget.yaml
+tests:
+  - it: "does not render when disabled"
+    set:
+      pdb.enabled: false
+    asserts:
+      - hasDocuments: { count: 0 }
+
+---
+suite: "pdb minAvailable precedence"
+templates:
+  - poddisruptionbudget.yaml
+tests:
+  - it: "uses minAvailable when both are set"
+    set:
+      pdb.enabled: true
+      pdb.minAvailable: "50%"
+      pdb.maxUnavailable: 1
+    asserts:
+      - isKind: { of: PodDisruptionBudget }
+      - equal: { path: apiVersion, value: policy/v1 }
+      - equal: { path: spec.minAvailable, value: "50%" }
+      - isNull: { path: spec.maxUnavailable }

deploy/charts/litellm-helm/values.yaml

@@ -93,6 +93,14 @@ masterkeySecretName: ""
 # if set, use this secret key for the master key; otherwise, use the default key
 masterkeySecretKey: ""
 
+proxyConfigMap:
+  # when true, creates a new configmap
+  create: true
+  # if create is false and name is set, use existing ConfigMap
+  # create: false
+  # name: ""
+  # key: "config.yaml"
+
 # The elements within proxy_config are rendered as config.yaml for the proxy
 #  Examples: https://github.com/BerriAI/litellm/tree/main/litellm/proxy/example_config_yaml
 #  Reference: https://docs.litellm.ai/docs/proxy/configs
@@ -232,4 +240,11 @@ extraEnvVars: {
     #   value: EXTRA_ENV_VAR_VALUE
 }
 
-
+# Pod Disruption Budget
+pdb:
+  enabled: false
+  # Set exactly one of the following. If both are set, minAvailable takes precedence.
+  minAvailable: null     # e.g. "50%" or 1
+  maxUnavailable: null   # e.g. 1 or "20%"
+  annotations: {}
+  labels: {}

docker/Dockerfile.non_root

@@ -33,11 +33,12 @@ WORKDIR /app
 # Install runtime dependencies
 USER root
 RUN apk upgrade --no-cache && \
-  apk add --no-cache bash libstdc++ ca-certificates openssl
+  apk add --no-cache bash libstdc++ ca-certificates openssl supervisor
 
 # Copy only necessary artifacts from builder stage for runtime
 COPY . .
 COPY --from=builder /app/docker/entrypoint.sh /app/docker/prod_entrypoint.sh /app/docker/
+COPY --from=builder /app/docker/supervisord.conf /etc/supervisord.conf
 COPY --from=builder /app/schema.prisma /app/schema.prisma
 COPY --from=builder /app/dist/*.whl .
 COPY --from=builder /wheels/ /wheels/

docs/my-website/docs/completion/image_generation_chat.md

@@ -0,0 +1,232 @@
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+
+# Image Generation in Chat Completions, Responses API
+
+This guide covers how to generate images when using the `chat/completions`. Note - if you want this on Responses API please file a Feature Request [here](https://github.com/BerriAI/litellm/issues/new).
+
+:::info
+
+Requires LiteLLM v1.76.1+
+
+:::
+
+Supported Providers:
+- Google AI Studio (`gemini`)
+- Vertex AI (`vertex_ai/`)
+
+LiteLLM will standardize the `image` response in the assistant message for models that support image generation during chat completions.
+
+```python title="Example response from litellm"
+"message": {
+    ...
+    "content": "Here's the image you requested:",
+    "image": {
+        "url": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAA...",
+        "detail": "auto"
+    }
+}
+```
+
+## Quick Start 
+
+<Tabs>
+<TabItem value="sdk" label="SDK">
+
+```python showLineNumbers title="Image generation with chat completion"
+from litellm import completion
+import os 
+
+os.environ["GEMINI_API_KEY"] = "your-api-key"
+
+response = completion(
+    model="gemini/gemini-2.5-flash-image-preview",
+    messages=[
+        {"role": "user", "content": "Generate an image of a banana wearing a costume that says LiteLLM"}
+    ],
+)
+
+print(response.choices[0].message.content)  # Text response
+print(response.choices[0].message.image)    # Image data
+```
+
+</TabItem>
+<TabItem value="proxy" label="PROXY">
+
+1. Setup config.yaml
+
+```yaml showLineNumbers title="config.yaml"
+model_list:
+  - model_name: gemini-image-gen
+    litellm_params:
+      model: gemini/gemini-2.5-flash-image-preview
+      api_key: os.environ/GEMINI_API_KEY
+```
+
+2. Run proxy server
+
+```bash showLineNumbers title="Start the proxy"
+litellm --config config.yaml
+
+# RUNNING on http://0.0.0.0:4000
+```
+
+3. Test it!
+
+```bash showLineNumbers title="Make request"
+curl http://0.0.0.0:4000/v1/chat/completions \
+  -H "Content-Type: application/json" \
+  -H "Authorization: Bearer $LITELLM_KEY" \
+  -d '{
+    "model": "gemini-image-gen",
+    "messages": [
+      {
+        "role": "user",
+        "content": "Generate an image of a banana wearing a costume that says LiteLLM"
+      }
+    ]
+  }'
+```
+
+</TabItem>
+</Tabs>
+
+**Expected Response**
+
+```bash
+{
+    "id": "chatcmpl-3b66124d79a708e10c603496b363574c",
+    "choices": [
+        {
+            "finish_reason": "stop",
+            "index": 0,
+            "message": {
+                "content": "Here's the image you requested:",
+                "role": "assistant",
+                "image": {
+                    "url": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAA...",
+                    "detail": "auto"
+                }
+            }
+        }
+    ],
+    "created": 1723323084,
+    "model": "gemini/gemini-2.5-flash-image-preview",
+    "object": "chat.completion",
+    "usage": {
+        "completion_tokens": 12,
+        "prompt_tokens": 16,
+        "total_tokens": 28
+    }
+}
+```
+
+## Streaming Support
+
+<Tabs>
+<TabItem value="sdk" label="SDK">
+
+```python showLineNumbers title="Streaming image generation"
+from litellm import completion
+import os 
+
+os.environ["GEMINI_API_KEY"] = "your-api-key"
+
+response = completion(
+    model="gemini/gemini-2.5-flash-image-preview",
+    messages=[
+        {"role": "user", "content": "Generate an image of a banana wearing a costume that says LiteLLM"}
+    ],
+    stream=True,
+)
+
+for chunk in response:
+    if hasattr(chunk.choices[0].delta, "image") and chunk.choices[0].delta.image is not None:
+        print("Generated image:", chunk.choices[0].delta.image["url"])
+        break
+```
+
+</TabItem>
+<TabItem value="proxy" label="PROXY">
+
+```bash showLineNumbers title="Streaming request"
+curl http://0.0.0.0:4000/v1/chat/completions \
+  -H "Content-Type: application/json" \
+  -H "Authorization: Bearer $LITELLM_KEY" \
+  -d '{
+    "model": "gemini-image-gen",
+    "messages": [
+      {
+        "role": "user",
+        "content": "Generate an image of a banana wearing a costume that says LiteLLM"
+      }
+    ],
+    "stream": true
+  }'
+```
+
+</TabItem>
+</Tabs>
+
+**Expected Streaming Response**
+
+```bash
+data: {"id":"chatcmpl-123","object":"chat.completion.chunk","created":1723323084,"model":"gemini/gemini-2.5-flash-image-preview","choices":[{"index":0,"delta":{"role":"assistant"},"finish_reason":null}]}
+
+data: {"id":"chatcmpl-123","object":"chat.completion.chunk","created":1723323084,"model":"gemini/gemini-2.5-flash-image-preview","choices":[{"index":0,"delta":{"content":"Here's the image you requested:"},"finish_reason":null}]}
+
+data: {"id":"chatcmpl-123","object":"chat.completion.chunk","created":1723323084,"model":"gemini/gemini-2.5-flash-image-preview","choices":[{"index":0,"delta":{"image":{"url":"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAA...","detail":"auto"}},"finish_reason":null}]}
+
+data: {"id":"chatcmpl-123","object":"chat.completion.chunk","created":1723323084,"model":"gemini/gemini-2.5-flash-image-preview","choices":[{"index":0,"delta":{},"finish_reason":"stop"}]}
+
+data: [DONE]
+```
+
+## Async Support
+
+```python showLineNumbers title="Async image generation"
+from litellm import acompletion
+import asyncio
+import os 
+
+os.environ["GEMINI_API_KEY"] = "your-api-key"
+
+async def generate_image():
+    response = await acompletion(
+        model="gemini/gemini-2.5-flash-image-preview",
+        messages=[
+            {"role": "user", "content": "Generate an image of a banana wearing a costume that says LiteLLM"}
+        ],
+    )
+    
+    print(response.choices[0].message.content)  # Text response
+    print(response.choices[0].message.image)    # Image data
+    
+    return response
+
+# Run the async function
+asyncio.run(generate_image())
+```
+
+## Supported Models
+
+| Provider | Model | 
+|----------|--------|
+| Google AI Studio | `gemini/gemini-2.5-flash-image-preview` |
+| Vertex AI | `vertex_ai/gemini-2.5-flash-image-preview` |
+
+## Spec 
+
+The `image` field in the response follows this structure:
+
+```python
+"image": {
+    "url": "data:image/png;base64,<base64_encoded_image>",
+    "detail": "auto"
+}
+```
+
+- `url` - str: Base64 encoded image data in data URI format
+- `detail` - str: Image detail level (always "auto" for generated images)
+
+The image is returned as a base64-encoded data URI that can be directly used in HTML `<img>` tags or saved to a file.

docs/my-website/docs/exception_mapping.md

@@ -12,6 +12,7 @@ All exceptions can be imported from `litellm` - e.g. `from litellm import BadReq
 | 400 | UnsupportedParamsError | litellm.BadRequestError | Raised when unsupported params are passed |
 | 400         | ContextWindowExceededError| litellm.BadRequestError | Special error type for context window exceeded error messages - enables context window fallbacks |
 | 400         | ContentPolicyViolationError| litellm.BadRequestError | Special error type for content policy violation error messages - enables content policy fallbacks |
+| 400         | ImageFetchError | litellm.BadRequestError | Raised when there are errors fetching or processing images |
 | 400 | InvalidRequestError | openai.BadRequestError | Deprecated error, use BadRequestError instead |
 | 401         | AuthenticationError      | openai.AuthenticationError |
 | 403         | PermissionDeniedError    | openai.PermissionDeniedError |

docs/my-website/docs/extras/gemini_img_migration.md

@@ -0,0 +1,205 @@
+# Gemini Image Generation Migration Guide
+
+## Who is impacted by this change?
+
+Anyone using the following models with /chat/completions:
+- `gemini/gemini-2.0-flash-exp-image-generation`
+- `vertex_ai/gemini-2.0-flash-exp-image-generation`
+
+## Key Change
+
+Gemini models now support image generation through chat completions. Images are returned in `response.choices[0].message.image` with base64 data URLs.
+
+## Before and After
+
+### Before
+```python
+from litellm import completion
+
+response = completion(
+    model="gemini/gemini-2.0-flash-exp-image-generation",
+    messages=[{"role": "user", "content": "Generate an image of a cat"}],
+    modalities=["image", "text"],
+)
+
+
+base_64_image_data = response.choices[0].message.content
+```
+
+### After  
+```python
+from litellm import completion
+
+response = completion(
+    model="gemini/gemini-2.0-flash-exp-image-generation",
+    messages=[{"role": "user", "content": "Generate an image of a cat"}],
+    modalities=["image", "text"],
+)
+
+# Image is now available in the response
+image_url = response.choices[0].message.image["url"]  # "data:image/png;base64,..."
+```
+
+### Why the change?
+
+Because the newer `gemini-2.5-flash-image-preview` model sends both text and image responses in the same response. This interface allows a developer to explicitly access the image or text components of the response. Before a developer would have needed to search through the message content to find the image generated by the model.
+
+## Usage
+
+### Using the Python SDK
+
+**Key Change:**
+```diff
+# Before
+-- base_64_image_data = response.choices[0].message.content
+
+# After
+++ image_url = response.choices[0].message.image["url"]
+```
+
+#### Basic Image Generation
+
+```python
+from litellm import completion
+import os
+
+# Set your API key
+os.environ["GEMINI_API_KEY"] = "your-api-key"
+
+# Generate an image
+response = completion(
+    model="gemini/gemini-2.0-flash-exp-image-generation",
+    messages=[{"role": "user", "content": "Generate an image of a cat"}],
+    modalities=["image", "text"],
+)
+
+# Access the generated image
+print(response.choices[0].message.content)  # Text response (if any)
+print(response.choices[0].message.image)    # Image data
+```
+
+#### Response Format
+
+The image is returned in the `message.image` field:
+
+```python
+{
+    "url": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAA...",
+    "detail": "auto"
+}
+```
+
+### Using the LiteLLM Proxy Server
+
+**Key Change:**
+```diff
+# Before
+-- "content": "base64-image-data..."
+
+# After  
+++ "image": {
+++   "url": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAA...",
+++   "detail": "auto"
+++ }
+```
+
+#### Configuration Setup
+
+1. **Configure your models in `config.yaml`:**
+
+```yaml
+model_list:
+  - model_name: gemini-image-gen
+    litellm_params:
+      model: gemini/gemini-2.0-flash-exp-image-generation
+      api_key: os.environ/GEMINI_API_KEY
+  - model_name: vertex-image-gen  
+    litellm_params:
+      model: vertex_ai/gemini-2.5-flash-image-preview
+      vertex_project: your-project-id
+      vertex_location: us-central1
+
+general_settings:
+  master_key: sk-1234  # Your proxy API key
+```
+
+2. **Start the proxy server:**
+
+```bash
+litellm --config /path/to/config.yaml
+
+# RUNNING on http://0.0.0.0:4000
+```
+
+#### Making Requests
+
+**Using OpenAI SDK:**
+
+```python
+from openai import OpenAI
+
+# Point to your proxy server
+client = OpenAI(
+    api_key="sk-1234",  # Your proxy API key
+    base_url="http://0.0.0.0:4000"
+)
+
+response = client.chat.completions.create(
+    model="gemini-image-gen",
+    messages=[{"role": "user", "content": "Generate an image of a cat"}],
+    extra_body={"modalities": ["image", "text"]}
+)
+
+# Access the generated image
+print(response.choices[0].message.content)  # Text response (if any)
+print(response.choices[0].message.image)    # Image data
+```
+
+**Using curl:**
+
+```bash
+curl -X POST 'http://0.0.0.0:4000/v1/chat/completions' \
+-H 'Content-Type: application/json' \
+-H 'Authorization: Bearer sk-1234' \
+-d '{
+  "model": "gemini-image-gen",
+  "messages": [
+    {
+      "role": "user",
+      "content": "Generate an image of a cat"
+    }
+  ],
+  "modalities": ["image", "text"]
+}'
+```
+
+**Response format from proxy:**
+
+```json
+{
+  "id": "chatcmpl-123",
+  "object": "chat.completion",
+  "created": 1704089632,
+  "model": "gemini-image-gen",
+  "choices": [
+    {
+      "index": 0,
+      "message": {
+        "role": "assistant",
+        "content": "Here's an image of a cat for you!",
+        "image": {
+          "url": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAA...",
+          "detail": "auto"
+        }
+      },
+      "finish_reason": "stop"
+    }
+  ],
+  "usage": {
+    "prompt_tokens": 10,
+    "completion_tokens": 8,
+    "total_tokens": 18
+  }
+}
+```
+

docs/my-website/docs/image_edits.md

@@ -4,7 +4,7 @@ import TabItem from '@theme/TabItem';
 
 # /images/edits
 
-LiteLLM provides image editing functionality that maps to OpenAI's `/images/edits` API endpoint.
+LiteLLM provides image editing functionality that maps to OpenAI's `/images/edits` API endpoint. Now supports both single and multiple image editing.
 
 | Feature | Supported | Notes |
 |---------|-----------|--------|
@@ -13,7 +13,7 @@ LiteLLM provides image editing functionality that maps to OpenAI's `/images/edit
 | End-user Tracking | ✅ | |
 | Fallbacks | ✅ | Works between supported models |
 | Loadbalancing | ✅ | Works between supported models |
-| Supported operations | Create image edits | |
+| Supported operations | Create image edits | Single and multiple images supported |
 | Supported LiteLLM SDK Versions | 1.63.8+ | |
 | Supported LiteLLM Proxy Versions | 1.71.1+ | |
 | Supported LLM providers | **OpenAI** | Currently only `openai` is supported |
@@ -41,6 +41,26 @@ response = litellm.image_edit(
 print(response)
 ```
 
+#### Multiple Images Edit
+```python showLineNumbers title="OpenAI Multiple Images Edit"
+import litellm
+
+# Edit multiple images with a prompt
+response = litellm.image_edit(
+    model="gpt-image-1",
+    image=[
+        open("image1.png", "rb"),
+        open("image2.png", "rb"),
+        open("image3.png", "rb")
+    ],
+    prompt="Apply vintage filter to all images",
+    n=1,
+    size="1024x1024"
+)
+
+print(response)
+```
+
 #### Image Edit with Mask
 ```python showLineNumbers title="OpenAI Image Edit with Mask"
 import litellm
@@ -80,6 +100,30 @@ response = asyncio.run(edit_image())
 print(response)
 ```
 
+#### Async Multiple Images Edit
+```python showLineNumbers title="Async OpenAI Multiple Images Edit"
+import litellm
+import asyncio
+
+async def edit_multiple_images():
+    response = await litellm.aimage_edit(
+        model="gpt-image-1",
+        image=[
+            open("portrait1.png", "rb"),
+            open("portrait2.png", "rb")
+        ],
+        prompt="Add professional lighting to the portraits",
+        n=1,
+        size="1024x1024",
+        response_format="url"
+    )
+    return response
+
+# Run the async function
+response = asyncio.run(edit_multiple_images())
+print(response)
+```
+
 #### Image Edit with Custom Parameters
 ```python showLineNumbers title="OpenAI Image Edit with Custom Parameters"
 import litellm
@@ -163,6 +207,20 @@ curl -X POST "http://localhost:4000/v1/images/edits" \
   -F "response_format=url"
 ```
 
+#### cURL Multiple Images Example
+```bash showLineNumbers title="cURL Multiple Images Edit Request"
+curl -X POST "http://localhost:4000/v1/images/edits" \
+  -H "Authorization: Bearer your-api-key" \
+  -F "model=gpt-image-1" \
+  -F "image=@image1.png" \
+  -F "image=@image2.png" \
+  -F "image=@image3.png" \
+  -F "prompt=Apply artistic filter to all images" \
+  -F "n=1" \
+  -F "size=1024x1024" \
+  -F "response_format=url"
+```
+
 </TabItem>
 </Tabs>
 

docs/my-website/docs/index.md

@@ -226,6 +226,23 @@ response = completion(
 
 </TabItem>
 
+<TabItem value="vercel" label="Vercel AI Gateway">
+
+```python
+from litellm import completion
+import os
+
+## set ENV variables. Visit https://vercel.com/docs/ai-gateway#using-the-ai-gateway-with-an-api-key for insturctions on obtaining a key
+os.environ["VERCEL_AI_GATEWAY_API_KEY"] = "your-vercel-api-key"
+
+response = completion(
+  model="vercel_ai_gateway/openai/gpt-4o",
+  messages=[{ "content": "Hello, how are you?","role": "user"}]
+)
+```
+
+</TabItem>
+
 </Tabs>
 
 ### Response Format (OpenAI Format)
@@ -446,6 +463,24 @@ response = completion(
 
 </TabItem>
 
+<TabItem value="vercel" label="Vercel AI Gateway">
+
+```python
+from litellm import completion
+import os
+
+## set ENV variables. Visit https://vercel.com/docs/ai-gateway#using-the-ai-gateway-with-an-api-key for insturctions on obtaining a key
+os.environ["VERCEL_AI_GATEWAY_API_KEY"] = "your-vercel-api-key"
+
+response = completion(
+  model="vercel_ai_gateway/openai/gpt-4o",
+  messages = [{ "content": "Hello, how are you?","role": "user"}],
+  stream=True,
+)
+```
+
+</TabItem>
+
 </Tabs>
 
 ### Streaming Response Format (OpenAI Format)

docs/my-website/docs/load_test_rpm.md

@@ -53,8 +53,8 @@ model_list = [
     },
 ]
 
-router_1 = Router(model_list=model_list, num_retries=0, enable_pre_call_checks=True, routing_strategy="usage-based-routing-v2", redis_host=os.getenv("REDIS_HOST"), redis_port=os.getenv("REDIS_PORT"), redis_password=os.getenv("REDIS_PASSWORD"))
-router_2 = Router(model_list=model_list, num_retries=0, routing_strategy="usage-based-routing-v2", enable_pre_call_checks=True, redis_host=os.getenv("REDIS_HOST"), redis_port=os.getenv("REDIS_PORT"), redis_password=os.getenv("REDIS_PASSWORD"))
+router_1 = Router(model_list=model_list, num_retries=0, enable_pre_call_checks=True, routing_strategy="simple-shuffle", redis_host=os.getenv("REDIS_HOST"), redis_port=os.getenv("REDIS_PORT"), redis_password=os.getenv("REDIS_PASSWORD"))
+router_2 = Router(model_list=model_list, num_retries=0, routing_strategy="simple-shuffle", enable_pre_call_checks=True, redis_host=os.getenv("REDIS_HOST"), redis_port=os.getenv("REDIS_PORT"), redis_password=os.getenv("REDIS_PASSWORD"))
 
 
 
@@ -142,7 +142,7 @@ router_settings:
   redis_host: os.environ/REDIS_HOST ## 👈 IMPORTANT! Setup the proxy w/ redis
   redis_password: os.environ/REDIS_PASSWORD
   redis_port: os.environ/REDIS_PORT
-  routing_strategy: usage-based-routing-v2
+  routing_strategy: simple-shuffle # recommended for best performance
 ```
 
 ### 2. Start proxy 2 instances

docs/my-website/docs/mcp.md

@@ -40,7 +40,28 @@ LiteLLM supports the following MCP transports:
   style={{width: '80%', display: 'block', margin: '0'}}
 />
 
-### Adding a stdio MCP Server
+<br/>
+<br/>
+
+### Add HTTP MCP Server
+
+This video walks through adding and using an HTTP MCP server on LiteLLM UI and using it in Cursor IDE.
+
+<iframe width="840" height="500" src="https://www.loom.com/embed/e2aebce78e8d46beafeb4bacdde31f14" frameborder="0" webkitallowfullscreen mozallowfullscreen allowfullscreen></iframe>
+
+<br/>
+<br/>
+
+### Add SSE MCP Server
+
+This video walks through adding and using an SSE MCP server on LiteLLM UI and using it in Cursor IDE.
+
+<iframe width="840" height="500" src="https://www.loom.com/embed/07e04e27f5e74475b9cf8ef8247d2c3e" frameborder="0" webkitallowfullscreen mozallowfullscreen allowfullscreen></iframe>
+
+<br/>
+<br/>
+
+### Add STDIO MCP Server
 
 For stdio MCP servers, select "Standard Input/Output (stdio)" as the transport type and provide the stdio configuration in JSON format:
 

docs/my-website/docs/observability/braintrust.md

@@ -71,6 +71,10 @@ curl -X POST 'http://0.0.0.0:4000/chat/completions' \
 
 It is recommended that you include the `project_id` or `project_name` to ensure your traces are being written out to the correct Braintrust project.
 
+### Custom Span Names
+
+You can customize the span name in Braintrust logging by passing `span_name` in the metadata. By default, the span name is set to "Chat Completion".
+
 <Tabs>
 <TabItem value="sdk" label="SDK">
 
@@ -84,7 +88,9 @@ response = litellm.completion(
     "project_id": "1234",
     # passing project_name will try to find a project with that name, or create one if it doesn't exist
     # if both project_id and project_name are passed, project_id will be used
-    # "project_name": "my-special-project"
+    # "project_name": "my-special-project",
+    # custom span name for this operation (default: "Chat Completion")
+    "span_name": "User Greeting Handler"
   }
 )
 ```
@@ -99,6 +105,7 @@ response = litellm.completion(
   ],
   metadata={
     "project_id": "1234",
+    "span_name": "Custom Operation",
     "item1": "an item",
     "item2": "another item"
   }
@@ -121,7 +128,8 @@ curl -X POST 'http://0.0.0.0:4000/chat/completions' \
         { "role": "user", "content": "What time is it now? Use your tool"}
     ],
     "metadata": {
-        "project_id": "my-special-project"
+        "project_id": "my-special-project",
+        "span_name": "Tool Usage Request"
     }
 }'
 ```
@@ -146,7 +154,8 @@ response = client.chat.completions.create(
     ],
     extra_body={ # pass in any provider-specific param, if not supported by openai, https://docs.litellm.ai/docs/completion/input#provider-specific-params
         "metadata": { # 👈 use for logging additional params (e.g. to braintrust)
-            "project_id": "my-special-project"
+            "project_id": "my-special-project",
+            "span_name": "Poetry Generation"
         }
     }
 )
@@ -168,3 +177,7 @@ Here's everything you can pass in metadata for a braintrust request
 `braintrust_*` - If you are adding metadata from _proxy request headers_, any metadata field starting with `braintrust_` will be passed as metadata to the logging request. If you are using the SDK, just pass your metadata like normal (e.g., `metadata={"project_name": "my-test-project", "item1": "an item", "item2": "another item"}`)
 
 `project_id` - Set the project id for a braintrust call. Default is `litellm`.
+
+`project_name` - Set the project name for a braintrust call. Will try to find a project with that name, or create one if it doesn't exist. If both `project_id` and `project_name` are passed, `project_id` will be used.
+
+`span_name` - Set a custom span name for the operation. Default is `"Chat Completion"`. Use this to provide more descriptive names for different types of operations in your application (e.g., "User Query", "Document Summary", "Code Generation").

docs/my-website/docs/observability/langfuse_otel_integration.md

@@ -35,14 +35,14 @@ The Langfuse OpenTelemetry integration allows you to send LiteLLM traces and obs
 |----------|----------|-------------|---------|
 | `LANGFUSE_PUBLIC_KEY` | Yes | Your Langfuse public key | `pk-lf-...` |
 | `LANGFUSE_SECRET_KEY` | Yes | Your Langfuse secret key | `sk-lf-...` |
-| `LANGFUSE_HOST` | No | Langfuse host URL | `https://us.cloud.langfuse.com` (default) |
+| `LANGFUSE_OTEL_HOST` | No | OTEL endpoint host | `https://otel.my-langfuse.com` |
 
 ### Endpoint Resolution
 
-The integration automatically constructs the OTEL endpoint from the `LANGFUSE_HOST`:
+The integration automatically constructs the OTEL endpoint from `LANGFUSE_OTEL_HOST`
 - **Default (US)**: `https://us.cloud.langfuse.com/api/public/otel`
 - **EU Region**: `https://cloud.langfuse.com/api/public/otel`
-- **Self-hosted**: `{LANGFUSE_HOST}/api/public/otel`
+- **Self-hosted**: `{LANGFUSE_OTEL_HOST}/api/public/otel`
 
 ## Usage
 
@@ -77,11 +77,11 @@ os.environ["LANGFUSE_PUBLIC_KEY"] = "pk-lf-..."
 os.environ["LANGFUSE_SECRET_KEY"] = "sk-lf-..."
 
 # Use EU region
-os.environ["LANGFUSE_HOST"] = "https://cloud.langfuse.com"  # EU region
-# os.environ["LANGFUSE_HOST"] = "https://us.cloud.langfuse.com"  # US region (default)
+os.environ["LANGFUSE_OTEL_HOST"] = "https://cloud.langfuse.com"  # EU region
+# os.environ["LANGFUSE_OTEL_HOST"] = "https://otel.my-langfuse.company.com"  # custom OTEL endpoint
 
 # Or use self-hosted instance
-# os.environ["LANGFUSE_HOST"] = "https://my-langfuse.company.com"
+# os.environ["LANGFUSE_OTEL_HOST"] = "https://my-langfuse.company.com"
 
 litellm.callbacks = ["langfuse_otel"]
 ```
@@ -98,14 +98,16 @@ import litellm
 # Get keys for your project from the project settings page: https://cloud.langfuse.com
 os.environ["LANGFUSE_PUBLIC_KEY"] = "pk-lf-..." 
 os.environ["LANGFUSE_SECRET_KEY"] = "sk-lf-..." 
-os.environ["LANGFUSE_HOST"] = "https://cloud.langfuse.com" # EU region
-# os.environ["LANGFUSE_HOST"] = "https://us.cloud.langfuse.com" # US region
+os.environ["LANGFUSE_OTEL_HOST"] = "https://cloud.langfuse.com" # EU region
+# os.environ["LANGFUSE_OTEL_HOST"] = "https://us.cloud.langfuse.com" # US region
+# os.environ["LANGFUSE_OTEL_HOST"] = "https://otel.my-langfuse.company.com" # custom OTEL endpoint
 
 LANGFUSE_AUTH = base64.b64encode(
     f"{os.environ.get('LANGFUSE_PUBLIC_KEY')}:{os.environ.get('LANGFUSE_SECRET_KEY')}".encode()
 ).decode()
 
-os.environ["OTEL_EXPORTER_OTLP_ENDPOINT"] = os.environ.get("LANGFUSE_HOST") + "/api/public/otel"
+host = os.environ.get("LANGFUSE_OTEL_HOST")
+os.environ["OTEL_EXPORTER_OTLP_ENDPOINT"] = host + "/api/public/otel"
 os.environ["OTEL_EXPORTER_OTLP_HEADERS"] = f"Authorization=Basic {LANGFUSE_AUTH}"
 
 litellm.callbacks = ["langfuse_otel"]
@@ -120,7 +122,8 @@ Add the integration to your proxy configuration:
 ```bash
 export LANGFUSE_PUBLIC_KEY="pk-lf-..."
 export LANGFUSE_SECRET_KEY="sk-lf-..."
-export LANGFUSE_HOST="https://us.cloud.langfuse.com"  # Default US region
+export LANGFUSE_OTEL_HOST="https://us.cloud.langfuse.com"  # Default US region
+# export LANGFUSE_OTEL_HOST="https://otel.my-langfuse.company.com"  # custom OTEL endpoint
 ```
 
 2. Setup config.yaml

docs/my-website/docs/providers/anthropic.md

@@ -55,8 +55,29 @@ import os
 
 os.environ["ANTHROPIC_API_KEY"] = "your-api-key"
 # os.environ["ANTHROPIC_API_BASE"] = "" # [OPTIONAL] or 'ANTHROPIC_BASE_URL'
+# os.environ["LITELLM_ANTHROPIC_DISABLE_URL_SUFFIX"] = "true" # [OPTIONAL] Disable automatic URL suffix appending
 ```
 
+### Custom API Base
+
+When using a custom API base for Anthropic (e.g., a proxy or custom endpoint), LiteLLM automatically appends the appropriate suffix (`/v1/messages` or `/v1/complete`) to your base URL.
+
+If your custom endpoint already includes the full path or doesn't follow Anthropic's standard URL structure, you can disable this automatic suffix appending:
+
+```python
+import os
+
+os.environ["ANTHROPIC_API_BASE"] = "https://my-custom-endpoint.com/custom/path"
+os.environ["LITELLM_ANTHROPIC_DISABLE_URL_SUFFIX"] = "true"  # Prevents automatic suffix
+```
+
+Without `LITELLM_ANTHROPIC_DISABLE_URL_SUFFIX`:
+- Base URL `https://my-proxy.com` → `https://my-proxy.com/v1/messages`
+- Base URL `https://my-proxy.com/api` → `https://my-proxy.com/api/v1/messages`
+
+With `LITELLM_ANTHROPIC_DISABLE_URL_SUFFIX=true`:
+- Base URL `https://my-proxy.com/custom/path` → `https://my-proxy.com/custom/path` (unchanged)
+
 ## Usage
 
 ```python

docs/my-website/docs/providers/cometapi.md

@@ -0,0 +1,144 @@
+# CometAPI
+LiteLLM supports all AI models from [CometAPI](https://www.cometapi.com/). CometAPI provides access to 500+ AI models through a unified API interface, including cutting-edge models like GPT-5, Claude Opus 4.1, and various other state-of-the-art language models.
+
+## Authentication
+
+To use CometAPI models, you need to obtain an API key from [CometAPI Token Console](https://api.cometapi.com/console/token). CometAPI offers free tokens for new users - you can get your free API key instantly by registering.
+
+## Usage
+
+Set your CometAPI key as an environment variable and use the completion function:
+
+```python
+import os
+from litellm import completion
+
+# Set API key
+os.environ["COMETAPI_KEY"] = "your_comet_api_key_here"
+
+# Define messages
+messages = [{"content": "Hello, how are you?", "role": "user"}]
+
+# Method 1: Using environment variable (recommended)
+response = completion(
+    model="cometapi/gpt-5", 
+    messages=messages
+)
+
+print(response.choices[0].message.content)
+```
+
+### Alternative Usage - Explicit API Key
+
+You can also pass the API key explicitly:
+
+```python
+import os
+from litellm import completion
+
+# Define messages
+messages = [{"content": "Hello, how are you?", "role": "user"}]
+
+# Method 2: Explicitly passing API key
+response = completion(
+    model="cometapi/gpt-4o", 
+    messages=messages, 
+    api_key="your_comet_api_key_here"
+)
+
+print(response.choices[0].message.content)
+```
+
+## Usage - Streaming
+
+Just set `stream=True` when calling completion:
+
+```python
+import os
+from litellm import completion
+
+os.environ["COMETAPI_KEY"] = "your_comet_api_key_here"
+
+messages = [{"content": "Hello, how are you?", "role": "user"}]
+
+response = completion(
+    model="cometapi/gpt-5",
+    messages=messages,
+    stream=True
+)
+
+for chunk in response:
+    print(chunk.choices[0].delta.content or "", end="")
+```
+
+## Usage - Async Streaming
+
+For async streaming, use `acompletion`:
+
+```python
+from litellm import acompletion
+import asyncio, os, traceback
+
+async def completion_call():
+    try:
+        os.environ["COMETAPI_KEY"] = "your_comet_api_key_here"
+        
+        print("test acompletion + streaming")
+        response = await acompletion(
+            model="cometapi/chatgpt-4o-latest", 
+            messages=[{"content": "Hello, how are you?", "role": "user"}], 
+            stream=True
+        )
+        print(f"response: {response}")
+        async for chunk in response:
+            print(chunk)
+    except:
+        print(f"error occurred: {traceback.format_exc()}")
+        pass
+
+# Run the async function
+await completion_call()
+```
+
+## CometAPI Models
+
+CometAPI offers access to 500+ AI models through a unified API. Some popular models include:
+
+| Model Name | Function Call |
+|------------|---------------|
+| cometapi/gpt-5 | `completion('cometapi/gpt-5', messages)` |
+| cometapi/gpt-5-mini | `completion('cometapi/gpt-5-mini', messages)` |
+| cometapi/gpt-5-nano | `completion('cometapi/gpt-5-nano', messages)` |
+| cometapi/gpt-oss-20b | `completion('cometapi/gpt-oss-20b', messages)` |
+| cometapi/gpt-oss-120b | `completion('cometapi/gpt-oss-120b', messages)` |
+| cometapi/chatgpt-4o-latest | `completion('cometapi/chatgpt-4o-latest', messages)` |
+
+For a complete list of available models, visit the [CometAPI Models page](https://www.cometapi.com/model/).
+
+## Environment Variables
+
+| Variable | Description | Required |
+|----------|-------------|----------|
+| `COMETAPI_KEY` | Your CometAPI API key | Yes |
+
+## Error Handling
+
+```python
+import os
+from litellm import completion
+
+try:
+    os.environ["COMETAPI_KEY"] = "your_comet_api_key_here"
+    
+    messages = [{"content": "Hello, how are you?", "role": "user"}]
+    
+    response = completion(
+        model="cometapi/gpt-5",
+        messages=messages
+    )
+    
+    print(response.choices[0].message.content)
+    
+except Exception as e:
+    print(f"Error: {e}")
+```

docs/my-website/docs/providers/datarobot.md

@@ -0,0 +1,43 @@
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+
+# DataRobot
+LiteLLM supports all models from [DataRobot](https://datarobot.com). Select `datarobot` as the provider to route your request through the `datarobot` OpenAI-compatible endpoint using the upstream [official OpenAI Python API library](https://github.com/openai/openai-python/blob/main/README.md).
+
+## Usage 
+
+### Environment variables
+```python
+import os
+from litellm import completion
+os.environ["DATAROBOT_API_KEY"] = ""
+os.environ["DATAROBOT_API_BASE"] = "" # [OPTIONAL] defaults to https://app.datarobot.com
+
+response = completion(
+            model="datarobot/openai/gpt-4o-mini",
+            messages=messages,
+        )
+
+
+### Completion
+```python
+import litellm
+import os
+
+response = litellm.completion(
+    model="datarobot/openai/gpt-4o-mini",   # add `datarobot/` prefix to model so litellm knows to route through DataRobot
+    messages=[
+                {
+                    "role": "user",
+                    "content": "Hey, how's it going?",
+                }
+    ],
+)
+print(response)
+```
+
+## DataRobot completion models
+
+🚨 LiteLLM supports _all_ DataRobot LLM gateway models. To get a list for your installation and user account, send the following CURL command:
+`curl -X GET -H "Authorization: Bearer $DATAROBOT_API_TOKEN" "$DATAROBOT_ENDPOINT/genai/llmgw/catalog/" | jq | grep 'model":'DATAROBOT_ENDPOINT/genai/llmgw/catalog/`
+

docs/my-website/docs/providers/google_ai_studio/image_gen.md

@@ -42,7 +42,7 @@ os.environ["GEMINI_API_KEY"] = "your-api-key-here"
 
 # Generate a single image
 response = litellm.image_generation(
-    model="gemini/imagen-4.0-generate-preview-06-06",
+    model="gemini/imagen-4.0-generate-001",
     prompt="A cute baby sea otter swimming in crystal clear water"
 )
 
@@ -64,7 +64,7 @@ async def generate_image():
     
     # Generate image asynchronously
     response = await litellm.aimage_generation(
-        model="gemini/imagen-4.0-generate-preview-06-06",
+        model="gemini/imagen-4.0-generate-001",
         prompt="A beautiful sunset over mountains with vibrant colors",
         n=1,
     )
@@ -89,7 +89,7 @@ os.environ["GEMINI_API_KEY"] = "your-api-key-here"
 
 # Generate image with additional parameters
 response = litellm.image_generation(
-    model="gemini/imagen-4.0-generate-preview-06-06",
+    model="gemini/imagen-4.0-generate-001",
     prompt="A futuristic cityscape at night with neon lights",
     n=1,
     size="1024x1024",
@@ -112,7 +112,7 @@ for image in response.data:
 model_list:
   - model_name: google-imagen
     litellm_params:
-      model: gemini/imagen-4.0-generate-preview-06-06
+      model: gemini/imagen-4.0-generate-001
       api_key: os.environ/GEMINI_API_KEY
   model_info:
     mode: image_generation
@@ -198,7 +198,7 @@ Google AI Studio Image Generation supports the following OpenAI-compatible param
 | Parameter | Type | Description | Default | Example |
 |-----------|------|-------------|---------|---------|
 | `prompt` | string | Text description of the image to generate | Required | `"A sunset over the ocean"` |
-| `model` | string | The model to use for generation | Required | `"gemini/imagen-4.0-generate-preview-06-06"` |
+| `model` | string | The model to use for generation | Required | `"gemini/imagen-4.0-generate-001"` |
 | `n` | integer | Number of images to generate (1-4) | `1` | `2` |
 | `size` | string | Image dimensions | `"1024x1024"` | `"512x512"`, `"1024x1024"` |
 

docs/my-website/docs/providers/oci.md

@@ -44,7 +44,11 @@ response = completion(
     oci_user=<your_oci_user>,
     oci_fingerprint=<your_oci_fingerprint>,
     oci_tenancy=<your_oci_tenancy>,
+    # Provide either the private key string OR the path to the key file:
+    # Option 1: pass the private key as a string
     oci_key=<string_with_content_of_oci_key>,
+    # Option 2: pass the private key file path
+    # oci_key_file="<path/to/oci_key.pem>",
     oci_compartment_id=<oci_compartment_id>,
 )
 print(response)
@@ -67,7 +71,11 @@ response = completion(
     oci_user=<your_oci_user>,
     oci_fingerprint=<your_oci_fingerprint>,
     oci_tenancy=<your_oci_tenancy>,
+    # Provide either the private key string OR the path to the key file:
+    # Option 1: pass the private key as a string
     oci_key=<string_with_content_of_oci_key>,
+    # Option 2: pass the private key file path
+    # oci_key_file="<path/to/oci_key.pem>",
     oci_compartment_id=<oci_compartment_id>,
 )
 for chunk in response:

docs/my-website/docs/providers/vercel_ai_gateway.md

@@ -0,0 +1,219 @@
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+
+# Vercel AI Gateway
+
+## Overview
+
+| Property | Details |
+|-------|-------|
+| Description | Vercel AI Gateway provides a unified interface to access multiple AI providers through a single endpoint, with built-in caching, rate limiting, and analytics. |
+| Provider Route on LiteLLM | `vercel_ai_gateway/` |
+| Link to Provider Doc | [Vercel AI Gateway Documentation ↗](https://vercel.com/docs/ai-gateway) |
+| Base URL | `https://ai-gateway.vercel.sh/v1` |
+| Supported Operations | `/chat/completions`, `/models` |
+
+<br />
+<br />
+
+https://vercel.com/docs/ai-gateway
+
+**We support ALL models available through Vercel AI Gateway, just set `vercel_ai_gateway/` as a prefix when sending completion requests**
+
+## Required Variables
+
+```python showLineNumbers title="Environment Variables"
+os.environ["VERCEL_AI_GATEWAY_API_KEY"] = ""  # your Vercel AI Gateway API key
+# OR
+os.environ["VERCEL_OIDC_TOKEN"] = ""  # your Vercel OIDC token for authentication
+```
+
+## Optional Variables
+
+```python showLineNumbers title="Environment Variables"
+os.environ["VERCEL_SITE_URL"] = ""  # your site url
+# OR
+os.environ["VERCEL_APP_NAME"] = ""  # your app name
+```
+
+Note: see the [Vercel AI Gateway docs](https://vercel.com/docs/ai-gateway#using-the-ai-gateway-with-an-api-key) for instructions on obtaining a key.
+
+## Usage - LiteLLM Python SDK
+
+### Non-streaming
+
+```python showLineNumbers title="Vercel AI Gateway Non-streaming Completion"
+import os
+import litellm
+from litellm import completion
+
+os.environ["VERCEL_AI_GATEWAY_API_KEY"] = "your-api-key"
+
+messages = [{"content": "Hello, how are you?", "role": "user"}]
+
+# Vercel AI Gateway call
+response = completion(
+    model="vercel_ai_gateway/openai/gpt-4o", 
+    messages=messages
+)
+
+print(response)
+```
+
+### Streaming
+
+```python showLineNumbers title="Vercel AI Gateway Streaming Completion"
+import os
+import litellm
+from litellm import completion
+
+os.environ["VERCEL_AI_GATEWAY_API_KEY"] = "your-api-key"
+
+messages = [{"content": "Hello, how are you?", "role": "user"}]
+
+# Vercel AI Gateway call with streaming
+response = completion(
+    model="vercel_ai_gateway/openai/gpt-4o", 
+    messages=messages,
+    stream=True
+)
+
+for chunk in response:
+    print(chunk)
+```
+
+## Usage - LiteLLM Proxy
+
+Add the following to your LiteLLM Proxy configuration file:
+
+```yaml showLineNumbers title="config.yaml"
+model_list:
+  - model_name: gpt-4o-gateway
+    litellm_params:
+      model: vercel_ai_gateway/openai/gpt-4o
+      api_key: os.environ/VERCEL_AI_GATEWAY_API_KEY
+
+  - model_name: claude-4-sonnet-gateway
+    litellm_params:
+      model: vercel_ai_gateway/anthropic/claude-4-sonnet
+      api_key: os.environ/VERCEL_AI_GATEWAY_API_KEY
+```
+
+Start your LiteLLM Proxy server:
+
+```bash showLineNumbers title="Start LiteLLM Proxy"
+litellm --config config.yaml
+
+# RUNNING on http://0.0.0.0:4000
+```
+
+<Tabs>
+<TabItem value="openai-sdk" label="OpenAI SDK">
+
+```python showLineNumbers title="Vercel AI Gateway via Proxy - Non-streaming"
+from openai import OpenAI
+
+# Initialize client with your proxy URL
+client = OpenAI(
+    base_url="http://localhost:4000",  # Your proxy URL
+    api_key="your-proxy-api-key"       # Your proxy API key
+)
+
+# Non-streaming response
+response = client.chat.completions.create(
+    model="gpt-4o-gateway",
+    messages=[{"role": "user", "content": "Hello, how are you?"}]
+)
+
+print(response.choices[0].message.content)
+```
+
+```python showLineNumbers title="Vercel AI Gateway via Proxy - Streaming"
+from openai import OpenAI
+
+# Initialize client with your proxy URL
+client = OpenAI(
+    base_url="http://localhost:4000",  # Your proxy URL
+    api_key="your-proxy-api-key"       # Your proxy API key
+)
+
+# Streaming response
+response = client.chat.completions.create(
+    model="gpt-4o-gateway",
+    messages=[{"role": "user", "content": "Hello, how are you?"}],
+    stream=True
+)
+
+for chunk in response:
+    if chunk.choices[0].delta.content is not None:
+        print(chunk.choices[0].delta.content, end="")
+```
+
+</TabItem>
+
+<TabItem value="litellm-sdk" label="LiteLLM SDK">
+
+```python showLineNumbers title="Vercel AI Gateway via Proxy - LiteLLM SDK"
+import litellm
+
+# Configure LiteLLM to use your proxy
+response = litellm.completion(
+    model="litellm_proxy/gpt-4o-gateway",
+    messages=[{"role": "user", "content": "Hello, how are you?"}],
+    api_base="http://localhost:4000",
+    api_key="your-proxy-api-key"
+)
+
+print(response.choices[0].message.content)
+```
+
+```python showLineNumbers title="Vercel AI Gateway via Proxy - LiteLLM SDK Streaming"
+import litellm
+
+# Configure LiteLLM to use your proxy with streaming
+response = litellm.completion(
+    model="litellm_proxy/gpt-4o-gateway",
+    messages=[{"role": "user", "content": "Hello, how are you?"}],
+    api_base="http://localhost:4000",
+    api_key="your-proxy-api-key",
+    stream=True
+)
+
+for chunk in response:
+    if hasattr(chunk.choices[0], 'delta') and chunk.choices[0].delta.content is not None:
+        print(chunk.choices[0].delta.content, end="")
+```
+
+</TabItem>
+
+<TabItem value="curl" label="cURL">
+
+```bash showLineNumbers title="Vercel AI Gateway via Proxy - cURL"
+curl http://localhost:4000/v1/chat/completions \
+  -H "Content-Type: application/json" \
+  -H "Authorization: Bearer your-proxy-api-key" \
+  -d '{
+    "model": "gpt-4o-gateway",
+    "messages": [{"role": "user", "content": "Hello, how are you?"}]
+  }'
+```
+
+```bash showLineNumbers title="Vercel AI Gateway via Proxy - cURL Streaming"
+curl http://localhost:4000/v1/chat/completions \
+  -H "Content-Type: application/json" \
+  -H "Authorization: Bearer your-proxy-api-key" \
+  -d '{
+    "model": "gpt-4o-gateway",
+    "messages": [{"role": "user", "content": "Hello, how are you?"}],
+    "stream": true
+  }'
+```
+
+</TabItem>
+</Tabs>
+
+For more detailed information on using the LiteLLM Proxy, see the [LiteLLM Proxy documentation](../providers/litellm_proxy).
+
+## Additional Resources
+
+- [Vercel AI Gateway Documentation](https://vercel.com/docs/ai-gateway)

docs/my-website/docs/providers/vertex_image.md

@@ -18,7 +18,7 @@ import litellm
 # Generate a single image
 response = await litellm.aimage_generation(
     prompt="An olympic size swimming pool with crystal clear water and modern architecture",
-    model="vertex_ai/imagen-4.0-generate-preview-06-06",
+    model="vertex_ai/imagen-4.0-generate-001",
     vertex_ai_project="your-project-id",
     vertex_ai_location="us-central1",
 )
@@ -34,7 +34,7 @@ print(response.data[0].url)
 model_list:
   - model_name: vertex-imagen
     litellm_params:
-      model: vertex_ai/imagen-4.0-generate-preview-06-06
+      model: vertex_ai/imagen-4.0-generate-001
       vertex_ai_project: "your-project-id"
       vertex_ai_location: "us-central1"
       vertex_ai_credentials: "path/to/service-account.json"  # Optional if using environment auth

docs/my-website/docs/proxy/config_settings.md

@@ -236,7 +236,7 @@ Most values can also be set via `litellm_settings`. If you see overlapping value
 
 ```yaml
 router_settings:
-  routing_strategy: usage-based-routing-v2 # Literal["simple-shuffle", "least-busy", "usage-based-routing","latency-based-routing"], default="simple-shuffle"
+  routing_strategy: simple-shuffle # Literal["simple-shuffle", "least-busy", "usage-based-routing","latency-based-routing"], default="simple-shuffle" - RECOMMENDED for best performance
   redis_host: <your-redis-host>           # string
   redis_password: <your-redis-password>   # string
   redis_port: <your-redis-port>           # string
@@ -431,6 +431,7 @@ router_settings:
 | DEFAULT_MOCK_RESPONSE_COMPLETION_TOKEN_COUNT | Default token count for mock response completions. Default is 20
 | DEFAULT_MOCK_RESPONSE_PROMPT_TOKEN_COUNT | Default token count for mock response prompts. Default is 10
 | DEFAULT_MODEL_CREATED_AT_TIME | Default creation timestamp for models. Default is 1677610602
+| DEFAULT_NUM_WORKERS_LITELLM_PROXY | Default number of workers for LiteLLM proxy. Default is 4. **We strongly recommend setting NUM Workers to Number of vCPUs available**
 | DEFAULT_PROMPT_INJECTION_SIMILARITY_THRESHOLD | Default threshold for prompt injection similarity. Default is 0.7
 | DEFAULT_POLLING_INTERVAL | Default polling interval for schedulers in seconds. Default is 0.03
 | DEFAULT_REASONING_EFFORT_DISABLE_THINKING_BUDGET | Default reasoning effort disable thinking budget. Default is 0
@@ -558,6 +559,7 @@ router_settings:
 | LITERAL_API_KEY | API key for Literal integration
 | LITERAL_API_URL | API URL for Literal service
 | LITERAL_BATCH_SIZE | Batch size for Literal operations
+| LITELLM_ANTHROPIC_DISABLE_URL_SUFFIX | Disable automatic URL suffix appending for Anthropic API base URLs. When set to `true`, prevents LiteLLM from automatically adding `/v1/messages` or `/v1/complete` to custom Anthropic API endpoints
 | LITELLM_DONT_SHOW_FEEDBACK_BOX | Flag to hide feedback box in LiteLLM UI
 | LITELLM_DROP_PARAMS | Parameters to drop in LiteLLM requests
 | LITELLM_MODIFY_PARAMS | Parameters to modify in LiteLLM requests
@@ -570,6 +572,7 @@ router_settings:
 | LITELLM_LICENSE | License key for LiteLLM usage
 | LITELLM_LOCAL_MODEL_COST_MAP | Local configuration for model cost mapping in LiteLLM
 | LITELLM_LOG | Enable detailed logging for LiteLLM
+| LITELLM_LOG_FILE | File path to write LiteLLM logs to. When set, logs will be written to both console and the specified file
 | LITELLM_MASTER_KEY | Master key for proxy authentication
 | LITELLM_MODE | Operating mode for LiteLLM (e.g., production, development)
 | LITELLM_RATE_LIMIT_WINDOW_SIZE | Rate limit window size for LiteLLM. Default is 60
@@ -580,6 +583,7 @@ router_settings:
 | LITELM_ENVIRONMENT | Environment for LiteLLM Instance. This is currently only logged to DeepEval to determine the environment for DeepEval integration.
 | LOGFIRE_TOKEN | Token for Logfire logging service
 | MAX_EXCEPTION_MESSAGE_LENGTH | Maximum length for exception messages. Default is 2000
+| MAX_STRING_LENGTH_PROMPT_IN_DB | Maximum length for strings in spend logs when sanitizing request bodies. Strings longer than this will be truncated. Default is 1000
 | MAX_IN_MEMORY_QUEUE_FLUSH_COUNT | Maximum count for in-memory queue flush operations. Default is 1000
 | MAX_LONG_SIDE_FOR_IMAGE_HIGH_RES | Maximum length for the long side of high-resolution images. Default is 2000
 | MAX_REDIS_BUFFER_DEQUEUE_COUNT | Maximum count for Redis buffer dequeue operations. Default is 100

docs/my-website/docs/proxy/deploy.md

@@ -12,10 +12,7 @@ To start using Litellm, run the following commands in a shell:
 
 ```bash
 # Get the code
-git clone https://github.com/BerriAI/litellm
-
-# Go to folder
-cd litellm
+curl -O https://raw.githubusercontent.com/BerriAI/litellm/main/docker-compose.yml
 
 # Add the master key - you can change this after setup
 echo 'LITELLM_MASTER_KEY="sk-1234"' > .env

docs/my-website/docs/proxy/docker_quick_start.md

@@ -35,6 +35,30 @@ $ pip install 'litellm[proxy]'
 
 </TabItem>
 
+<TabItem value="docker-compose" label="Docker Compose (Proxy + DB)">
+
+Use this docker compose to spin up the proxy with a postgres database running locally. 
+
+```bash
+# Get the docker compose file
+curl -O https://raw.githubusercontent.com/BerriAI/litellm/main/docker-compose.yml
+
+# Add the master key - you can change this after setup
+echo 'LITELLM_MASTER_KEY="sk-1234"' > .env
+
+# Add the litellm salt key - you cannot change this after adding a model
+# It is used to encrypt / decrypt your LLM API Key credentials
+# We recommend - https://1password.com/password-generator/ 
+# password generator to get a random hash for litellm salt key
+echo 'LITELLM_SALT_KEY="sk-1234"' >> .env
+
+source .env
+
+# Start
+docker-compose up
+```
+
+</TabItem>
 </Tabs>
 
 ## 1. Add a model 
@@ -43,6 +67,8 @@ Control LiteLLM Proxy with a config.yaml file.
 
 Setup your config.yaml with your azure model.
 
+Note: When using the proxy with a database, you can also **just add models via UI** (UI is available on `/ui` route).
+
 ```yaml
 model_list:
   - model_name: gpt-4o

docs/my-website/docs/proxy/guardrails/custom_guardrail.md

@@ -23,15 +23,14 @@ A CustomGuardrail has 4 methods to enforce guardrails
 
 Create a new file called `custom_guardrail.py` and add this code to it
 ```python
-from typing import Any, Dict, List, Literal, Optional, Union
+from typing import Any, AsyncGenerator, Literal, Optional, Union
 
 import litellm
 from litellm._logging import verbose_proxy_logger
 from litellm.caching.caching import DualCache
 from litellm.integrations.custom_guardrail import CustomGuardrail
 from litellm.proxy._types import UserAPIKeyAuth
-from litellm.proxy.guardrails.guardrail_helpers import should_proceed_based_on_metadata
-from litellm.types.guardrails import GuardrailEventHooks
+from litellm.types.utils import ModelResponseStream
 
 
 class myCustomGuardrail(CustomGuardrail):

docs/my-website/docs/proxy/guardrails/noma_security.md

@@ -0,0 +1,299 @@
+import Image from '@theme/IdealImage';
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+
+# Noma Security
+
+Use [Noma Security](https://noma.security/) to protect your LLM applications with comprehensive AI content moderation and safety guardrails.
+
+## Quick Start
+
+### 1. Define Guardrails on your LiteLLM config.yaml
+
+Define your guardrails under the `guardrails` section:
+
+```yaml showLineNumbers title="litellm config.yaml"
+model_list:
+  - model_name: gpt-4o-mini
+    litellm_params:
+      model: openai/gpt-4o-mini
+      api_key: os.environ/OPENAI_API_KEY
+
+guardrails:
+  - guardrail_name: "noma-guard"
+    litellm_params:
+      guardrail: noma
+      mode: "during_call"
+      api_key: os.environ/NOMA_API_KEY
+      api_base: os.environ/NOMA_API_BASE
+  - guardrail_name: "noma-pre-guard"
+    litellm_params:
+      guardrail: noma
+      mode: "pre_call"
+      api_key: os.environ/NOMA_API_KEY
+      api_base: os.environ/NOMA_API_BASE
+```
+
+#### Supported values for `mode`
+
+- `pre_call` Run **before** LLM call, on **input**
+- `post_call` Run **after** LLM call, on **input & output**
+- `during_call` Run **during** LLM call, on **input**. Same as `pre_call` but runs in parallel with the LLM call. Response not returned until guardrail check completes
+
+### 2. Start LiteLLM Gateway
+
+```shell
+litellm --config config.yaml --detailed_debug
+```
+
+### 3. Test request
+
+<Tabs>
+<TabItem label="Unsuccessful call" value="not-allowed">
+
+Expect this to fail since the request contains harmful content:
+
+```shell showLineNumbers title="Curl Request"
+curl -i http://0.0.0.0:4000/v1/chat/completions \
+  -H "Content-Type: application/json" \
+  -d '{
+    "model": "gpt-4o-mini",
+    "messages": [
+      {"role": "user", "content": "Tell me how to hack into someone's email account"}
+    ]
+  }'
+```
+
+Expected response on failure:
+
+```json
+{
+  "error": {
+    "message": "{\n      \"error\": \"Request blocked by Noma guardrail\",\n      \"details\": {\n        \"prompt\": {\n          \"harmfulContent\": {\n            \"result\": true,\n            \"confidence\": 0.95\n          }\n        }\n      }\n    }",
+    "type": "None",
+    "param": "None",
+    "code": "400"
+  }
+}
+```
+
+</TabItem>
+
+<TabItem label="Successful Call" value="allowed">
+
+```shell showLineNumbers title="Curl Request"
+curl -i http://0.0.0.0:4000/v1/chat/completions \
+  -H "Content-Type: application/json" \
+  -d '{
+    "model": "gpt-4o-mini",
+    "messages": [
+      {"role": "user", "content": "What is the capital of France?"}
+    ]
+  }'
+```
+
+Expected response:
+
+```json
+{
+  "id": "chatcmpl-123",
+  "object": "chat.completion",
+  "created": 1677652288,
+  "model": "gpt-4o-mini",
+  "choices": [
+    {
+      "index": 0,
+      "message": {
+        "role": "assistant",
+        "content": "The capital of France is Paris."
+      },
+      "finish_reason": "stop"
+    }
+  ],
+  "usage": {
+    "prompt_tokens": 9,
+    "completion_tokens": 12,
+    "total_tokens": 21
+  }
+}
+```
+
+</TabItem>
+</Tabs>
+
+## Supported Params
+
+```yaml
+guardrails:
+  - guardrail_name: "noma-guard"
+    litellm_params:
+      guardrail: noma
+      mode: "pre_call"
+      api_key: os.environ/NOMA_API_KEY
+      api_base: os.environ/NOMA_API_BASE
+      ### OPTIONAL ###
+      # application_id: "my-app"
+      # monitor_mode: false
+      # block_failures: true
+```
+
+### Required Parameters
+
+- **`api_key`**: Your Noma Security API key (set as `os.environ/NOMA_API_KEY` in YAML config)
+
+### Optional Parameters
+
+- **`api_base`**: Noma API base URL (defaults to `https://api.noma.security/`)
+- **`application_id`**: Your application identifier (defaults to `"litellm"`)
+- **`monitor_mode`**: If `true`, logs violations without blocking (defaults to `false`)
+- **`block_failures`**: If `true`, blocks requests when guardrail API failures occur (defaults to `true`)
+
+## Environment Variables
+
+You can set these environment variables instead of hardcoding values in your config:
+
+```shell
+export NOMA_API_KEY="your-api-key-here"
+export NOMA_API_BASE="https://api.noma.security/"   # Optional
+export NOMA_APPLICATION_ID="my-app"                 # Optional
+export NOMA_MONITOR_MODE="false"                    # Optional
+export NOMA_BLOCK_FAILURES="true"                   # Optional
+```
+
+## Advanced Configuration
+
+### Monitor Mode
+
+Use monitor mode to test your guardrails without blocking requests:
+
+```yaml
+guardrails:
+  - guardrail_name: "noma-monitor"
+    litellm_params:
+      guardrail: noma
+      mode: "pre_call"
+      api_key: os.environ/NOMA_API_KEY
+      monitor_mode: true  # Log violations but don't block
+```
+
+### Handling API Failures
+
+Control behavior when the Noma API is unavailable:
+
+```yaml
+guardrails:
+  - guardrail_name: "noma-failopen"
+    litellm_params:
+      guardrail: noma
+      mode: "pre_call"
+      api_key: os.environ/NOMA_API_KEY
+      block_failures: false  # Allow requests to proceed if guardrail API fails
+```
+
+### Multiple Guardrails
+
+Apply different configurations for input and output:
+
+```yaml
+guardrails:
+  - guardrail_name: "noma-strict-input"
+    litellm_params:
+      guardrail: noma
+      mode: "pre_call"
+      api_key: os.environ/NOMA_API_KEY
+      block_failures: true
+
+  - guardrail_name: "noma-monitor-output"
+    litellm_params:
+      guardrail: noma
+      mode: "post_call"
+      api_key: os.environ/NOMA_API_KEY
+      monitor_mode: true
+```
+
+## ✨ Pass Additional Parameters
+
+Use `extra_body` to pass additional parameters to the Noma Security API call, such as dynamically setting the application ID for specific requests.
+
+<Tabs>
+<TabItem value="openai" label="OpenAI Python">
+
+```python
+import openai
+client = openai.OpenAI(
+    api_key="your-api-key",
+    base_url="http://0.0.0.0:4000"
+)
+
+response = client.chat.completions.create(
+    model="gpt-4o-mini",
+    messages=[{"role": "user", "content": "Hello, how are you?"}],
+    extra_body={
+        "guardrails": {
+            "noma-guard": {
+                "extra_body": {
+                    "application_id": "my-specific-app-id"
+                }
+            }
+        }
+    }
+)
+```
+</TabItem>
+
+<TabItem value="curl" label="Curl">
+
+```shell
+curl 'http://0.0.0.0:4000/v1/chat/completions' \
+    -H 'Content-Type: application/json' \
+    -d '{
+    "model": "gpt-4o-mini",
+    "messages": [
+        {
+            "role": "user",
+            "content": "Hello, how are you?"
+        }
+    ],
+    "guardrails": {
+        "noma-guard": {
+            "extra_body": {
+                "application_id": "my-specific-app-id"
+            }
+        }
+    }
+}'
+```
+</TabItem>
+</Tabs>
+
+This allows you to override the default `application_id` parameter for specific requests, which is useful for tracking usage across different applications or components.
+
+## Response Details
+
+When content is blocked, Noma provides detailed information about the violations as JSON inside the `message` field, with the following structure:
+
+```json
+{
+  "error": "Request blocked by Noma guardrail",
+  "details": {
+    "prompt": {
+      "harmfulContent": {
+        "result": true,
+        "confidence": 0.95
+      },
+      "sensitiveData": {
+        "email": {
+          "result": true,
+          "entities": ["user@example.com"]
+        }
+      },
+      "bannedTopics": {
+        "violence": {
+          "result": true,
+          "confidence": 0.88
+        }
+      }
+    }
+  }
+}
+```

docs/my-website/docs/proxy/prod.md

@@ -90,7 +90,7 @@ Recommended to do this for prod:
 
 ```yaml
 router_settings:
-  routing_strategy: usage-based-routing-v2 
+  routing_strategy: simple-shuffle # (default) - recommended for best performance
   # redis_url: "os.environ/REDIS_URL"
   redis_host: os.environ/REDIS_HOST
   redis_port: os.environ/REDIS_PORT
@@ -105,6 +105,9 @@ litellm_settings:
     password: os.environ/REDIS_PASSWORD
 ```
 
+> **WARNING**
+**Usage-based routing is not recommended for production due to performance impacts.** Use `simple-shuffle` (default) for optimal performance in high-traffic scenarios.
+
 ## 5. Disable 'load_dotenv'
 
 Set `export LITELLM_MODE="PRODUCTION"`

docs/my-website/docs/proxy/request_headers.md

@@ -6,6 +6,8 @@ Special headers that are supported by LiteLLM.
 
 `x-litellm-timeout` Optional[float]: The timeout for the request in seconds.
 
+`x-litellm-stream-timeout` Optional[float]: The timeout for getting the first chunk of the response in seconds (only applies for streaming requests). [Demo Video](https://www.loom.com/share/8da67e4845ce431a98c901d4e45db0e5)
+
 `x-litellm-enable-message-redaction`: Optional[bool]: Don't log the message content to logging integrations. Just track spend. [Learn More](./logging#redact-messages-response-content)
 
 `x-litellm-tags`: Optional[str]: A comma separated list (e.g. `tag1,tag2,tag3`) of tags to use for [tag-based routing](./tag_routing) **OR** [spend-tracking](./enterprise.md#tracking-spend-for-custom-tags).

docs/my-website/docs/proxy/self_serve.md

@@ -309,6 +309,37 @@ curl -X POST '<PROXY_BASE_URL>/team/new' \
 </TabItem>
 </Tabs>
 
+### Team Member Rate Limits
+
+Set a default tpm/rpm limit for an individual team member. 
+
+You can do this when creating a new team, or by updating an existing team. 
+
+
+<Tabs>
+<TabItem value="ui" label="UI">
+
+<Image img={require('../../img/create_team_member_rate_limits.png')}  style={{ width: '600px', height: 'auto' }} />
+
+</TabItem>
+<TabItem value="api" label="API">
+
+```bash
+curl -X POST '<PROXY_BASE_URL>/team/new' \
+-H 'Authorization: Bearer <PROXY_MASTER_KEY>' \
+-H 'Content-Type: application/json' \
+-D '{
+    "team_alias": "team_1",
+    "team_member_rpm_limit": 100,
+    "team_member_tpm_limit": 1000
+}'
+```
+
+</TabItem>
+</Tabs>
+
+
+
 ### Set default params for new teams
 
 When you connect litellm to your SSO provider, litellm can auto-create teams. Use this to set the default `models`, `max_budget`, `budget_duration` for these auto-created teams. 

docs/my-website/docs/proxy/timeout.md

@@ -38,9 +38,15 @@ $ litellm --config /path/to/config.yaml
 </TabItem>
 </Tabs>
 
-### Custom Timeouts, Stream Timeouts - Per Model
-For each model you can set `timeout` & `stream_timeout` under `litellm_params`
+### Custom Timeouts & Stream Timeouts (Per Model)
 
+For each model, you can set `timeout` and `stream_timeout` under `litellm_params`:
+
+- **`timeout`** → maximum time for the *complete response*.  
+  Use this to cap long-running completions.
+
+- **`stream_timeout`** → maximum time to wait for the *first chunk* (i.e., first token) in a streaming response.  
+  Use this to abort “hanging” providers (e.g., Bedrock slow start) and retry another model.
 <Tabs>
 <TabItem value="sdk" label="SDK">
 

docs/my-website/docs/routing.md

@@ -154,11 +154,153 @@ curl -X POST 'http://0.0.0.0:4000/chat/completions' \
 ## Advanced - Routing Strategies ⭐️
 #### Routing Strategies - Weighted Pick, Rate Limit Aware, Least Busy, Latency Based, Cost Based
 
-Router provides 4 strategies for routing your calls across multiple deployments: 
+Router provides multiple strategies for routing your calls across multiple deployments. **We recommend using `simple-shuffle` (default) for best performance in production.**
 
 <Tabs>
+<TabItem value="simple-shuffle" label="(Default) Weighted Pick - RECOMMENDED">
+
+**Default and Recommended for Production** - Best performance with minimal latency overhead.
+
+Picks a deployment based on the provided **Requests per minute (rpm) or Tokens per minute (tpm)**
+
+If `rpm` or `tpm` is not provided, it randomly picks a deployment
+
+You can also set a `weight` param, to specify which model should get picked when.
+
+<Tabs>
+<TabItem value="rpm" label="RPM-based shuffling">
+
+##### **LiteLLM Proxy Config.yaml**
+
+```yaml
+model_list:
+	- model_name: gpt-3.5-turbo
+	  litellm_params:
+	  	model: azure/chatgpt-v-2
+		api_key: os.environ/AZURE_API_KEY
+		api_version: os.environ/AZURE_API_VERSION
+		api_base: os.environ/AZURE_API_BASE
+		rpm: 900 
+	- model_name: gpt-3.5-turbo
+	  litellm_params:
+	  	model: azure/chatgpt-functioncalling
+		api_key: os.environ/AZURE_API_KEY
+		api_version: os.environ/AZURE_API_VERSION
+		api_base: os.environ/AZURE_API_BASE
+		rpm: 10 
+```
+
+##### **Python SDK**
+
+```python
+from litellm import Router 
+import asyncio
+
+model_list = [{ # list of model deployments 
+	"model_name": "gpt-3.5-turbo", # model alias 
+	"litellm_params": { # params for litellm completion/embedding call 
+		"model": "azure/chatgpt-v-2", # actual model name
+		"api_key": os.getenv("AZURE_API_KEY"),
+		"api_version": os.getenv("AZURE_API_VERSION"),
+		"api_base": os.getenv("AZURE_API_BASE"),
+		"rpm": 900,			# requests per minute for this API
+	}
+}, {
+    "model_name": "gpt-3.5-turbo", 
+	"litellm_params": { # params for litellm completion/embedding call 
+		"model": "azure/chatgpt-functioncalling", 
+		"api_key": os.getenv("AZURE_API_KEY"),
+		"api_version": os.getenv("AZURE_API_VERSION"),
+		"api_base": os.getenv("AZURE_API_BASE"),
+		"rpm": 10,
+	}
+},]
+
+# init router
+router = Router(model_list=model_list, routing_strategy="simple-shuffle")
+async def router_acompletion():
+	response = await router.acompletion(
+		model="gpt-3.5-turbo", 
+		messages=[{"role": "user", "content": "Hey, how's it going?"}]
+	)
+	print(response)
+	return response
+
+asyncio.run(router_acompletion())
+```
+
+</TabItem>
+<TabItem value="weight" label="Weight-based shuffling">
+
+##### **LiteLLM Proxy Config.yaml**
+
+```yaml
+model_list:
+	- model_name: gpt-3.5-turbo
+	  litellm_params:
+	  	model: azure/chatgpt-v-2
+		api_key: os.environ/AZURE_API_KEY
+		api_version: os.environ/AZURE_API_VERSION
+		api_base: os.environ/AZURE_API_BASE
+		weight: 9
+	- model_name: gpt-3.5-turbo
+	  litellm_params:
+	  	model: azure/chatgpt-functioncalling
+		api_key: os.environ/AZURE_API_KEY
+		api_version: os.environ/AZURE_API_VERSION
+		api_base: os.environ/AZURE_API_BASE
+		weight: 1 
+```
+
+##### **Python SDK**
+
+```python
+from litellm import Router 
+import asyncio
+
+model_list = [{
+	"model_name": "gpt-3.5-turbo", # model alias 
+	"litellm_params": { 
+		"model": "azure/chatgpt-v-2", # actual model name
+		"api_key": os.getenv("AZURE_API_KEY"),
+		"api_version": os.getenv("AZURE_API_VERSION"),
+		"api_base": os.getenv("AZURE_API_BASE"),
+		"weight": 9, # pick this 90% of the time
+	}
+}, {
+    "model_name": "gpt-3.5-turbo", 
+	"litellm_params": { 
+		"model": "azure/chatgpt-functioncalling", 
+		"api_key": os.getenv("AZURE_API_KEY"),
+		"api_version": os.getenv("AZURE_API_VERSION"),
+		"api_base": os.getenv("AZURE_API_BASE"),
+		"weight": 1,
+	}
+}]
+
+# init router
+router = Router(model_list=model_list, routing_strategy="simple-shuffle")
+async def router_acompletion():
+	response = await router.acompletion(
+		model="gpt-3.5-turbo", 
+		messages=[{"role": "user", "content": "Hey, how's it going?"}]
+	)
+	print(response)
+	return response
+
+asyncio.run(router_acompletion())
+```
+
+</TabItem>
+</Tabs>
+
+</TabItem>
 <TabItem value="usage-based-v2" label="Rate-Limit Aware v2 (ASYNC)">
 
+> [!WARNING]  
+**Usage-based routing is not recommended for production due to performance impacts.** Use `simple-shuffle` (default) for optimal performance in high-traffic scenarios. Usage-based routing adds significant latency due to Redis operations for tracking usage across deployments.
+
+
 **🎉 NEW** This is an async implementation of usage-based-routing.
 
 **Filters out deployment if tpm/rpm limit exceeded** - If you pass in the deployment's tpm/rpm limits.
@@ -209,7 +351,7 @@ router = Router(model_list=model_list,
                 redis_host=os.environ["REDIS_HOST"], 
 				redis_password=os.environ["REDIS_PASSWORD"], 
 				redis_port=os.environ["REDIS_PORT"], 
-                routing_strategy="usage-based-routing-v2" # 👈 KEY CHANGE
+                routing_strategy="simple-shuffle" # 👈 RECOMMENDED - best performance
 				enable_pre_call_checks=True, # enables router rate limits for concurrent calls
 				)
 
@@ -241,7 +383,7 @@ model_list:
 	  rpm: 1000
 
 router_settings:
-  routing_strategy: usage-based-routing-v2 # 👈 KEY CHANGE
+  routing_strategy: simple-shuffle # 👈 RECOMMENDED - best performance
   redis_host: <your-redis-host>
   redis_password: <your-redis-password>
   redis_port: <your-redis-port>
@@ -365,143 +507,7 @@ router_settings:
 ```
 
 </TabItem>
-<TabItem value="simple-shuffle" label="(Default) Weighted Pick (Async)">
 
-**Default** Picks a deployment based on the provided **Requests per minute (rpm) or Tokens per minute (tpm)**
-
-If `rpm` or `tpm` is not provided, it randomly picks a deployment
-
-You can also set a `weight` param, to specify which model should get picked when.
-
-<Tabs>
-<TabItem value="rpm" label="RPM-based shuffling">
-
-##### **LiteLLM Proxy Config.yaml**
-
-```yaml
-model_list:
-	- model_name: gpt-3.5-turbo
-	  litellm_params:
-	  	model: azure/chatgpt-v-2
-		api_key: os.environ/AZURE_API_KEY
-		api_version: os.environ/AZURE_API_VERSION
-		api_base: os.environ/AZURE_API_BASE
-		rpm: 900 
-	- model_name: gpt-3.5-turbo
-	  litellm_params:
-	  	model: azure/chatgpt-functioncalling
-		api_key: os.environ/AZURE_API_KEY
-		api_version: os.environ/AZURE_API_VERSION
-		api_base: os.environ/AZURE_API_BASE
-		rpm: 10 
-```
-
-##### **Python SDK**
-
-```python
-from litellm import Router 
-import asyncio
-
-model_list = [{ # list of model deployments 
-	"model_name": "gpt-3.5-turbo", # model alias 
-	"litellm_params": { # params for litellm completion/embedding call 
-		"model": "azure/chatgpt-v-2", # actual model name
-		"api_key": os.getenv("AZURE_API_KEY"),
-		"api_version": os.getenv("AZURE_API_VERSION"),
-		"api_base": os.getenv("AZURE_API_BASE"),
-		"rpm": 900,			# requests per minute for this API
-	}
-}, {
-    "model_name": "gpt-3.5-turbo", 
-	"litellm_params": { # params for litellm completion/embedding call 
-		"model": "azure/chatgpt-functioncalling", 
-		"api_key": os.getenv("AZURE_API_KEY"),
-		"api_version": os.getenv("AZURE_API_VERSION"),
-		"api_base": os.getenv("AZURE_API_BASE"),
-		"rpm": 10,
-	}
-},]
-
-# init router
-router = Router(model_list=model_list, routing_strategy="simple-shuffle")
-async def router_acompletion():
-	response = await router.acompletion(
-		model="gpt-3.5-turbo", 
-		messages=[{"role": "user", "content": "Hey, how's it going?"}]
-	)
-	print(response)
-	return response
-
-asyncio.run(router_acompletion())
-```
-
-</TabItem>
-<TabItem value="weight" label="Weight-based shuffling">
-
-##### **LiteLLM Proxy Config.yaml**
-
-```yaml
-model_list:
-	- model_name: gpt-3.5-turbo
-	  litellm_params:
-	  	model: azure/chatgpt-v-2
-		api_key: os.environ/AZURE_API_KEY
-		api_version: os.environ/AZURE_API_VERSION
-		api_base: os.environ/AZURE_API_BASE
-		weight: 9
-	- model_name: gpt-3.5-turbo
-	  litellm_params:
-	  	model: azure/chatgpt-functioncalling
-		api_key: os.environ/AZURE_API_KEY
-		api_version: os.environ/AZURE_API_VERSION
-		api_base: os.environ/AZURE_API_BASE
-		weight: 1 
-```
-
-
-##### **Python SDK**
-
-```python
-from litellm import Router 
-import asyncio
-
-model_list = [{
-	"model_name": "gpt-3.5-turbo", # model alias 
-	"litellm_params": { 
-		"model": "azure/chatgpt-v-2", # actual model name
-		"api_key": os.getenv("AZURE_API_KEY"),
-		"api_version": os.getenv("AZURE_API_VERSION"),
-		"api_base": os.getenv("AZURE_API_BASE"),
-		"weight": 9, # pick this 90% of the time
-	}
-}, {
-    "model_name": "gpt-3.5-turbo", 
-	"litellm_params": { 
-		"model": "azure/chatgpt-functioncalling", 
-		"api_key": os.getenv("AZURE_API_KEY"),
-		"api_version": os.getenv("AZURE_API_VERSION"),
-		"api_base": os.getenv("AZURE_API_BASE"),
-		"weight": 1,
-	}
-}]
-
-# init router
-router = Router(model_list=model_list, routing_strategy="simple-shuffle")
-async def router_acompletion():
-	response = await router.acompletion(
-		model="gpt-3.5-turbo", 
-		messages=[{"role": "user", "content": "Hey, how's it going?"}]
-	)
-	print(response)
-	return response
-
-asyncio.run(router_acompletion())
-```
-
-</TabItem>
-</Tabs>
-
-</TabItem>
 <TabItem value="usage-based" label="Rate-Limit Aware">
 
 This will route to the deployment with the lowest TPM usage for that minute. 

docs/my-website/docs/scheduler.md

@@ -41,7 +41,7 @@ router = Router(
         },
     ],
     timeout=2, # timeout request if takes > 2s
-    routing_strategy="usage-based-routing-v2",
+    routing_strategy="simple-shuffle", # recommended for best performance
     polling_interval=0.03 # poll queue every 3ms if no healthy deployments
 )
 

docs/my-website/release_notes/v1.74.15-stable/index.md

@@ -86,9 +86,9 @@ This is great to central AI Platform teams looking to track how they are helping
 | Provider    | Model                                  | Context Window | Input ($/1M tokens) | Output ($/1M tokens) | Cost per Image |
 | ----------- | -------------------------------------- | -------------- | ------------------- | -------------------- | -------------- |
 | OpenRouter | `openrouter/x-ai/grok-4` | 256k | $3 | $15 | N/A |
-| Google AI Studio | `gemini/imagen-4.0-generate-preview-06-06` | N/A | N/A | N/A | $0.04 |
-| Google AI Studio | `gemini/imagen-4.0-ultra-generate-preview-06-06` | N/A | N/A | N/A | $0.06 |
-| Google AI Studio | `gemini/imagen-4.0-fast-generate-preview-06-06` | N/A | N/A | N/A | $0.02 |
+| Google AI Studio | `gemini/imagen-4.0-generate-001` | N/A | N/A | N/A | $0.04 |
+| Google AI Studio | `gemini/imagen-4.0-ultra-generate-001` | N/A | N/A | N/A | $0.06 |
+| Google AI Studio | `gemini/imagen-4.0-fast-generate-001` | N/A | N/A | N/A | $0.02 |
 | Google AI Studio | `gemini/imagen-3.0-generate-002` | N/A | N/A | N/A | $0.04 |
 | Google AI Studio | `gemini/imagen-3.0-generate-001` | N/A | N/A | N/A | $0.04 |
 | Google AI Studio | `gemini/imagen-3.0-fast-generate-001` | N/A | N/A | N/A | $0.02 |

docs/my-website/release_notes/v1.75.8/index.md

@@ -1,5 +1,5 @@
 ---
-title: "[PRE-RELEASE]v1.75.8"
+title: "v1.75.8-stable - Team Member Rate Limits"
 slug: "v1-75-8"
 date: 2025-08-16T10:00:00
 authors:
@@ -28,7 +28,7 @@ import TabItem from '@theme/TabItem';
 docker run \
 -e STORE_MODEL_IN_DB=True \
 -p 4000:4000 \
-ghcr.io/berriai/litellm:v1.75.8
+ghcr.io/berriai/litellm:v1.75.8-stable
 ```
 </TabItem>
 
@@ -52,6 +52,22 @@ pip install litellm==1.75.8
 
 ---
 
+## Team Member Rate Limits
+
+<Image 
+  img={require('../../img/release_notes/team_member_rate_limits.png')}
+  style={{width: '100%', display: 'block', margin: '2rem auto'}}
+/>
+<p style={{textAlign: 'left', color: '#666'}}>
+  LiteLLM MCP Architecture: Use MCP tools with all LiteLLM supported models
+</p>
+
+
+This release adds support for setting rate limits on individual members (including machine users) within a team. Teams can now give each agent its own rate limits—so that heavy-traffic agents don’t impact other agents or human users. 
+
+Agents can authenticate with LiteLLM using JWT and the same team role as human users, while still enforcing per-agent rate limits.
+
+
 ## New Models / Updated Models
 
 #### New Model Support

docs/my-website/release_notes/v1.76.1-stable/index.md

@@ -0,0 +1,269 @@
+---
+title: "v1.76.1-stable - Gemini 2.5 Flash Image"
+slug: "v1-76-1"
+date: 2025-08-30T10:00:00
+authors:
+  - name: Krrish Dholakia
+    title: CEO, LiteLLM
+    url: https://www.linkedin.com/in/krish-d/
+    image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg
+  - name: Ishaan Jaffer
+    title: CTO, LiteLLM
+    url: https://www.linkedin.com/in/reffajnaahsi/
+    image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg
+
+hide_table_of_contents: false
+---
+
+import Image from '@theme/IdealImage';
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+
+## Deploy this version
+
+<Tabs>
+<TabItem value="docker" label="Docker">
+
+``` showLineNumbers title="docker run litellm"
+docker run \
+-e STORE_MODEL_IN_DB=True \
+-p 4000:4000 \
+ghcr.io/berriai/litellm:v1.76.1
+```
+</TabItem>
+
+<TabItem value="pip" label="Pip">
+
+``` showLineNumbers title="pip install litellm"
+pip install litellm==1.76.1
+```
+
+</TabItem>
+</Tabs>
+
+---
+
+## Key Highlights
+
+- **Major Performance Improvements** - 6.5x faster LiteLLM Python SDK completion with fastuuid integration.
+- **New Model Support** - Gemini 2.5 Flash Image Preview, Grok Code Fast, and GPT Realtime models
+- **Enhanced Provider Support** - DeepSeek-v3.1 pricing on Fireworks AI, Vercel AI Gateway, and improved Anthropic/GitHub Copilot integration
+- **MCP Improvements** - Better connection testing and SSE MCP tools bug fixes
+
+## Major Changes 
+- Added support for using Gemini 2.5 Flash Image Preview with /chat/completions. **🚨 Warning** If you were using `gemini-2.0-flash-exp-image-generation` please follow this migration guide.
+  [Gemini Image Generation Migration Guide](../../docs/extras/gemini_img_migration)
+---
+
+## Performance Improvements
+
+This release includes significant performance optimizations:
+
+- **6.5x faster LiteLLM Python SDK Completion** - Major performance boost for completion operations - [PR #13990](https://github.com/BerriAI/litellm/pull/13990)
+- **fastuuid Integration** - 2.1x faster UUID generation with +80 RPS improvement for /chat/completions and other LLM endpoints - [PR #13992](https://github.com/BerriAI/litellm/pull/13992), [PR #14016](https://github.com/BerriAI/litellm/pull/14016)
+- **Optimized Request Logging** - Don't print request params by default for +50 RPS improvement - [PR #14015](https://github.com/BerriAI/litellm/pull/14015)
+- **Cache Performance** - 21% speedup in InMemoryCache.evict_cache and 45% speedup in `_is_debugging_on` function - [PR #14012](https://github.com/BerriAI/litellm/pull/14012), [PR #13988](https://github.com/BerriAI/litellm/pull/13988)
+
+---
+
+## New Models / Updated Models
+
+#### New Model Support
+
+| Provider    | Model                                  | Context Window | Input ($/1M tokens) | Output ($/1M tokens) | Features |
+| ----------- | -------------------------------------- | -------------- | ------------------- | -------------------- | -------- |
+| Google | `gemini-2.5-flash-image-preview` | 1M | $0.30 | $2.50 | Chat completions + image generation ($0.039/image) |
+| X.AI | `xai/grok-code-fast` | 256K | $0.20 | $1.50 | Code generation |
+| OpenAI | `gpt-realtime` | 32K | $4.00 | $16.00 | Real-time conversation + audio |
+| Vercel AI Gateway | `vercel_ai_gateway/openai/o3` | 200K | $2.00 | $8.00 | Advanced reasoning |
+| Vercel AI Gateway | `vercel_ai_gateway/openai/o3-mini` | 200K | $1.10 | $4.40 | Efficient reasoning |
+| Vercel AI Gateway | `vercel_ai_gateway/openai/o4-mini` | 200K | $1.10 | $4.40 | Latest mini model |
+| DeepInfra | `deepinfra/zai-org/GLM-4.5` | 131K | $0.55 | $2.00 | Chat completions |
+| Perplexity | `perplexity/codellama-34b-instruct` | 16K | $0.35 | $1.40 | Code generation |
+| Fireworks AI | `fireworks_ai/accounts/fireworks/models/deepseek-v3p1` | 128K | $0.56 | $1.68 | Chat completions |
+
+**Additional Models Added:** Various other Vercel AI Gateway models were added too. See [models.litellm.ai](https://models.litellm.ai) for the full list.
+
+#### Features
+
+- **[Google Gemini](../../docs/providers/gemini)**
+    - Added support for `gemini-2.5-flash-image-preview` with image return capability - [PR #13979](https://github.com/BerriAI/litellm/pull/13979), [PR #13983](https://github.com/BerriAI/litellm/pull/13983)
+    - Support for requests with only system prompt - [PR #14010](https://github.com/BerriAI/litellm/pull/14010)
+    - Fixed invalid model name error for Gemini Imagen models - [PR #13991](https://github.com/BerriAI/litellm/pull/13991)
+- **[X.AI](../../docs/providers/xai)**
+    - Added `xai/grok-code-fast` model family support - [PR #14054](https://github.com/BerriAI/litellm/pull/14054)
+    - Fixed frequency_penalty parameter for grok-4 models - [PR #14078](https://github.com/BerriAI/litellm/pull/14078)
+- **[OpenAI](../../docs/providers/openai)**
+    - Added support for gpt-realtime models - [PR #14082](https://github.com/BerriAI/litellm/pull/14082)
+    - Support for reasoning and reasoning_effort parameters by default - [PR #12865](https://github.com/BerriAI/litellm/pull/12865)
+- **[Fireworks AI](../../docs/providers/fireworks_ai)**
+    - Added DeepSeek-v3.1 pricing - [PR #13958](https://github.com/BerriAI/litellm/pull/13958)
+- **[DeepInfra](../../docs/providers/deepinfra)**
+    - Fixed reasoning_effort setting for DeepSeek-V3.1 - [PR #14053](https://github.com/BerriAI/litellm/pull/14053)
+- **[GitHub Copilot](../../docs/providers/github_copilot)**
+    - Added support for thinking and reasoning_effort parameters - [PR #13691](https://github.com/BerriAI/litellm/pull/13691)
+    - Added image headers support - [PR #13955](https://github.com/BerriAI/litellm/pull/13955)
+- **[Anthropic](../../docs/providers/anthropic)**
+    - Support for custom Anthropic-compatible API endpoints - [PR #13945](https://github.com/BerriAI/litellm/pull/13945)
+    - Fixed /messages fallback from Anthropic API to Bedrock API - [PR #13946](https://github.com/BerriAI/litellm/pull/13946)
+- **[Nebius](../../docs/providers/nebius)**
+    - Expanded provider models and normalized model IDs - [PR #13965](https://github.com/BerriAI/litellm/pull/13965)
+- **[Vertex AI](../../docs/providers/vertex)**
+    - Fixed Vertex Mistral streaming issues - [PR #13952](https://github.com/BerriAI/litellm/pull/13952)
+    - Fixed anyOf corner cases for Gemini tool calls - [PR #12797](https://github.com/BerriAI/litellm/pull/12797)
+- **[Bedrock](../../docs/providers/bedrock)**
+    - Fixed structure output issues - [PR #14005](https://github.com/BerriAI/litellm/pull/14005)
+- **[OpenRouter](../../docs/providers/openrouter)**
+    - Added GPT-5 family models pricing - [PR #13536](https://github.com/BerriAI/litellm/pull/13536)
+
+#### New Provider Support
+
+- **[Vercel AI Gateway](../../docs/providers/vercel_ai_gateway)**
+    - New provider support added - [PR #13144](https://github.com/BerriAI/litellm/pull/13144)
+- **[DataRobot](../../docs/providers/datarobot)**
+    - Added provider documentation - [PR #14038](https://github.com/BerriAI/litellm/pull/14038), [PR #14074](https://github.com/BerriAI/litellm/pull/14074)
+
+---
+
+## LLM API Endpoints
+
+#### Features
+
+- **[Images API](../../docs/image_generation)**
+    - Support for multiple images in OpenAI images/edits endpoint - [PR #13916](https://github.com/BerriAI/litellm/pull/13916)
+    - Allow using dynamic `api_key` for image generation requests - [PR #14007](https://github.com/BerriAI/litellm/pull/14007)
+- **[Responses API](../../docs/response_api)**
+    - Fixed `/responses` endpoint ignoring extra_headers in GitHub Copilot - [PR #13775](https://github.com/BerriAI/litellm/pull/13775)
+    - Added support for new web_search tool - [PR #14083](https://github.com/BerriAI/litellm/pull/14083)
+- **[Azure Passthrough](../../docs/providers/azure/azure)**
+    - Fixed Azure Passthrough request with streaming - [PR #13831](https://github.com/BerriAI/litellm/pull/13831)
+
+#### Bugs
+
+- **General**
+    - Fixed handling of None metadata in batch requests - [PR #13996](https://github.com/BerriAI/litellm/pull/13996)
+    - Fixed token_counter with special token input - [PR #13374](https://github.com/BerriAI/litellm/pull/13374)
+    - Removed incorrect web search support for azure/gpt-4.1 family - [PR #13566](https://github.com/BerriAI/litellm/pull/13566)
+
+---
+
+## [MCP Gateway](../../docs/mcp)
+
+#### Features
+
+- **SSE MCP Tools**
+    - Bug fix for adding SSE MCP tools - improved connection testing when adding MCPs - [PR #14048](https://github.com/BerriAI/litellm/pull/14048)
+
+[Read More](../../docs/mcp)
+
+---
+
+## Management Endpoints / UI
+
+#### Features
+
+- **Team Management**
+    - Allow setting Team Member RPM/TPM limits when creating a team - [PR #13943](https://github.com/BerriAI/litellm/pull/13943)
+- **UI Improvements**
+    - Fixed Next.js Security Vulnerabilities in UI Dashboard - [PR #14084](https://github.com/BerriAI/litellm/pull/14084)
+    - Fixed collapsible navbar design - [PR #14075](https://github.com/BerriAI/litellm/pull/14075)
+
+#### Bugs
+
+- **Authentication**
+    - Fixed Virtual keys with llm_api type causing Internal Server Error for /anthropic/* and other LLM passthrough routes - [PR #14046](https://github.com/BerriAI/litellm/pull/14046)
+
+---
+
+## Logging / Guardrail Integrations
+
+#### Features
+
+- **[Langfuse OTEL](../../docs/proxy/logging#langfuse)**
+    - Allow using LANGFUSE_OTEL_HOST for configuring host - [PR #14013](https://github.com/BerriAI/litellm/pull/14013)
+- **[Braintrust](../../docs/proxy/logging#braintrust)**
+    - Added span name metadata feature - [PR #13573](https://github.com/BerriAI/litellm/pull/13573)
+    - Fixed tests to reference moved attributes in `braintrust_logging` module - [PR #13978](https://github.com/BerriAI/litellm/pull/13978)
+- **[OpenMeter](../../docs/proxy/logging#openmeter)**
+    - Set user from token user_id for OpenMeter integration - [PR #13152](https://github.com/BerriAI/litellm/pull/13152)
+
+#### New Guardrail Support
+
+- **[Noma Security](../../docs/proxy/guardrails)**
+    - Added Noma Security guardrail support - [PR #13572](https://github.com/BerriAI/litellm/pull/13572)
+- **[Pangea](../../docs/proxy/guardrails)**
+    - Updated Pangea Guardrail to support new AIDR endpoint - [PR #13160](https://github.com/BerriAI/litellm/pull/13160)
+
+---
+
+## Performance / Loadbalancing / Reliability improvements
+
+#### Features
+
+- **Caching**
+    - Verify if cache entry has expired prior to serving it to client - [PR #13933](https://github.com/BerriAI/litellm/pull/13933)
+    - Fixed error saving latency as timedelta on Redis - [PR #14040](https://github.com/BerriAI/litellm/pull/14040)
+- **Router**
+    - Refactored router to choose weights by 'weight', 'rpm', 'tpm' in one loop for simple_shuffle - [PR #13562](https://github.com/BerriAI/litellm/pull/13562)
+- **Logging**
+    - Fixed LoggingWorker graceful shutdown to prevent CancelledError warnings - [PR #14050](https://github.com/BerriAI/litellm/pull/14050)
+    - Enhanced logging for containers to log on files both with usual format and json format - [PR #13394](https://github.com/BerriAI/litellm/pull/13394)
+
+#### Bugs
+
+- **Dependencies**
+    - Bumped `orjson` version to "3.11.2" - [PR #13969](https://github.com/BerriAI/litellm/pull/13969)
+
+---
+
+## General Proxy Improvements
+
+#### Features
+
+- **AWS**
+    - Add support for AWS assume_role with a session token - [PR #13919](https://github.com/BerriAI/litellm/pull/13919)
+- **OCI Provider**
+    - Added oci_key_file as an optional_parameter - [PR #14036](https://github.com/BerriAI/litellm/pull/14036)
+- **Configuration**
+    - Allow configuration to set threshold before request entry in spend log gets truncated - [PR #14042](https://github.com/BerriAI/litellm/pull/14042)
+    - Enhanced proxy_config configuration: add support for existing configmap in Helm charts - [PR #14041](https://github.com/BerriAI/litellm/pull/14041)
+- **Docker**
+    - Added back supervisor to non-root image - [PR #13922](https://github.com/BerriAI/litellm/pull/13922)
+
+
+---
+
+## New Contributors
+* @ArthurRenault made their first contribution in [PR #13922](https://github.com/BerriAI/litellm/pull/13922)
+* @stevenmanton made their first contribution in [PR #13919](https://github.com/BerriAI/litellm/pull/13919)
+* @uc4w6c made their first contribution in [PR #13914](https://github.com/BerriAI/litellm/pull/13914)
+* @nielsbosma made their first contribution in [PR #13573](https://github.com/BerriAI/litellm/pull/13573)
+* @Yuki-Imajuku made their first contribution in [PR #13567](https://github.com/BerriAI/litellm/pull/13567)
+* @codeflash-ai[bot] made their first contribution in [PR #13988](https://github.com/BerriAI/litellm/pull/13988)
+* @ColeFrench made their first contribution in [PR #13978](https://github.com/BerriAI/litellm/pull/13978)
+* @dttran-glo made their first contribution in [PR #13969](https://github.com/BerriAI/litellm/pull/13969)
+* @manascb1344 made their first contribution in [PR #13965](https://github.com/BerriAI/litellm/pull/13965)
+* @DorZion made their first contribution in [PR #13572](https://github.com/BerriAI/litellm/pull/13572)
+* @edwardsamuel made their first contribution in [PR #13536](https://github.com/BerriAI/litellm/pull/13536)
+* @blahgeek made their first contribution in [PR #13374](https://github.com/BerriAI/litellm/pull/13374)
+* @Deviad made their first contribution in [PR #13394](https://github.com/BerriAI/litellm/pull/13394)
+* @XSAM made their first contribution in [PR #13775](https://github.com/BerriAI/litellm/pull/13775)
+* @KRRT7 made their first contribution in [PR #14012](https://github.com/BerriAI/litellm/pull/14012)
+* @ikaadil made their first contribution in [PR #13991](https://github.com/BerriAI/litellm/pull/13991)
+* @timelfrink made their first contribution in [PR #13691](https://github.com/BerriAI/litellm/pull/13691)
+* @qidu made their first contribution in [PR #13562](https://github.com/BerriAI/litellm/pull/13562)
+* @nagyv made their first contribution in [PR #13243](https://github.com/BerriAI/litellm/pull/13243)
+* @xywei made their first contribution in [PR #12885](https://github.com/BerriAI/litellm/pull/12885)
+* @ericgtkb made their first contribution in [PR #12797](https://github.com/BerriAI/litellm/pull/12797)
+* @NoWall57 made their first contribution in [PR #13945](https://github.com/BerriAI/litellm/pull/13945)
+* @lmwang9527 made their first contribution in [PR #14050](https://github.com/BerriAI/litellm/pull/14050)
+* @WilsonSunBritten made their first contribution in [PR #14042](https://github.com/BerriAI/litellm/pull/14042)
+* @Const-antine made their first contribution in [PR #14041](https://github.com/BerriAI/litellm/pull/14041)
+* @dmvieira made their first contribution in [PR #14040](https://github.com/BerriAI/litellm/pull/14040)
+* @gotsysdba made their first contribution in [PR #14036](https://github.com/BerriAI/litellm/pull/14036)
+* @moshemorad made their first contribution in [PR #14005](https://github.com/BerriAI/litellm/pull/14005)
+* @joshualipman123 made their first contribution in [PR #13144](https://github.com/BerriAI/litellm/pull/13144)
+
+---
+
+## **[Full Changelog](https://github.com/BerriAI/litellm/compare/v1.76.0-nightly...v1.76.1)**

docs/my-website/sidebars.js

@@ -40,6 +40,7 @@ const sidebars = {
           "proxy/guardrails/guardrails_ai",
           "proxy/guardrails/lakera_ai",
           "proxy/guardrails/model_armor",
+          "proxy/guardrails/noma_security",
           "proxy/guardrails/openai_moderation",
           "proxy/guardrails/pangea",
           "proxy/guardrails/pillar_security",
@@ -463,6 +464,7 @@ const sidebars = {
         "providers/replicate",
         "providers/togetherai",
         "providers/v0",
+        "providers/vercel_ai_gateway",
         "providers/morph",
         "providers/lambda_ai",
         "providers/novita",
@@ -481,6 +483,7 @@ const sidebars = {
         "providers/dashscope",
         "providers/bytez",
         "providers/oci",
+        "providers/datarobot",  
       ],
     },
     {
@@ -492,6 +495,7 @@ const sidebars = {
         "guides/finetuned_models",
         "guides/security_settings",
         "completion/audio",
+        "completion/image_generation_chat",
         "completion/web_search",
         "completion/document_understanding",
         "completion/vision",

enterprise/litellm_enterprise/integrations/prometheus.py

@@ -95,13 +95,14 @@ class PrometheusLogger(CustomLogger):
             self.litellm_llm_api_time_to_first_token_metric = self._histogram_factory(
                 "litellm_llm_api_time_to_first_token_metric",
                 "Time to first token for a models LLM API call",
-                labelnames=[
-                    "model",
-                    "hashed_api_key",
-                    "api_key_alias",
-                    "team",
-                    "team_alias",
-                ],
+                # labelnames=[
+                #     "model",
+                #     "hashed_api_key",
+                #     "api_key_alias",
+                #     "team",
+                #     "team_alias",
+                # ],
+                labelnames=self.get_labels_for_metric("litellm_llm_api_time_to_first_token_metric"),
                 buckets=LATENCY_BUCKETS,
             )
 
@@ -109,15 +110,7 @@ class PrometheusLogger(CustomLogger):
             self.litellm_spend_metric = self._counter_factory(
                 "litellm_spend_metric",
                 "Total spend on LLM requests",
-                labelnames=[
-                    "end_user",
-                    "hashed_api_key",
-                    "api_key_alias",
-                    "model",
-                    "team",
-                    "team_alias",
-                    "user",
-                ],
+                labelnames=self.get_labels_for_metric("litellm_spend_metric"),
             )
 
             # Counter for total_output_tokens
@@ -243,25 +236,18 @@ class PrometheusLogger(CustomLogger):
                 labelnames=["api_provider"],
             )
 
-            # Get all keys
-            _logged_llm_labels = [
-                UserAPIKeyLabelNames.v2_LITELLM_MODEL_NAME.value,
-                UserAPIKeyLabelNames.MODEL_ID.value,
-                UserAPIKeyLabelNames.API_BASE.value,
-                UserAPIKeyLabelNames.API_PROVIDER.value,
-            ]
-
             # Metric for deployment state
             self.litellm_deployment_state = self._gauge_factory(
                 "litellm_deployment_state",
                 "LLM Deployment Analytics - The state of the deployment: 0 = healthy, 1 = partial outage, 2 = complete outage",
-                labelnames=_logged_llm_labels,
+                labelnames=self.get_labels_for_metric("litellm_deployment_state")
             )
 
             self.litellm_deployment_cooled_down = self._counter_factory(
                 "litellm_deployment_cooled_down",
                 "LLM Deployment Analytics - Number of times a deployment has been cooled down by LiteLLM load balancing logic. exception_status is the status of the exception that caused the deployment to be cooled down",
-                labelnames=_logged_llm_labels + [EXCEPTION_STATUS],
+                # labelnames=_logged_llm_labels + [EXCEPTION_STATUS],
+                labelnames=self.get_labels_for_metric("litellm_deployment_cooled_down")
             )
 
             self.litellm_deployment_success_responses = self._counter_factory(
@@ -327,6 +313,7 @@ class PrometheusLogger(CustomLogger):
                 documentation="deprecated - use litellm_proxy_total_requests_metric. Total number of LLM calls to litellm - track total per API Key, team, user",
                 labelnames=self.get_labels_for_metric("litellm_requests_metric"),
             )
+
         except Exception as e:
             print_verbose(f"Got exception on init prometheus client {str(e)}")
             raise e

litellm/__init__.py

@@ -226,6 +226,7 @@ vertex_location: Optional[str] = None
 predibase_tenant_id: Optional[str] = None
 togetherai_api_key: Optional[str] = None
 cloudflare_api_key: Optional[str] = None
+vercel_ai_gateway_key: Optional[str] = None
 baseten_key: Optional[str] = None
 llama_api_key: Optional[str] = None
 aleph_alpha_key: Optional[str] = None
@@ -542,6 +543,7 @@ hyperbolic_models: Set = set()
 recraft_models: Set = set()
 cometapi_models: Set = set()
 oci_models: Set = set()
+vercel_ai_gateway_models: Set = set()
 
 
 def is_bedrock_pricing_only_model(key: str) -> bool:
@@ -599,6 +601,8 @@ def add_known_models():
             empower_models.add(key)
         elif value.get("litellm_provider") == "openrouter":
             openrouter_models.add(key)
+        elif value.get("litellm_provider") == "vercel_ai_gateway":
+            vercel_ai_gateway_models.add(key)
         elif value.get("litellm_provider") == "datarobot":
             datarobot_models.add(key)
         elif value.get("litellm_provider") == "vertex_ai-text-models":
@@ -835,6 +839,7 @@ model_list = list(
     | recraft_models
     | cometapi_models
     | oci_models
+    | vercel_ai_gateway_models
 )
 
 model_list_set = set(model_list)
@@ -853,6 +858,7 @@ models_by_provider: dict = {
     "together_ai": together_ai_models,
     "baseten": baseten_models,
     "openrouter": openrouter_models,
+    "vercel_ai_gateway": vercel_ai_gateway_models,
     "datarobot": datarobot_models,
     "vertex_ai": vertex_chat_models | vertex_text_models | vertex_anthropic_models | vertex_vision_models | vertex_language_models | vertex_deepseek_models,
     "ai21": ai21_models,
@@ -1247,6 +1253,7 @@ from .llms.oci.chat.transformation import OCIChatConfig
 from .llms.morph.chat.transformation import MorphChatConfig
 from .llms.lambda_ai.chat.transformation import LambdaAIChatConfig
 from .llms.hyperbolic.chat.transformation import HyperbolicChatConfig
+from .llms.vercel_ai_gateway.chat.transformation import VercelAIGatewayConfig
 from .main import *  # type: ignore
 from .integrations import *
 from .llms.custom_httpx.async_client_cleanup import close_litellm_async_clients
@@ -1254,6 +1261,7 @@ from .exceptions import (
     AuthenticationError,
     InvalidRequestError,
     BadRequestError,
+    ImageFetchError,
     NotFoundError,
     RateLimitError,
     ServiceUnavailableError,
@@ -1278,7 +1286,6 @@ from .router import Router
 from .assistants.main import *
 from .batches.main import *
 from .images.main import *
-from .vector_stores import *
 from .batch_completion.main import *  # type: ignore
 from .rerank_api.main import *
 from .llms.anthropic.experimental_pass_through.messages.handler import *

litellm/_logging.py

@@ -108,6 +108,7 @@ verbose_router_logger.addHandler(handler)
 verbose_proxy_logger.addHandler(handler)
 verbose_logger.addHandler(handler)
 
+
 def _suppress_loggers():
     """Suppress noisy loggers at INFO level"""
     # Suppress httpx request logging at INFO level
@@ -120,6 +121,7 @@ def _suppress_loggers():
     apscheduler_scheduler_logger = logging.getLogger("apscheduler.scheduler")
     apscheduler_scheduler_logger.setLevel(logging.WARNING)
 
+
 # Call the suppression function
 _suppress_loggers()
 
@@ -187,6 +189,4 @@ def _is_debugging_on() -> bool:
     """
     Returns True if debugging is on
     """
-    if verbose_logger.isEnabledFor(logging.DEBUG) or set_verbose is True:
-        return True
-    return False
+    return verbose_logger.isEnabledFor(logging.DEBUG) or set_verbose is True

litellm/caching/in_memory_cache.py

@@ -112,14 +112,15 @@ class InMemoryCache(BaseCache):
         - 3. the size of in-memory cache is bounded
 
         """
-        for key in list(self.ttl_dict.keys()):
-            if self._is_key_expired(key):
-                self._remove_key(key)
+        current_time = time.time()
+        expired_keys = [key for key, ttl in self.ttl_dict.items() if current_time > ttl]
+        for key in expired_keys:
+            self._remove_key(key)
 
-                # de-reference the removed item
-                # https://www.geeksforgeeks.org/diagnosing-and-fixing-memory-leaks-in-python/
-                # One of the most common causes of memory leaks in Python is the retention of objects that are no longer being used.
-                # This can occur when an object is referenced by another object, but the reference is never removed.
+        # de-reference the removed item
+        # https://www.geeksforgeeks.org/diagnosing-and-fixing-memory-leaks-in-python/
+        # One of the most common causes of memory leaks in Python is the retention of objects that are no longer being used.
+        # This can occur when an object is referenced by another object, but the reference is never removed.
 
     def allow_ttl_override(self, key: str) -> bool:
         """

litellm/caching/s3_cache.py

@@ -13,6 +13,7 @@ import asyncio
 import json
 from functools import partial
 from typing import Optional
+from datetime import datetime, timezone, timedelta
 
 from litellm._logging import print_verbose, verbose_logger
 
@@ -69,11 +70,9 @@ class S3Cache(BaseCache):
 
             if ttl is not None:
                 cache_control = f"immutable, max-age={ttl}, s-maxage={ttl}"
-                import datetime
 
                 # Calculate expiration time
-                expiration_time = datetime.datetime.now() + datetime.timedelta(seconds=ttl)
-
+                expiration_time = datetime.now(timezone.utc) + timedelta(seconds=ttl)
                 # Upload the data to S3 with the calculated expiration time
                 self.s3_client.put_object(
                     Bucket=self.bucket_name,
@@ -126,6 +125,13 @@ class S3Cache(BaseCache):
             )
 
             if cached_response is not None:
+                if "Expires" in cached_response:
+                    expires_time = cached_response['Expires']
+                    current_time = datetime.now(expires_time.tzinfo)
+
+                    if current_time > expires_time:
+                        return None
+
                 # cached_response is in `b{} convert it to ModelResponse
                 cached_response = (
                     cached_response["Body"].read().decode("utf-8")

litellm/constants.py

@@ -14,6 +14,7 @@ DEFAULT_S3_BATCH_SIZE = int(os.getenv("DEFAULT_S3_BATCH_SIZE", 512))
 DEFAULT_SQS_FLUSH_INTERVAL_SECONDS = int(
     os.getenv("DEFAULT_SQS_FLUSH_INTERVAL_SECONDS", 10)
 )
+DEFAULT_NUM_WORKERS_LITELLM_PROXY = int(os.getenv("DEFAULT_NUM_WORKERS_LITELLM_PROXY", 4))
 DEFAULT_SQS_BATCH_SIZE = int(os.getenv("DEFAULT_SQS_BATCH_SIZE", 512))
 SQS_SEND_MESSAGE_ACTION = "SendMessage"
 SQS_API_VERSION = "2012-11-05"
@@ -157,6 +158,7 @@ NON_LLM_CONNECTION_TIMEOUT = int(
     os.getenv("NON_LLM_CONNECTION_TIMEOUT", 15)
 )  # timeout for adjacent services (e.g. jwt auth)
 MAX_EXCEPTION_MESSAGE_LENGTH = int(os.getenv("MAX_EXCEPTION_MESSAGE_LENGTH", 2000))
+MAX_STRING_LENGTH_PROMPT_IN_DB = int(os.getenv("MAX_STRING_LENGTH_PROMPT_IN_DB", 1000))
 BEDROCK_MAX_POLICY_SIZE = int(os.getenv("BEDROCK_MAX_POLICY_SIZE", 75))
 REPLICATE_POLLING_DELAY_SECONDS = float(
     os.getenv("REPLICATE_POLLING_DELAY_SECONDS", 0.5)
@@ -288,6 +290,7 @@ LITELLM_CHAT_PROVIDERS = [
     "oci",
     "morph",
     "lambda_ai",
+    "vercel_ai_gateway",
 ]
 
 LITELLM_EMBEDDING_PROVIDERS_SUPPORTING_INPUT_ARRAY_OF_TOKENS = [
@@ -420,6 +423,7 @@ openai_compatible_endpoints: List = [
     "https://api.morphllm.com/v1",
     "https://api.lambda.ai/v1",
     "https://api.hyperbolic.xyz/v1",
+    "https://ai-gateway.vercel.sh/v1",
 ]
 
 
@@ -462,6 +466,7 @@ openai_compatible_providers: List = [
     "morph",
     "lambda_ai",
     "hyperbolic",
+    "vercel_ai_gateway",
     "aiml",
 ]
 openai_text_completion_compatible_providers: List = (
@@ -486,190 +491,247 @@ _openai_like_providers: List = [
     "watsonx",
 ]  # private helper. similar to openai but require some custom auth / endpoint handling, so can't use the openai sdk
 # well supported replicate llms
-replicate_models: set = set([
-    # llama replicate supported LLMs
-    "replicate/llama-2-70b-chat:2796ee9483c3fd7aa2e171d38f4ca12251a30609463dcfd4cd76703f22e96cdf",
-    "a16z-infra/llama-2-13b-chat:2a7f981751ec7fdf87b5b91ad4db53683a98082e9ff7bfd12c8cd5ea85980a52",
-    "meta/codellama-13b:1c914d844307b0588599b8393480a3ba917b660c7e9dfae681542b5325f228db",
-    # Vicuna
-    "replicate/vicuna-13b:6282abe6a492de4145d7bb601023762212f9ddbbe78278bd6771c8b3b2f2a13b",
-    "joehoover/instructblip-vicuna13b:c4c54e3c8c97cd50c2d2fec9be3b6065563ccf7d43787fb99f84151b867178fe",
-    # Flan T-5
-    "daanelson/flan-t5-large:ce962b3f6792a57074a601d3979db5839697add2e4e02696b3ced4c022d4767f",
-    # Others
-    "replicate/dolly-v2-12b:ef0e1aefc61f8e096ebe4db6b2bacc297daf2ef6899f0f7e001ec445893500e5",
-    "replit/replit-code-v1-3b:b84f4c074b807211cd75e3e8b1589b6399052125b4c27106e43d47189e8415ad",
-])
+replicate_models: set = set(
+    [
+        # llama replicate supported LLMs
+        "replicate/llama-2-70b-chat:2796ee9483c3fd7aa2e171d38f4ca12251a30609463dcfd4cd76703f22e96cdf",
+        "a16z-infra/llama-2-13b-chat:2a7f981751ec7fdf87b5b91ad4db53683a98082e9ff7bfd12c8cd5ea85980a52",
+        "meta/codellama-13b:1c914d844307b0588599b8393480a3ba917b660c7e9dfae681542b5325f228db",
+        # Vicuna
+        "replicate/vicuna-13b:6282abe6a492de4145d7bb601023762212f9ddbbe78278bd6771c8b3b2f2a13b",
+        "joehoover/instructblip-vicuna13b:c4c54e3c8c97cd50c2d2fec9be3b6065563ccf7d43787fb99f84151b867178fe",
+        # Flan T-5
+        "daanelson/flan-t5-large:ce962b3f6792a57074a601d3979db5839697add2e4e02696b3ced4c022d4767f",
+        # Others
+        "replicate/dolly-v2-12b:ef0e1aefc61f8e096ebe4db6b2bacc297daf2ef6899f0f7e001ec445893500e5",
+        "replit/replit-code-v1-3b:b84f4c074b807211cd75e3e8b1589b6399052125b4c27106e43d47189e8415ad",
+    ]
+)
 
-clarifai_models: set = set([
-    "clarifai/meta.Llama-3.Llama-3-8B-Instruct",
-    "clarifai/gcp.generate.gemma-1_1-7b-it",
-    "clarifai/mistralai.completion.mixtral-8x22B",
-    "clarifai/cohere.generate.command-r-plus",
-    "clarifai/databricks.drbx.dbrx-instruct",
-    "clarifai/mistralai.completion.mistral-large",
-    "clarifai/mistralai.completion.mistral-medium",
-    "clarifai/mistralai.completion.mistral-small",
-    "clarifai/mistralai.completion.mixtral-8x7B-Instruct-v0_1",
-    "clarifai/gcp.generate.gemma-2b-it",
-    "clarifai/gcp.generate.gemma-7b-it",
-    "clarifai/deci.decilm.deciLM-7B-instruct",
-    "clarifai/mistralai.completion.mistral-7B-Instruct",
-    "clarifai/gcp.generate.gemini-pro",
-    "clarifai/anthropic.completion.claude-v1",
-    "clarifai/anthropic.completion.claude-instant-1_2",
-    "clarifai/anthropic.completion.claude-instant",
-    "clarifai/anthropic.completion.claude-v2",
-    "clarifai/anthropic.completion.claude-2_1",
-    "clarifai/meta.Llama-2.codeLlama-70b-Python",
-    "clarifai/meta.Llama-2.codeLlama-70b-Instruct",
-    "clarifai/openai.completion.gpt-3_5-turbo-instruct",
-    "clarifai/meta.Llama-2.llama2-7b-chat",
-    "clarifai/meta.Llama-2.llama2-13b-chat",
-    "clarifai/meta.Llama-2.llama2-70b-chat",
-    "clarifai/openai.chat-completion.gpt-4-turbo",
-    "clarifai/microsoft.text-generation.phi-2",
-    "clarifai/meta.Llama-2.llama2-7b-chat-vllm",
-    "clarifai/upstage.solar.solar-10_7b-instruct",
-    "clarifai/openchat.openchat.openchat-3_5-1210",
-    "clarifai/togethercomputer.stripedHyena.stripedHyena-Nous-7B",
-    "clarifai/gcp.generate.text-bison",
-    "clarifai/meta.Llama-2.llamaGuard-7b",
-    "clarifai/fblgit.una-cybertron.una-cybertron-7b-v2",
-    "clarifai/openai.chat-completion.GPT-4",
-    "clarifai/openai.chat-completion.GPT-3_5-turbo",
-    "clarifai/ai21.complete.Jurassic2-Grande",
-    "clarifai/ai21.complete.Jurassic2-Grande-Instruct",
-    "clarifai/ai21.complete.Jurassic2-Jumbo-Instruct",
-    "clarifai/ai21.complete.Jurassic2-Jumbo",
-    "clarifai/ai21.complete.Jurassic2-Large",
-    "clarifai/cohere.generate.cohere-generate-command",
-    "clarifai/wizardlm.generate.wizardCoder-Python-34B",
-    "clarifai/wizardlm.generate.wizardLM-70B",
-    "clarifai/tiiuae.falcon.falcon-40b-instruct",
-    "clarifai/togethercomputer.RedPajama.RedPajama-INCITE-7B-Chat",
-    "clarifai/gcp.generate.code-gecko",
-    "clarifai/gcp.generate.code-bison",
-    "clarifai/mistralai.completion.mistral-7B-OpenOrca",
-    "clarifai/mistralai.completion.openHermes-2-mistral-7B",
-    "clarifai/wizardlm.generate.wizardLM-13B",
-    "clarifai/huggingface-research.zephyr.zephyr-7B-alpha",
-    "clarifai/wizardlm.generate.wizardCoder-15B",
-    "clarifai/microsoft.text-generation.phi-1_5",
-    "clarifai/databricks.Dolly-v2.dolly-v2-12b",
-    "clarifai/bigcode.code.StarCoder",
-    "clarifai/salesforce.xgen.xgen-7b-8k-instruct",
-    "clarifai/mosaicml.mpt.mpt-7b-instruct",
-    "clarifai/anthropic.completion.claude-3-opus",
-    "clarifai/anthropic.completion.claude-3-sonnet",
-    "clarifai/gcp.generate.gemini-1_5-pro",
-    "clarifai/gcp.generate.imagen-2",
-    "clarifai/salesforce.blip.general-english-image-caption-blip-2",
-])
+clarifai_models: set = set(
+    [
+        "clarifai/meta.Llama-3.Llama-3-8B-Instruct",
+        "clarifai/gcp.generate.gemma-1_1-7b-it",
+        "clarifai/mistralai.completion.mixtral-8x22B",
+        "clarifai/cohere.generate.command-r-plus",
+        "clarifai/databricks.drbx.dbrx-instruct",
+        "clarifai/mistralai.completion.mistral-large",
+        "clarifai/mistralai.completion.mistral-medium",
+        "clarifai/mistralai.completion.mistral-small",
+        "clarifai/mistralai.completion.mixtral-8x7B-Instruct-v0_1",
+        "clarifai/gcp.generate.gemma-2b-it",
+        "clarifai/gcp.generate.gemma-7b-it",
+        "clarifai/deci.decilm.deciLM-7B-instruct",
+        "clarifai/mistralai.completion.mistral-7B-Instruct",
+        "clarifai/gcp.generate.gemini-pro",
+        "clarifai/anthropic.completion.claude-v1",
+        "clarifai/anthropic.completion.claude-instant-1_2",
+        "clarifai/anthropic.completion.claude-instant",
+        "clarifai/anthropic.completion.claude-v2",
+        "clarifai/anthropic.completion.claude-2_1",
+        "clarifai/meta.Llama-2.codeLlama-70b-Python",
+        "clarifai/meta.Llama-2.codeLlama-70b-Instruct",
+        "clarifai/openai.completion.gpt-3_5-turbo-instruct",
+        "clarifai/meta.Llama-2.llama2-7b-chat",
+        "clarifai/meta.Llama-2.llama2-13b-chat",
+        "clarifai/meta.Llama-2.llama2-70b-chat",
+        "clarifai/openai.chat-completion.gpt-4-turbo",
+        "clarifai/microsoft.text-generation.phi-2",
+        "clarifai/meta.Llama-2.llama2-7b-chat-vllm",
+        "clarifai/upstage.solar.solar-10_7b-instruct",
+        "clarifai/openchat.openchat.openchat-3_5-1210",
+        "clarifai/togethercomputer.stripedHyena.stripedHyena-Nous-7B",
+        "clarifai/gcp.generate.text-bison",
+        "clarifai/meta.Llama-2.llamaGuard-7b",
+        "clarifai/fblgit.una-cybertron.una-cybertron-7b-v2",
+        "clarifai/openai.chat-completion.GPT-4",
+        "clarifai/openai.chat-completion.GPT-3_5-turbo",
+        "clarifai/ai21.complete.Jurassic2-Grande",
+        "clarifai/ai21.complete.Jurassic2-Grande-Instruct",
+        "clarifai/ai21.complete.Jurassic2-Jumbo-Instruct",
+        "clarifai/ai21.complete.Jurassic2-Jumbo",
+        "clarifai/ai21.complete.Jurassic2-Large",
+        "clarifai/cohere.generate.cohere-generate-command",
+        "clarifai/wizardlm.generate.wizardCoder-Python-34B",
+        "clarifai/wizardlm.generate.wizardLM-70B",
+        "clarifai/tiiuae.falcon.falcon-40b-instruct",
+        "clarifai/togethercomputer.RedPajama.RedPajama-INCITE-7B-Chat",
+        "clarifai/gcp.generate.code-gecko",
+        "clarifai/gcp.generate.code-bison",
+        "clarifai/mistralai.completion.mistral-7B-OpenOrca",
+        "clarifai/mistralai.completion.openHermes-2-mistral-7B",
+        "clarifai/wizardlm.generate.wizardLM-13B",
+        "clarifai/huggingface-research.zephyr.zephyr-7B-alpha",
+        "clarifai/wizardlm.generate.wizardCoder-15B",
+        "clarifai/microsoft.text-generation.phi-1_5",
+        "clarifai/databricks.Dolly-v2.dolly-v2-12b",
+        "clarifai/bigcode.code.StarCoder",
+        "clarifai/salesforce.xgen.xgen-7b-8k-instruct",
+        "clarifai/mosaicml.mpt.mpt-7b-instruct",
+        "clarifai/anthropic.completion.claude-3-opus",
+        "clarifai/anthropic.completion.claude-3-sonnet",
+        "clarifai/gcp.generate.gemini-1_5-pro",
+        "clarifai/gcp.generate.imagen-2",
+        "clarifai/salesforce.blip.general-english-image-caption-blip-2",
+    ]
+)
 
 
-huggingface_models: set = set([
-    "meta-llama/Llama-2-7b-hf",
-    "meta-llama/Llama-2-7b-chat-hf",
-    "meta-llama/Llama-2-13b-hf",
-    "meta-llama/Llama-2-13b-chat-hf",
-    "meta-llama/Llama-2-70b-hf",
-    "meta-llama/Llama-2-70b-chat-hf",
-    "meta-llama/Llama-2-7b",
-    "meta-llama/Llama-2-7b-chat",
-    "meta-llama/Llama-2-13b",
-    "meta-llama/Llama-2-13b-chat",
-    "meta-llama/Llama-2-70b",
-    "meta-llama/Llama-2-70b-chat",
-])  # these have been tested on extensively. But by default all text2text-generation and text-generation models are supported by liteLLM. - https://docs.litellm.ai/docs/providers
-empower_models = set([
-    "empower/empower-functions",
-    "empower/empower-functions-small",
-])
+huggingface_models: set = set(
+    [
+        "meta-llama/Llama-2-7b-hf",
+        "meta-llama/Llama-2-7b-chat-hf",
+        "meta-llama/Llama-2-13b-hf",
+        "meta-llama/Llama-2-13b-chat-hf",
+        "meta-llama/Llama-2-70b-hf",
+        "meta-llama/Llama-2-70b-chat-hf",
+        "meta-llama/Llama-2-7b",
+        "meta-llama/Llama-2-7b-chat",
+        "meta-llama/Llama-2-13b",
+        "meta-llama/Llama-2-13b-chat",
+        "meta-llama/Llama-2-70b",
+        "meta-llama/Llama-2-70b-chat",
+    ]
+)  # these have been tested on extensively. But by default all text2text-generation and text-generation models are supported by liteLLM. - https://docs.litellm.ai/docs/providers
+empower_models = set(
+    [
+        "empower/empower-functions",
+        "empower/empower-functions-small",
+    ]
+)
 
-together_ai_models: set = set([
-    # llama llms - chat
-    "togethercomputer/llama-2-70b-chat",
-    # llama llms - language / instruct
-    "togethercomputer/llama-2-70b",
-    "togethercomputer/LLaMA-2-7B-32K",
-    "togethercomputer/Llama-2-7B-32K-Instruct",
-    "togethercomputer/llama-2-7b",
-    # falcon llms
-    "togethercomputer/falcon-40b-instruct",
-    "togethercomputer/falcon-7b-instruct",
-    # alpaca
-    "togethercomputer/alpaca-7b",
-    # chat llms
-    "HuggingFaceH4/starchat-alpha",
-    # code llms
-    "togethercomputer/CodeLlama-34b",
-    "togethercomputer/CodeLlama-34b-Instruct",
-    "togethercomputer/CodeLlama-34b-Python",
-    "defog/sqlcoder",
-    "NumbersStation/nsql-llama-2-7B",
-    "WizardLM/WizardCoder-15B-V1.0",
-    "WizardLM/WizardCoder-Python-34B-V1.0",
-    # language llms
-    "NousResearch/Nous-Hermes-Llama2-13b",
-    "Austism/chronos-hermes-13b",
-    "upstage/SOLAR-0-70b-16bit",
-    "WizardLM/WizardLM-70B-V1.0",
-])
-  # supports all together ai models, just pass in the model id e.g. completion(model="together_computer/replit_code_3b",...)
+together_ai_models: set = set(
+    [
+        # llama llms - chat
+        "togethercomputer/llama-2-70b-chat",
+        # llama llms - language / instruct
+        "togethercomputer/llama-2-70b",
+        "togethercomputer/LLaMA-2-7B-32K",
+        "togethercomputer/Llama-2-7B-32K-Instruct",
+        "togethercomputer/llama-2-7b",
+        # falcon llms
+        "togethercomputer/falcon-40b-instruct",
+        "togethercomputer/falcon-7b-instruct",
+        # alpaca
+        "togethercomputer/alpaca-7b",
+        # chat llms
+        "HuggingFaceH4/starchat-alpha",
+        # code llms
+        "togethercomputer/CodeLlama-34b",
+        "togethercomputer/CodeLlama-34b-Instruct",
+        "togethercomputer/CodeLlama-34b-Python",
+        "defog/sqlcoder",
+        "NumbersStation/nsql-llama-2-7B",
+        "WizardLM/WizardCoder-15B-V1.0",
+        "WizardLM/WizardCoder-Python-34B-V1.0",
+        # language llms
+        "NousResearch/Nous-Hermes-Llama2-13b",
+        "Austism/chronos-hermes-13b",
+        "upstage/SOLAR-0-70b-16bit",
+        "WizardLM/WizardLM-70B-V1.0",
+    ]
+)
+# supports all together ai models, just pass in the model id e.g. completion(model="together_computer/replit_code_3b",...)
 
 
-baseten_models: set = set([
-    "qvv0xeq",
-    "q841o8w",
-    "31dxrj3",
-])  # FALCON 7B  # WizardLM  # Mosaic ML
+baseten_models: set = set(
+    [
+        "qvv0xeq",
+        "q841o8w",
+        "31dxrj3",
+    ]
+)  # FALCON 7B  # WizardLM  # Mosaic ML
 
-featherless_ai_models: set = set([
-    "featherless-ai/Qwerky-72B",
-    "featherless-ai/Qwerky-QwQ-32B",
-    "Qwen/Qwen2.5-72B-Instruct",
-    "all-hands/openhands-lm-32b-v0.1",
-    "Qwen/Qwen2.5-Coder-32B-Instruct",
-    "deepseek-ai/DeepSeek-V3-0324",
-    "mistralai/Mistral-Small-24B-Instruct-2501",
-    "mistralai/Mistral-Nemo-Instruct-2407",
-    "ProdeusUnity/Stellar-Odyssey-12b-v0.0",
-])
+featherless_ai_models: set = set(
+    [
+        "featherless-ai/Qwerky-72B",
+        "featherless-ai/Qwerky-QwQ-32B",
+        "Qwen/Qwen2.5-72B-Instruct",
+        "all-hands/openhands-lm-32b-v0.1",
+        "Qwen/Qwen2.5-Coder-32B-Instruct",
+        "deepseek-ai/DeepSeek-V3-0324",
+        "mistralai/Mistral-Small-24B-Instruct-2501",
+        "mistralai/Mistral-Nemo-Instruct-2407",
+        "ProdeusUnity/Stellar-Odyssey-12b-v0.0",
+    ]
+)
 
-nebius_models: set = set([
-    "Qwen/Qwen3-235B-A22B",
-    "Qwen/Qwen3-30B-A3B-fast",
-    "Qwen/Qwen3-32B",
-    "Qwen/Qwen3-14B",
-    "nvidia/Llama-3_1-Nemotron-Ultra-253B-v1",
-    "deepseek-ai/DeepSeek-V3-0324",
-    "deepseek-ai/DeepSeek-V3-0324-fast",
-    "deepseek-ai/DeepSeek-R1",
-    "deepseek-ai/DeepSeek-R1-fast",
-    "meta-llama/Llama-3.3-70B-Instruct-fast",
-    "Qwen/Qwen2.5-32B-Instruct-fast",
-    "Qwen/Qwen2.5-Coder-32B-Instruct-fast",
-])
+nebius_models: set = set(
+    [
+        # deepseek models
+        "deepseek-ai/DeepSeek-R1-0528",
+        "deepseek-ai/DeepSeek-V3-0324",
+        "deepseek-ai/DeepSeek-V3",
+        "deepseek-ai/DeepSeek-R1",
+        "deepseek-ai/DeepSeek-R1-Distill-Llama-70B",
+        # google models
+        "google/gemma-2-2b-it",
+        "google/gemma-2-9b-it-fast",
+        # llama models
+        "meta-llama/Llama-3.3-70B-Instruct",
+        "meta-llama/Meta-Llama-3.1-70B-Instruct",
+        "meta-llama/Meta-Llama-3.1-8B-Instruct",
+        "meta-llama/Meta-Llama-3.1-405B-Instruct",
+        "NousResearch/Hermes-3-Llama-405B",
+        # microsoft models
+        "microsoft/phi-4",
+        # mistral models
+        "mistralai/Mistral-Nemo-Instruct-2407",
+        "mistralai/Devstral-Small-2505",
+        # moonshot models
+        "moonshotai/Kimi-K2-Instruct",
+        # nvidia models
+        "nvidia/Llama-3_1-Nemotron-Ultra-253B-v1",
+        "nvidia/Llama-3_3-Nemotron-Super-49B-v1",
+        # openai models
+        "openai/gpt-oss-120b",
+        "openai/gpt-oss-20b",
+        # qwen models
+        "Qwen/Qwen3-Coder-480B-A35B-Instruct",
+        "Qwen/Qwen3-235B-A22B-Instruct-2507",
+        "Qwen/Qwen3-235B-A22B",
+        "Qwen/Qwen3-30B-A3B",
+        "Qwen/Qwen3-32B",
+        "Qwen/Qwen3-14B",
+        "Qwen/Qwen3-4B-fast",
+        "Qwen/Qwen2.5-Coder-7B",
+        "Qwen/Qwen2.5-Coder-32B-Instruct",
+        "Qwen/Qwen2.5-72B-Instruct",
+        "Qwen/QwQ-32B",
+        "Qwen/Qwen3-30B-A3B-Thinking-2507",
+        "Qwen/Qwen3-30B-A3B-Instruct-2507",
+        # zai models
+        "zai-org/GLM-4.5",
+        "zai-org/GLM-4.5-Air",
+        # other models
+        "aaditya/Llama3-OpenBioLLM-70B",
+        "ProdeusUnity/Stellar-Odyssey-12b-v0.0",
+        "all-hands/openhands-lm-32b-v0.1",
+    ]
+)
 
-dashscope_models: set = set([
-    "qwen-turbo",
-    "qwen-plus",
-    "qwen-max",
-    "qwen-turbo-latest",
-    "qwen-plus-latest",
-    "qwen-max-latest",
-    "qwq-32b",
-    "qwen3-235b-a22b",
-    "qwen3-32b",
-    "qwen3-30b-a3b",
-])
+dashscope_models: set = set(
+    [
+        "qwen-turbo",
+        "qwen-plus",
+        "qwen-max",
+        "qwen-turbo-latest",
+        "qwen-plus-latest",
+        "qwen-max-latest",
+        "qwq-32b",
+        "qwen3-235b-a22b",
+        "qwen3-32b",
+        "qwen3-30b-a3b",
+    ]
+)
 
-nebius_embedding_models: set = set([
-    "BAAI/bge-en-icl",
-    "BAAI/bge-multilingual-gemma2",
-    "intfloat/e5-mistral-7b-instruct",
-])
+nebius_embedding_models: set = set(
+    [
+        "BAAI/bge-en-icl",
+        "BAAI/bge-multilingual-gemma2",
+        "intfloat/e5-mistral-7b-instruct",
+    ]
+)
 
 BEDROCK_INVOKE_PROVIDERS_LITERAL = Literal[
     "cohere",
@@ -684,20 +746,24 @@ BEDROCK_INVOKE_PROVIDERS_LITERAL = Literal[
 ]
 
 open_ai_embedding_models: set = set(["text-embedding-ada-002"])
-cohere_embedding_models: set = set([
-    "embed-v4.0",
-    "embed-english-v3.0",
-    "embed-english-light-v3.0",
-    "embed-multilingual-v3.0",
-    "embed-english-v2.0",
-    "embed-english-light-v2.0",
-    "embed-multilingual-v2.0",
-])
-bedrock_embedding_models: set = set([
-    "amazon.titan-embed-text-v1",
-    "cohere.embed-english-v3",
-    "cohere.embed-multilingual-v3",
-])
+cohere_embedding_models: set = set(
+    [
+        "embed-v4.0",
+        "embed-english-v3.0",
+        "embed-english-light-v3.0",
+        "embed-multilingual-v3.0",
+        "embed-english-v2.0",
+        "embed-english-light-v2.0",
+        "embed-multilingual-v2.0",
+    ]
+)
+bedrock_embedding_models: set = set(
+    [
+        "amazon.titan-embed-text-v1",
+        "cohere.embed-english-v3",
+        "cohere.embed-multilingual-v3",
+    ]
+)
 
 known_tokenizer_config = {
     "mistralai/Mistral-7B-Instruct-v0.1": {

litellm/cost_calculator.py

@@ -57,6 +57,7 @@ from litellm.llms.vertex_ai.cost_calculator import (
     cost_per_token as google_cost_per_token,
 )
 from litellm.llms.vertex_ai.cost_calculator import cost_router as google_cost_router
+from litellm.llms.xai.cost_calculator import cost_per_token as xai_cost_per_token
 from litellm.responses.utils import ResponseAPILoggingUtils
 from litellm.types.llms.openai import (
     HttpxBinaryResponseContent,
@@ -341,6 +342,8 @@ def cost_per_token(  # noqa: PLR0915
         return deepseek_cost_per_token(model=model, usage=usage_block)
     elif custom_llm_provider == "perplexity":
         return perplexity_cost_per_token(model=model, usage=usage_block)
+    elif custom_llm_provider == "xai":
+        return xai_cost_per_token(model=model, usage=usage_block)
     else:
         model_info = _cached_get_model_info_helper(
             model=model, custom_llm_provider=custom_llm_provider
@@ -675,9 +678,9 @@ def completion_cost(  # noqa: PLR0915
                     or isinstance(completion_response, dict)
                 ):  # tts returns a custom class
                     if isinstance(completion_response, dict):
-                        usage_obj: Optional[Union[dict, Usage]] = (
-                            completion_response.get("usage", {})
-                        )
+                        usage_obj: Optional[
+                            Union[dict, Usage]
+                        ] = completion_response.get("usage", {})
                     else:
                         usage_obj = getattr(completion_response, "usage", {})
                     if isinstance(usage_obj, BaseModel) and not _is_known_usage_objects(
@@ -1279,7 +1282,9 @@ class BaseTokenUsageProcessor:
                     not hasattr(combined, "completion_tokens_details")
                     or not combined.completion_tokens_details
                 ):
-                    combined.completion_tokens_details = CompletionTokensDetailsWrapper()
+                    combined.completion_tokens_details = (
+                        CompletionTokensDetailsWrapper()
+                    )
 
                 # Check what keys exist in the model's completion_tokens_details
                 for attr in usage.completion_tokens_details.model_fields:

litellm/exceptions.py

@@ -153,6 +153,29 @@ class BadRequestError(openai.BadRequestError):  # type: ignore
             _message += f", LiteLLM Max Retries: {self.max_retries}"
         return _message
 
+class ImageFetchError(BadRequestError):
+    def __init__(
+        self,
+        message,
+        model=None,
+        llm_provider=None,
+        response: Optional[httpx.Response] = None,
+        litellm_debug_info: Optional[str] = None,
+        max_retries: Optional[int] = None,
+        num_retries: Optional[int] = None,
+        body: Optional[dict] = None,
+    ):
+        super().__init__(
+            message=message,
+            model=model,
+            llm_provider=llm_provider,
+            response=response,
+            litellm_debug_info=litellm_debug_info,
+            max_retries=max_retries,
+            num_retries=num_retries,
+            body=body,
+        )
+
 
 class UnprocessableEntityError(openai.UnprocessableEntityError):  # type: ignore
     def __init__(

litellm/google_genai/adapters/handler.py

@@ -37,6 +37,10 @@ class GenerateContentToCompletionHandler:
 
         completion_kwargs: Dict[str, Any] = dict(completion_request)
 
+        # feed metadata for custom callback
+        if extra_kwargs is not None and "metadata" in extra_kwargs:
+            completion_kwargs["metadata"] = extra_kwargs["metadata"]
+
         if stream:
             completion_kwargs["stream"] = stream
 

litellm/images/main.py

@@ -1,7 +1,7 @@
 import asyncio
 import contextvars
 from functools import partial
-from typing import Any, Coroutine, Dict, Literal, Optional, Union, cast, overload
+from typing import Any, Coroutine, Dict, List, Literal, Optional, Union, cast, overload
 
 import httpx
 
@@ -347,6 +347,7 @@ def image_generation(  # noqa: PLR0915
                 raise ValueError(f"image generation config is not supported for {custom_llm_provider}")
             
             return llm_http_handler.image_generation_handler(
+                api_key=api_key,
                 model=model,
                 prompt=prompt,
                 image_generation_provider_config=image_generation_config,
@@ -676,7 +677,7 @@ def image_variation(
 
 @client
 def image_edit(
-    image: FileTypes,
+    image: Union[FileTypes, List[FileTypes]],
     prompt: str,
     model: Optional[str] = None,
     mask: Optional[str] = None,
@@ -704,6 +705,9 @@ def image_edit(
         litellm_call_id: Optional[str] = kwargs.get("litellm_call_id", None)
         _is_async = kwargs.pop("async_call", False) is True
 
+        #add images / or return a single image
+        images = image if isinstance(image, list) else [image]
+
         # get llm provider logic
         litellm_params = GenericLiteLLMParams(**kwargs)
         model, custom_llm_provider, _, _ = get_llm_provider(
@@ -752,7 +756,7 @@ def image_edit(
         # Call the handler with _is_async flag instead of directly calling the async handler
         return base_llm_http_handler.image_edit_handler(
             model=model,
-            image=image,
+            image=images,
             prompt=prompt,
             image_edit_provider_config=image_edit_provider_config,
             image_edit_optional_request_params=image_edit_request_params,
@@ -778,7 +782,7 @@ def image_edit(
 
 @client
 async def aimage_edit(
-    image: FileTypes,
+    image: Union[FileTypes, List[FileTypes]],
     model: str,
     prompt: str,
     mask: Optional[str] = None,
@@ -818,9 +822,11 @@ async def aimage_edit(
                 model=model, api_base=local_vars.get("base_url", None)
             )
 
+        images = image if isinstance(image, list) else [image]
+
         func = partial(
             image_edit,
-            image=image,
+            image=images,
             prompt=prompt,
             mask=mask,
             model=model,

litellm/integrations/braintrust_logging.py

@@ -1,13 +1,11 @@
 # What is this?
 ## Log success + failure events to Braintrust
 
-import copy
 import os
 from datetime import datetime
 from typing import Dict, Optional
 
 import httpx
-from pydantic import BaseModel
 
 import litellm
 from litellm import verbose_logger
@@ -24,7 +22,6 @@ API_BASE = "https://api.braintrustdata.com/v1"
 
 def get_utc_datetime():
     import datetime as dt
-    from datetime import datetime
 
     if hasattr(dt, "UTC"):
         return datetime.now(dt.UTC)  # type: ignore
@@ -45,9 +42,9 @@ class BraintrustLogger(CustomLogger):
             "Authorization": "Bearer " + self.api_key,
             "Content-Type": "application/json",
         }
-        self._project_id_cache: Dict[
-            str, str
-        ] = {}  # Cache mapping project names to IDs
+        self._project_id_cache: Dict[str, str] = (
+            {}
+        )  # Cache mapping project names to IDs
         self.global_braintrust_http_handler = get_async_httpx_client(
             llm_provider=httpxSpecialProvider.LoggingCallback
         )
@@ -108,43 +105,6 @@ class BraintrustLogger(CustomLogger):
         except httpx.HTTPStatusError as e:
             raise Exception(f"Failed to register project: {e.response.text}")
 
-    @staticmethod
-    def add_metadata_from_header(litellm_params: dict, metadata: dict) -> dict:
-        """
-        Adds metadata from proxy request headers to Braintrust logging if keys start with "braintrust_"
-        and overwrites litellm_params.metadata if already included.
-
-        For example if you want to append your trace to an existing `trace_id` via header, send
-        `headers: { ..., langfuse_existing_trace_id: your-existing-trace-id }` via proxy request.
-        """
-        if litellm_params is None:
-            return metadata
-
-        if litellm_params.get("proxy_server_request") is None:
-            return metadata
-
-        if metadata is None:
-            metadata = {}
-
-        proxy_headers = (
-            litellm_params.get("proxy_server_request", {}).get("headers", {}) or {}
-        )
-
-        for metadata_param_key in proxy_headers:
-            if metadata_param_key.startswith("braintrust"):
-                trace_param_key = metadata_param_key.replace("braintrust", "", 1)
-                if trace_param_key in metadata:
-                    verbose_logger.warning(
-                        f"Overwriting Braintrust `{trace_param_key}` from request header"
-                    )
-                else:
-                    verbose_logger.debug(
-                        f"Found Braintrust `{trace_param_key}` in request header"
-                    )
-                metadata[trace_param_key] = proxy_headers.get(metadata_param_key)
-
-        return metadata
-
     async def create_default_project_and_experiment(self):
         project = await self.global_braintrust_http_handler.post(
             f"{self.api_base}/project", headers=self.headers, json={"name": "litellm"}
@@ -169,7 +129,9 @@ class BraintrustLogger(CustomLogger):
         verbose_logger.debug("REACHES BRAINTRUST SUCCESS")
         try:
             litellm_call_id = kwargs.get("litellm_call_id")
+            standard_logging_object = kwargs.get("standard_logging_object", {})
             prompt = {"messages": kwargs.get("messages")}
+
             output = None
             choices = []
             if response_obj is not None and (
@@ -192,33 +154,13 @@ class BraintrustLogger(CustomLogger):
             ):
                 output = response_obj["data"]
 
-            litellm_params = kwargs.get("litellm_params", {})
-            metadata = (
-                litellm_params.get("metadata", {}) or {}
-            )  # if litellm_params['metadata'] == None
-            metadata = self.add_metadata_from_header(litellm_params, metadata)
-            clean_metadata = {}
-            try:
-                metadata = copy.deepcopy(
-                    metadata
-                )  # Avoid modifying the original metadata
-            except Exception:
-                new_metadata = {}
-                for key, value in metadata.items():
-                    if (
-                        isinstance(value, list)
-                        or isinstance(value, dict)
-                        or isinstance(value, str)
-                        or isinstance(value, int)
-                        or isinstance(value, float)
-                    ):
-                        new_metadata[key] = copy.deepcopy(value)
-                metadata = new_metadata
+            litellm_params = kwargs.get("litellm_params", {}) or {}
+            dynamic_metadata = litellm_params.get("metadata", {}) or {}
 
             # Get project_id from metadata or create default if needed
-            project_id = metadata.get("project_id")
+            project_id = dynamic_metadata.get("project_id")
             if project_id is None:
-                project_name = metadata.get("project_name")
+                project_name = dynamic_metadata.get("project_name")
                 project_id = (
                     self.get_project_id_sync(project_name) if project_name else None
                 )
@@ -229,8 +171,9 @@ class BraintrustLogger(CustomLogger):
                 project_id = self.default_project_id
 
             tags = []
-            if isinstance(metadata, dict):
-                for key, value in metadata.items():
+
+            if isinstance(dynamic_metadata, dict):
+                for key, value in dynamic_metadata.items():
                     # generate langfuse tags - Default Tags sent to Langfuse from LiteLLM Proxy
                     if (
                         litellm.langfuse_default_tags is not None
@@ -239,25 +182,12 @@ class BraintrustLogger(CustomLogger):
                     ):
                         tags.append(f"{key}:{value}")
 
-                    # clean litellm metadata before logging
-                    if key in [
-                        "headers",
-                        "endpoint",
-                        "caching_groups",
-                        "previous_models",
-                    ]:
-                        continue
-                    else:
-                        clean_metadata[key] = value
+                    if (
+                        isinstance(value, str) and key not in standard_logging_object
+                    ):  # support logging dynamic metadata to braintrust
+                        standard_logging_object[key] = value
 
             cost = kwargs.get("response_cost", None)
-            if cost is not None:
-                clean_metadata["litellm_response_cost"] = cost
-
-            # metadata.model is required for braintrust to calculate the "Estimated cost" metric
-            litellm_model = kwargs.get("model", None)
-            if litellm_model is not None:
-                clean_metadata["model"] = litellm_model
 
             metrics: Optional[dict] = None
             usage_obj = getattr(response_obj, "usage", None)
@@ -274,12 +204,15 @@ class BraintrustLogger(CustomLogger):
                     "end": end_time.timestamp(),
                 }
 
+            # Allow metadata override for span name
+            span_name = dynamic_metadata.get("span_name", "Chat Completion")
+
             request_data = {
                 "id": litellm_call_id,
                 "input": prompt["messages"],
-                "metadata": clean_metadata,
+                "metadata": standard_logging_object,
                 "tags": tags,
-                "span_attributes": {"name": "Chat Completion", "type": "llm"},
+                "span_attributes": {"name": span_name, "type": "llm"},
             }
             if choices is not None:
                 request_data["output"] = [choice.dict() for choice in choices]
@@ -309,6 +242,7 @@ class BraintrustLogger(CustomLogger):
         verbose_logger.debug("REACHES BRAINTRUST SUCCESS")
         try:
             litellm_call_id = kwargs.get("litellm_call_id")
+            standard_logging_object = kwargs.get("standard_logging_object", {})
             prompt = {"messages": kwargs.get("messages")}
             output = None
             choices = []
@@ -333,32 +267,12 @@ class BraintrustLogger(CustomLogger):
                 output = response_obj["data"]
 
             litellm_params = kwargs.get("litellm_params", {})
-            metadata = (
-                litellm_params.get("metadata", {}) or {}
-            )  # if litellm_params['metadata'] == None
-            metadata = self.add_metadata_from_header(litellm_params, metadata)
-            clean_metadata = {}
-            new_metadata = {}
-            for key, value in metadata.items():
-                if (
-                    isinstance(value, list)
-                    or isinstance(value, str)
-                    or isinstance(value, int)
-                    or isinstance(value, float)
-                ):
-                    new_metadata[key] = value
-                elif isinstance(value, BaseModel):
-                    new_metadata[key] = value.model_dump_json()
-                elif isinstance(value, dict):
-                    for k, v in value.items():
-                        if isinstance(v, datetime):
-                            value[k] = v.isoformat()
-                    new_metadata[key] = value
+            dynamic_metadata = litellm_params.get("metadata", {}) or {}
 
             # Get project_id from metadata or create default if needed
-            project_id = metadata.get("project_id")
+            project_id = dynamic_metadata.get("project_id")
             if project_id is None:
-                project_name = metadata.get("project_name")
+                project_name = dynamic_metadata.get("project_name")
                 project_id = (
                     await self.get_project_id_async(project_name)
                     if project_name
@@ -371,8 +285,9 @@ class BraintrustLogger(CustomLogger):
                 project_id = self.default_project_id
 
             tags = []
-            if isinstance(metadata, dict):
-                for key, value in metadata.items():
+
+            if isinstance(dynamic_metadata, dict):
+                for key, value in dynamic_metadata.items():
                     # generate langfuse tags - Default Tags sent to Langfuse from LiteLLM Proxy
                     if (
                         litellm.langfuse_default_tags is not None
@@ -381,25 +296,12 @@ class BraintrustLogger(CustomLogger):
                     ):
                         tags.append(f"{key}:{value}")
 
-                    # clean litellm metadata before logging
-                    if key in [
-                        "headers",
-                        "endpoint",
-                        "caching_groups",
-                        "previous_models",
-                    ]:
-                        continue
-                    else:
-                        clean_metadata[key] = value
+                    if (
+                        isinstance(value, str) and key not in standard_logging_object
+                    ):  # support logging dynamic metadata to braintrust
+                        standard_logging_object[key] = value
 
             cost = kwargs.get("response_cost", None)
-            if cost is not None:
-                clean_metadata["litellm_response_cost"] = cost
-
-            # metadata.model is required for braintrust to calculate the "Estimated cost" metric
-            litellm_model = kwargs.get("model", None)
-            if litellm_model is not None:
-                clean_metadata["model"] = litellm_model
 
             metrics: Optional[dict] = None
             usage_obj = getattr(response_obj, "usage", None)
@@ -426,13 +328,16 @@ class BraintrustLogger(CustomLogger):
                         - api_call_start_time.timestamp()
                     )
 
+            # Allow metadata override for span name
+            span_name = dynamic_metadata.get("span_name", "Chat Completion")
+
             request_data = {
                 "id": litellm_call_id,
                 "input": prompt["messages"],
                 "output": output,
-                "metadata": clean_metadata,
+                "metadata": standard_logging_object,
                 "tags": tags,
-                "span_attributes": {"name": "Chat Completion", "type": "llm"},
+                "span_attributes": {"name": span_name, "type": "llm"},
             }
             if choices is not None:
                 request_data["output"] = [choice.dict() for choice in choices]

litellm/integrations/langfuse/langfuse_otel.py

@@ -141,6 +141,17 @@ class LangfuseOtelLogger(OpenTelemetry):
                         value = str(value)
                 safe_set_attribute(span, enum_attr.value, value)
 
+    @staticmethod
+    def _get_langfuse_otel_host() -> Optional[str]:
+        """
+        Returns the Langfuse OTEL host based on environment variables.
+
+        Returned in the following order of precedence:
+        1. LANGFUSE_OTEL_HOST
+        2. LANGFUSE_HOST
+        """
+        return os.environ.get("LANGFUSE_OTEL_HOST") or os.environ.get("LANGFUSE_HOST")
+
     @staticmethod
     def get_langfuse_otel_config() -> LangfuseOtelConfig:
         """
@@ -166,7 +177,7 @@ class LangfuseOtelLogger(OpenTelemetry):
             )
 
         # Determine endpoint - default to US cloud
-        langfuse_host = os.environ.get("LANGFUSE_HOST", None)
+        langfuse_host = LangfuseOtelLogger._get_langfuse_otel_host()
 
         if langfuse_host:
             # If LANGFUSE_HOST is provided, construct OTEL endpoint from it

litellm/integrations/openmeter.py

@@ -66,8 +66,18 @@ class OpenMeterLogger(CustomLogger):
             }
 
         user_param = kwargs.get("user", None)  # end-user passed in via 'user' param
+        
+        # If no user provided directly, try to get it from token user_id
         if user_param is None:
-            raise Exception("OpenMeter: user is required")
+            # Check if user_id is available from the API key metadata
+            litellm_params = kwargs.get("litellm_params", {})
+            metadata = litellm_params.get("metadata", {})
+            user_api_key_user_id = metadata.get("user_api_key_user_id", None)
+            
+            if user_api_key_user_id is not None:
+                user_param = user_api_key_user_id
+            else:
+                raise Exception("OpenMeter: user is required")
         
         # Ensure subject is always a string for OpenMeter API
         subject = str(user_param)

litellm/integrations/vector_store_integrations/vector_store_pre_call_hook.py

@@ -8,6 +8,7 @@ It searches the vector store for relevant context and appends it to the messages
 from typing import TYPE_CHECKING, Dict, List, Optional, Tuple, cast
 
 import litellm
+import litellm.vector_stores
 from litellm._logging import verbose_logger
 from litellm.integrations.custom_logger import CustomLogger
 from litellm.types.llms.openai import AllMessageValues, ChatCompletionUserMessage
@@ -192,4 +193,4 @@ class VectorStorePreCallHook(CustomLogger):
             modified_messages.insert(-1, cast(AllMessageValues, context_message))
             return modified_messages
         
-        return messages 
+        return messages

litellm/litellm_core_utils/get_llm_provider_logic.py

@@ -249,6 +249,9 @@ def get_llm_provider(  # noqa: PLR0915
                     elif endpoint == "https://api.hyperbolic.xyz/v1":
                         custom_llm_provider = "hyperbolic"
                         dynamic_api_key = get_secret_str("HYPERBOLIC_API_KEY")
+                    elif endpoint == "https://ai-gateway.vercel.sh/v1":
+                        custom_llm_provider = "vercel_ai_gateway"
+                        dynamic_api_key = get_secret_str("VERCEL_AI_GATEWAY_API_KEY")
 
                     if api_base is not None and not isinstance(api_base, str):
                         raise Exception(
@@ -742,6 +745,13 @@ def _get_openai_compatible_provider_info(  # noqa: PLR0915
         ) = litellm.HyperbolicChatConfig()._get_openai_compatible_provider_info(
             api_base, api_key
         )
+    elif custom_llm_provider == "vercel_ai_gateway":
+        (
+            api_base,
+            dynamic_api_key,
+        ) = litellm.VercelAIGatewayConfig()._get_openai_compatible_provider_info(
+            api_base, api_key
+        )
     elif custom_llm_provider == "aiml":
         (
             api_base,

litellm/litellm_core_utils/get_provider_specific_headers.py

@@ -0,0 +1,23 @@
+from typing import Dict, Optional
+
+from litellm.types.utils import ProviderSpecificHeader
+
+
+class ProviderSpecificHeaderUtils:
+    @staticmethod
+    def get_provider_specific_headers(
+        provider_specific_header: Optional[ProviderSpecificHeader],
+        custom_llm_provider: Optional[str],
+    ) -> Dict:
+        """
+        Get the provider specific headers for the given custom llm provider
+
+        Returns:
+            Optional[Dict]: The provider specific headers for the given custom llm provider
+        """
+        if (
+            provider_specific_header is not None
+            and provider_specific_header.get("custom_llm_provider") == custom_llm_provider
+        ):
+            return provider_specific_header.get("extra_headers", {})
+        return {}

litellm/litellm_core_utils/get_supported_openai_params.py

@@ -131,6 +131,8 @@ def get_supported_openai_params(  # noqa: PLR0915
             return litellm.AzureOpenAIConfig().get_supported_openai_params(model=model)
     elif custom_llm_provider == "openrouter":
         return litellm.OpenrouterConfig().get_supported_openai_params(model=model)
+    elif custom_llm_provider == "vercel_ai_gateway":
+        return litellm.VercelAIGatewayConfig().get_supported_openai_params(model=model)
     elif custom_llm_provider == "mistral" or custom_llm_provider == "codestral":
         # mistal and codestral api have the exact same params
         if request_type == "chat_completion":
@@ -268,10 +270,9 @@ def get_supported_openai_params(  # noqa: PLR0915
             from litellm.llms.elevenlabs.audio_transcription.transformation import (
                 ElevenLabsAudioTranscriptionConfig,
             )
-            return (
-                ElevenLabsAudioTranscriptionConfig().get_supported_openai_params(
-                    model=model
-                )
+
+            return ElevenLabsAudioTranscriptionConfig().get_supported_openai_params(
+                model=model
             )
     elif custom_llm_provider in litellm._custom_providers:
         if request_type == "chat_completion":

litellm/litellm_core_utils/llm_cost_calc/tool_call_cost_tracking.py

@@ -580,7 +580,9 @@ class StandardBuiltInToolCostTracking:
             return WebSearchOptions(**kwargs.get("web_search_options", {}))
 
         tools = StandardBuiltInToolCostTracking._get_tools_from_kwargs(
-            kwargs, "web_search_preview"
+            kwargs=kwargs, tool_type="web_search_preview"
+        ) or StandardBuiltInToolCostTracking._get_tools_from_kwargs(
+            kwargs=kwargs, tool_type="web_search"
         )
         if tools:
             # Look for web search tool in the tools array
@@ -612,6 +614,8 @@ class StandardBuiltInToolCostTracking:
     def _is_web_search_tool_call(tool: Dict) -> bool:
         if tool.get("type", None) == "web_search_preview":
             return True
+        if tool.get("type", None) == "web_search":
+            return True
         if "search_context_size" in tool:
             return True
         return False

litellm/litellm_core_utils/logging_worker.py

@@ -57,9 +57,10 @@ class LoggingWorker:
                 finally:
                     self._queue.task_done()
                     
-        except asyncio.CancelledError as e:
-            verbose_logger.exception(f"LoggingWorker cancelled: {e}")
-            pass
+        except asyncio.CancelledError:
+            verbose_logger.debug("LoggingWorker cancelled during shutdown")
+            # Attempt to clear remaining items to prevent "never awaited" warnings
+            await self.clear_queue()
     
     def enqueue(self, coroutine: Coroutine) -> None:
         """

litellm/litellm_core_utils/prompt_templates/factory.py

@@ -3846,7 +3846,13 @@ def function_call_prompt(messages: list, functions: list):
     function_added_to_prompt = False
     for message in messages:
         if "system" in message["role"]:
-            message["content"] += f""" {function_prompt}"""
+            if isinstance(message["content"], str):
+                message["content"] += f""" {function_prompt}"""
+            else:
+                message["content"].append({
+                    "type": "text",
+                    "text": f""" {function_prompt}"""
+                })
             function_added_to_prompt = True
 
     if function_added_to_prompt is False:

litellm/litellm_core_utils/prompt_templates/image_handling.py

@@ -17,7 +17,7 @@ in_memory_cache = InMemoryCache(max_size_in_memory=MAX_IMGS_IN_MEMORY)
 
 def _process_image_response(response: Response, url: str) -> str:
     if response.status_code != 200:
-        raise Exception(
+        raise litellm.ImageFetchError(
             f"Error: Unable to fetch image from URL. Status code: {response.status_code}, url={url}"
         )
 
@@ -57,9 +57,11 @@ async def async_convert_url_to_base64(url: str) -> str:
         try:
             response = await client.get(url, follow_redirects=True)
             return _process_image_response(response, url)
+        except litellm.ImageFetchError:
+            raise
         except Exception:
             pass
-    raise Exception(
+    raise litellm.ImageFetchError(
         f"Error: Unable to fetch image from URL after 3 attempts. url={url}"
     )
 
@@ -74,10 +76,11 @@ def convert_url_to_base64(url: str) -> str:
         try:
             response = client.get(url, follow_redirects=True)
             return _process_image_response(response, url)
+        except litellm.ImageFetchError:
+            raise
         except Exception as e:
             verbose_logger.exception(e)
-            # print(e)
             pass
-    raise Exception(
-        f"Error: Unable to fetch image from URL after 3 attempts. url={url}"
+    raise litellm.ImageFetchError(
+        f"Error: Unable to fetch image from URL after 3 attempts. url={url}",
     )

litellm/litellm_core_utils/safe_json_dumps.py

@@ -1,5 +1,6 @@
 import json
 from typing import Any, Union
+
 from litellm.constants import DEFAULT_MAX_RECURSE_DEPTH
 
 

litellm/litellm_core_utils/streaming_handler.py

@@ -20,7 +20,9 @@ from litellm.litellm_core_utils.redact_messages import LiteLLMLoggingObject
 from litellm.litellm_core_utils.thread_pool_executor import executor
 from litellm.types.llms.openai import ChatCompletionChunk
 from litellm.types.router import GenericLiteLLMParams
-from litellm.types.utils import Delta
+from litellm.types.utils import (
+    Delta,
+)
 from litellm.types.utils import GenericStreamingChunk as GChunk
 from litellm.types.utils import (
     ModelResponse,
@@ -35,6 +37,12 @@ from .exception_mapping_utils import exception_type
 from .llm_response_utils.get_api_base import get_api_base
 from .rules import Rules
 
+# Constants for special delta attribute names
+AUDIO_ATTRIBUTE = "audio"
+IMAGE_ATTRIBUTE = "image"
+TOOL_CALLS_ATTRIBUTE = "tool_calls"
+FUNCTION_CALL_ATTRIBUTE = "function_call"
+
 
 def is_async_iterable(obj: Any) -> bool:
     """
@@ -766,6 +774,66 @@ class CustomStreamWrapper:
             model_response.choices[0].delta = Delta(**_initial_delta)
         return model_response
 
+    def _has_special_delta_content(self, model_response: ModelResponseStream) -> bool:
+        """
+        Check if the delta contains special content types (tool_calls, function_call, audio, or image).
+        """
+        if len(model_response.choices) == 0:
+            return False
+            
+        delta = model_response.choices[0].delta
+        
+        # Check for tool_calls or function_call
+        if getattr(delta, TOOL_CALLS_ATTRIBUTE, None) is not None or getattr(delta, FUNCTION_CALL_ATTRIBUTE, None) is not None:
+            return True
+            
+        # Check for audio
+        if hasattr(delta, AUDIO_ATTRIBUTE) and getattr(delta, AUDIO_ATTRIBUTE, None) is not None:
+            return True
+            
+        # Check for image
+        if hasattr(delta, IMAGE_ATTRIBUTE) and getattr(delta, IMAGE_ATTRIBUTE, None) is not None:
+            return True
+            
+        return False
+
+    def _handle_special_delta_content(self, model_response: ModelResponseStream) -> ModelResponseStream:
+        """
+        Handle special delta content types by stripping role and returning the response.
+        """
+        return self.strip_role_from_delta(model_response)
+
+    def _has_special_delta_attribute(self, delta, attribute_name: str) -> bool:
+        """
+        Check if delta has a specific attribute and it's not None.
+        """
+        return delta is not None and getattr(delta, attribute_name, None) is not None
+
+    def _copy_delta_attribute(self, source_delta, target_delta, attribute_name: str) -> None:
+        """
+        Copy a specific attribute from source delta to target delta.
+        """
+        setattr(target_delta, attribute_name, getattr(source_delta, attribute_name))
+
+    def _has_any_special_delta_attributes(self, delta) -> bool:
+        """
+        Check if delta has any special attributes (audio, image).
+        """
+        special_attributes = [AUDIO_ATTRIBUTE, IMAGE_ATTRIBUTE]
+        for attribute in special_attributes:
+            if self._has_special_delta_attribute(delta, attribute):
+                return True
+        return False
+
+    def _handle_special_delta_attributes(self, delta, model_response: "ModelResponseStream") -> None:
+        """
+        Handle special delta attributes (audio, image) by copying them to model_response.
+        """
+        special_attributes = [AUDIO_ATTRIBUTE, IMAGE_ATTRIBUTE]
+        for attribute in special_attributes:
+            if self._has_special_delta_attribute(delta, attribute):
+                self._copy_delta_attribute(delta, model_response.choices[0].delta, attribute)
+
     def return_processed_chunk_logic(  # noqa
         self,
         completion_obj: Dict[str, Any],
@@ -888,20 +956,8 @@ class CustomStreamWrapper:
                 self.sent_last_chunk = True
 
             return model_response
-        elif (
-            model_response.choices[0].delta.tool_calls is not None
-            or model_response.choices[0].delta.function_call is not None
-        ):
-            model_response = self.strip_role_from_delta(model_response)
-
-            return model_response
-        elif (
-            len(model_response.choices) > 0
-            and hasattr(model_response.choices[0].delta, "audio")
-            and model_response.choices[0].delta.audio is not None
-        ):
-            model_response = self.strip_role_from_delta(model_response)
-            return model_response
+        elif self._has_special_delta_content(model_response):
+            return self._handle_special_delta_content(model_response)
         else:
             if hasattr(model_response, "usage"):
                 self.chunks.append(model_response)
@@ -1374,10 +1430,8 @@ class CustomStreamWrapper:
                                 )
                             )
                             model_response.choices[0].delta = Delta()
-                    elif (
-                        delta is not None and getattr(delta, "audio", None) is not None
-                    ):
-                        model_response.choices[0].delta.audio = delta.audio
+                    elif self._has_any_special_delta_attributes(delta):
+                        self._handle_special_delta_attributes(delta, model_response)
                     else:
                         try:
                             delta = (

litellm/litellm_core_utils/token_counter.py

@@ -529,7 +529,7 @@ def _get_count_function(
                 encoding = tiktoken.get_encoding("cl100k_base")
 
             def count_tokens(text: str) -> int:
-                return len(encoding.encode(text))
+                return len(encoding.encode(text, disallowed_special=()))
 
         else:
             raise ValueError("Unsupported tokenizer type")

litellm/llms/bedrock/base_aws_llm.py

@@ -202,6 +202,7 @@ class BaseAWSLLM:
                 credentials, _cache_ttl = self._auth_with_aws_role(
                     aws_access_key_id=aws_access_key_id,
                     aws_secret_access_key=aws_secret_access_key,
+                    aws_session_token=aws_session_token,
                     aws_role_name=aws_role_name,
                     aws_session_name=aws_session_name,
                 )
@@ -554,6 +555,7 @@ class BaseAWSLLM:
         self,
         aws_access_key_id: Optional[str],
         aws_secret_access_key: Optional[str],
+        aws_session_token: Optional[str],
         aws_role_name: str,
         aws_session_name: str,
     ) -> Tuple[Credentials, Optional[int]]:
@@ -614,6 +616,7 @@ class BaseAWSLLM:
                     "sts",
                     aws_access_key_id=aws_access_key_id,
                     aws_secret_access_key=aws_secret_access_key,
+                    aws_session_token=aws_session_token,
                 )
 
         sts_response = sts_client.assume_role(

litellm/llms/custom_httpx/llm_http_handler.py

@@ -1257,6 +1257,10 @@ class BaseLLMHTTPHandler:
         stream: Optional[bool] = False,
         kwargs: Optional[Dict[str, Any]] = None,
     ) -> Union[AnthropicMessagesResponse, AsyncIterator]:
+        from litellm.litellm_core_utils.get_provider_specific_headers import (
+            ProviderSpecificHeaderUtils,
+        )
+
         if client is None or not isinstance(client, AsyncHTTPHandler):
             async_httpx_client = get_async_httpx_client(
                 llm_provider=litellm.LlmProviders.ANTHROPIC
@@ -1270,10 +1274,9 @@ class BaseLLMHTTPHandler:
             Optional[litellm.types.utils.ProviderSpecificHeader],
             kwargs.get("provider_specific_header", None),
         )
-        extra_headers = (
-            provider_specific_header.get("extra_headers", {})
-            if provider_specific_header
-            else {}
+        extra_headers = ProviderSpecificHeaderUtils.get_provider_specific_headers(
+            provider_specific_header=provider_specific_header,
+            custom_llm_provider=custom_llm_provider,
         )
         (
             headers,
@@ -2678,6 +2681,7 @@ class BaseLLMHTTPHandler:
         _is_async: bool = False,
         fake_stream: bool = False,
         litellm_metadata: Optional[Dict[str, Any]] = None,
+        api_key: Optional[str] = None,
     ) -> Union[
         ImageResponse,
         Coroutine[Any, Any, ImageResponse],
@@ -2702,6 +2706,7 @@ class BaseLLMHTTPHandler:
                 client=client if isinstance(client, AsyncHTTPHandler) else None,
                 fake_stream=fake_stream,
                 litellm_metadata=litellm_metadata,
+                api_key=api_key,
             )
 
         if client is None or not isinstance(client, HTTPHandler):
@@ -2712,7 +2717,7 @@ class BaseLLMHTTPHandler:
             sync_httpx_client = client
 
         headers = image_generation_provider_config.validate_environment(
-            api_key=litellm_params.get("api_key", None),
+            api_key=api_key,
             headers=image_generation_optional_request_params.get("extra_headers", {})
             or {},
             model=model,
@@ -2795,6 +2800,7 @@ class BaseLLMHTTPHandler:
         client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None,
         fake_stream: bool = False,
         litellm_metadata: Optional[Dict[str, Any]] = None,
+        api_key: Optional[str] = None,
     ) -> ImageResponse:
         """
         Async version of the image generation handler.
@@ -2809,7 +2815,7 @@ class BaseLLMHTTPHandler:
             async_httpx_client = client
 
         headers = image_generation_provider_config.validate_environment(
-            api_key=litellm_params.get("api_key", None),
+            api_key=api_key,
             headers=image_generation_optional_request_params.get("extra_headers", {})
             or {},
             model=model,

litellm/llms/deepinfra/chat/transformation.py

@@ -12,6 +12,9 @@ class DeepInfraConfig(OpenAIGPTConfig):
 
     The class `DeepInfra` provides configuration for the DeepInfra's Chat Completions API interface. Below are the parameters:
     """
+    @property
+    def custom_llm_provider(self) -> Optional[str]:
+        return "deepinfra"
 
     frequency_penalty: Optional[int] = None
     function_call: Optional[Union[str, dict]] = None
@@ -53,7 +56,7 @@ class DeepInfraConfig(OpenAIGPTConfig):
         return super().get_config()
 
     def get_supported_openai_params(self, model: str):
-        return [
+        supported_openai_params = [
             "stream",
             "frequency_penalty",
             "function_call",
@@ -68,9 +71,16 @@ class DeepInfraConfig(OpenAIGPTConfig):
             "top_p",
             "response_format",
             "tools",
-            "tool_choice",
+            "tool_choice"
         ]
 
+        if litellm.supports_reasoning(
+            model=model,
+            custom_llm_provider=self.custom_llm_provider,
+        ):
+            supported_openai_params.append("reasoning_effort")
+        return supported_openai_params
+
     def map_openai_params(
         self,
         non_default_params: dict,

litellm/llms/github_copilot/chat/transformation.py

@@ -75,8 +75,36 @@ class GithubCopilotConfig(OpenAIConfig):
         initiator = self._determine_initiator(messages)
         validated_headers["X-Initiator"] = initiator
 
+        # Add Copilot-Vision-Request header if request contains images
+        if self._has_vision_content(messages):
+            validated_headers["Copilot-Vision-Request"] = "true"
+
         return validated_headers
 
+    def get_supported_openai_params(self, model: str) -> list:
+        """
+        Get supported OpenAI parameters for GitHub Copilot.
+
+        For Claude models that support extended thinking (Claude 4 family and Claude 3-7), includes thinking and reasoning_effort parameters.
+        For other models, returns standard OpenAI parameters (which may include reasoning_effort for o-series models).
+        """
+        from litellm.utils import supports_reasoning
+        
+        # Get base OpenAI parameters
+        base_params = super().get_supported_openai_params(model)
+
+        # Add Claude-specific parameters for models that support extended thinking
+        if "claude" in model.lower() and supports_reasoning(
+            model=model.lower(),
+        ):
+            if "thinking" not in base_params:
+                base_params.append("thinking")
+            # reasoning_effort is not included by parent for Claude models, so add it
+            if "reasoning_effort" not in base_params:
+                base_params.append("reasoning_effort")
+
+        return base_params
+
     def _determine_initiator(self, messages: List[AllMessageValues]) -> str:
         """
         Determine if request is user or agent initiated based on message roles.
@@ -87,3 +115,27 @@ class GithubCopilotConfig(OpenAIConfig):
             if role in ["tool", "assistant"]:
                 return "agent"
         return "user"
+
+    def _has_vision_content(self, messages: List[AllMessageValues]) -> bool:
+        """
+        Check if any message contains vision content (images).
+        Returns True if any message has content with vision-related types, otherwise False.
+        
+        Checks for:
+        - image_url content type (OpenAI format)
+        - Content items with type 'image_url'
+        """
+        for message in messages:
+            content = message.get("content")
+            if isinstance(content, list):
+                # Check if any content item indicates vision content
+                for content_item in content:
+                    if isinstance(content_item, dict):
+                        # Check for image_url field (direct image URL)
+                        if "image_url" in content_item:
+                            return True
+                        # Check for type field indicating image content
+                        content_type = content_item.get("type")
+                        if content_type == "image_url":
+                            return True
+        return False

litellm/llms/github_copilot/common_utils.py

@@ -28,7 +28,6 @@ class GithubCopilotError(BaseLLMException):
         )
 
 
-
 class GetDeviceCodeError(GithubCopilotError):
     pass
 

litellm/llms/oci/chat/transformation.py

@@ -92,6 +92,22 @@ def load_private_key_from_str(key_str: str):
     return key
 
 
+def load_private_key_from_file(file_path: str):
+    """Loads a private key from a file path"""
+    try:
+        with open(file_path, "r", encoding="utf-8") as f:
+            key_str = f.read().strip()
+    except FileNotFoundError:
+        raise FileNotFoundError(f"Private key file not found: {file_path}")
+    except OSError as e:
+        raise OSError(f"Failed to read private key file '{file_path}': {e}") from e
+
+    if not key_str:
+        raise ValueError(f"Private key file is empty: {file_path}")
+
+    return load_private_key_from_str(key_str)
+
+
 def get_vendor_from_model(model: str) -> OCIVendors:
     """
     Extracts the vendor from the model name.
@@ -237,10 +253,17 @@ class OCIChatConfig(BaseConfig):
         oci_fingerprint = optional_params.get("oci_fingerprint")
         oci_tenancy = optional_params.get("oci_tenancy")
         oci_key = optional_params.get("oci_key")
+        oci_key_file = optional_params.get("oci_key_file")
 
-        if not oci_user or not oci_fingerprint or not oci_tenancy or not oci_key:
+        if (
+            not oci_user
+            or not oci_fingerprint
+            or not oci_tenancy
+            or not (oci_key or oci_key_file)
+        ):
             raise Exception(
-                "Missing one of the following parameters: oci_user, oci_fingerprint, oci_tenancy, oci_key"
+                "Missing required parameters: oci_user, oci_fingerprint, oci_tenancy, "
+                "and at least one of oci_key or oci_key_file."
             )
 
         method = str(optional_params.get("method", "POST")).upper()
@@ -283,7 +306,17 @@ class OCIChatConfig(BaseConfig):
                 "Please install it with: pip install cryptography"
             ) from e
 
-        private_key = load_private_key_from_str(oci_key)
+        private_key = (
+            load_private_key_from_str(oci_key)
+            if oci_key
+            else load_private_key_from_file(oci_key_file) if oci_key_file else None
+        )
+
+        if private_key is None:
+            raise Exception(
+                "Private key is required for OCI authentication. Please provide either oci_key or oci_key_file."
+            )
+
         signature = private_key.sign(
             signing_string.encode("utf-8"),
             padding.PKCS1v15(),
@@ -334,17 +367,19 @@ class OCIChatConfig(BaseConfig):
         oci_fingerprint = optional_params.get("oci_fingerprint")
         oci_tenancy = optional_params.get("oci_tenancy")
         oci_key = optional_params.get("oci_key")
+        oci_key_file = optional_params.get("oci_key_file")
         oci_compartment_id = optional_params.get("oci_compartment_id")
 
         if (
             not oci_user
             or not oci_fingerprint
             or not oci_tenancy
-            or not oci_key
+            or not (oci_key or oci_key_file)
             or not oci_compartment_id
         ):
             raise Exception(
-                "Missing one of the following parameters: oci_user, oci_fingerprint, oci_tenancy, oci_key, oci_compartment_id"
+                "Missing required parameters: oci_user, oci_fingerprint, oci_tenancy, "
+                "and at least one of oci_key or oci_key_file."
             )
 
         if not api_base:

litellm/llms/ollama/chat/transformation.py

@@ -137,6 +137,7 @@ class OllamaChatConfig(BaseConfig):
             "tool_choice",
             "functions",
             "response_format",
+            "reasoning_effort",
         ]
 
     def map_openai_params(
@@ -175,6 +176,8 @@ class OllamaChatConfig(BaseConfig):
                 if value.get("json_schema") and value["json_schema"].get("schema"):
                     optional_params["format"] = value["json_schema"]["schema"]
             ### FUNCTION CALLING LOGIC ###
+            if param == "reasoning_effort" and value is not None:
+                optional_params["think"] = True
             if param == "tools":
                 ## CHECK IF MODEL SUPPORTS TOOL CALLING ##
                 try:
@@ -212,9 +215,9 @@ class OllamaChatConfig(BaseConfig):
                     litellm.add_function_to_prompt = (
                         True  # so that main.py adds the function call to the prompt
                     )
-                    optional_params[
-                        "functions_unsupported_model"
-                    ] = non_default_params.get("functions")
+                    optional_params["functions_unsupported_model"] = (
+                        non_default_params.get("functions")
+                    )
         non_default_params.pop("tool_choice", None)  # causes ollama requests to hang
         non_default_params.pop("functions", None)  # causes ollama requests to hang
         return optional_params
@@ -346,11 +349,31 @@ class OllamaChatConfig(BaseConfig):
 
         ## RESPONSE OBJECT
         model_response.choices[0].finish_reason = "stop"
+        response_json_message = response_json.get("message")
+        if response_json_message is not None:
+            if "thinking" in response_json_message:
+                # remap 'thinking' to 'reasoning_content'
+                response_json_message["reasoning_content"] = response_json_message[
+                    "thinking"
+                ]
+                del response_json_message["thinking"]
+            elif response_json_message.get("content") is not None:
+                # parse reasoning content from content
+                from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import (
+                    _parse_content_for_reasoning,
+                )
+
+                reasoning_content, content = _parse_content_for_reasoning(
+                    response_json_message["content"]
+                )
+                response_json_message["reasoning_content"] = reasoning_content
+                response_json_message["content"] = content
+
         if (
             request_data.get("format", "") == "json"
             and litellm_params.get("function_name") is not None
         ):
-            function_call = json.loads(response_json["message"]["content"])
+            function_call = json.loads(response_json_message["content"])
             message = litellm.Message(
                 content=None,
                 tool_calls=[
@@ -367,11 +390,13 @@ class OllamaChatConfig(BaseConfig):
                         "type": "function",
                     }
                 ],
+                reasoning_content=response_json_message.get("reasoning_content"),
             )
             model_response.choices[0].message = message  # type: ignore
             model_response.choices[0].finish_reason = "tool_calls"
         else:
-            _message = litellm.Message(**response_json["message"])
+
+            _message = litellm.Message(**response_json_message)
             model_response.choices[0].message = _message  # type: ignore
         model_response.created = int(time.time())
         model_response.model = "ollama_chat/" + model
@@ -412,6 +437,9 @@ class OllamaChatConfig(BaseConfig):
 
 
 class OllamaChatCompletionResponseIterator(BaseModelResponseIterator):
+    started_reasoning_content: bool = False
+    finished_reasoning_content: bool = False
+
     def _is_function_call_complete(self, function_args: Union[str, dict]) -> bool:
         if isinstance(function_args, dict):
             return True
@@ -465,8 +493,38 @@ class OllamaChatCompletionResponseIterator(BaseModelResponseIterator):
                         if is_function_call_complete:
                             tool_call["id"] = str(uuid.uuid4())
 
+            # PROCESS REASONING CONTENT
+            reasoning_content: Optional[str] = None
+            content: Optional[str] = None
+            if chunk["message"].get("thinking") is not None:
+                if self.started_reasoning_content is False:
+                    reasoning_content = chunk["message"].get("thinking")
+                    self.started_reasoning_content = True
+                elif self.finished_reasoning_content is False:
+                    reasoning_content = chunk["message"].get("thinking")
+                    self.finished_reasoning_content = True
+            elif chunk["message"].get("content") is not None:
+                message_content = chunk["message"].get("content")
+                if "<think>" in message_content:
+                    message_content = message_content.replace("<think>", "")
+
+                    self.started_reasoning_content = True
+
+                if "</think>" in message_content and self.started_reasoning_content:
+                    message_content = message_content.replace("</think>", "")
+                    self.finished_reasoning_content = True
+
+                if (
+                    self.started_reasoning_content
+                    and not self.finished_reasoning_content
+                ):
+                    reasoning_content = message_content
+                else:
+                    content = message_content
+
             delta = Delta(
-                content=chunk["message"].get("content", ""),
+                content=content,
+                reasoning_content=reasoning_content,
                 tool_calls=tool_calls,
             )
 

litellm/llms/ollama/completion/transformation.py

@@ -19,13 +19,13 @@ from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMExcepti
 from litellm.secret_managers.main import get_secret_str
 from litellm.types.llms.openai import AllMessageValues, ChatCompletionUsageBlock
 from litellm.types.utils import (
+    Delta,
     GenericStreamingChunk,
     ModelInfoBase,
     ModelResponse,
     ModelResponseStream,
     ProviderField,
     StreamingChoices,
-    Delta,
 )
 
 from ..common_utils import OllamaError, _convert_image
@@ -92,9 +92,9 @@ class OllamaConfig(BaseConfig):
     repeat_penalty: Optional[float] = None
     temperature: Optional[float] = None
     seed: Optional[int] = None
-    stop: Optional[
-        list
-    ] = None  # stop is a list based on this - https://github.com/ollama/ollama/pull/442
+    stop: Optional[list] = (
+        None  # stop is a list based on this - https://github.com/ollama/ollama/pull/442
+    )
     tfs_z: Optional[float] = None
     num_predict: Optional[int] = None
     top_k: Optional[int] = None
@@ -154,6 +154,7 @@ class OllamaConfig(BaseConfig):
             "stop",
             "response_format",
             "max_completion_tokens",
+            "reasoning_effort",
         ]
 
     def map_openai_params(
@@ -166,19 +167,21 @@ class OllamaConfig(BaseConfig):
         for param, value in non_default_params.items():
             if param == "max_tokens" or param == "max_completion_tokens":
                 optional_params["num_predict"] = value
-            if param == "stream":
+            elif param == "stream":
                 optional_params["stream"] = value
-            if param == "temperature":
+            elif param == "temperature":
                 optional_params["temperature"] = value
-            if param == "seed":
+            elif param == "seed":
                 optional_params["seed"] = value
-            if param == "top_p":
+            elif param == "top_p":
                 optional_params["top_p"] = value
-            if param == "frequency_penalty":
+            elif param == "frequency_penalty":
                 optional_params["frequency_penalty"] = value
-            if param == "stop":
+            elif param == "stop":
                 optional_params["stop"] = value
-            if param == "response_format" and isinstance(value, dict):
+            elif param == "reasoning_effort" and value is not None:
+                optional_params["think"] = True
+            elif param == "response_format" and isinstance(value, dict):
                 if value["type"] == "json_object":
                     optional_params["format"] = "json"
                 elif value["type"] == "json_schema":
@@ -258,12 +261,17 @@ class OllamaConfig(BaseConfig):
         api_key: Optional[str] = None,
         json_mode: Optional[bool] = None,
     ) -> ModelResponse:
+        from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import (
+            _parse_content_for_reasoning,
+        )
+
         response_json = raw_response.json()
         ## RESPONSE OBJECT
         model_response.choices[0].finish_reason = "stop"
         if request_data.get("format", "") == "json":
             # Check if response field exists and is not empty before parsing JSON
             response_text = response_json.get("response", "")
+
             if not response_text or not response_text.strip():
                 # Handle empty response gracefully - set empty content
                 message = litellm.Message(content="")
@@ -288,7 +296,9 @@ class OllamaConfig(BaseConfig):
                                     "id": f"call_{str(uuid.uuid4())}",
                                     "function": {
                                         "name": function_call["name"],
-                                        "arguments": json.dumps(function_call["arguments"]),
+                                        "arguments": json.dumps(
+                                            function_call["arguments"]
+                                        ),
                                     },
                                     "type": "function",
                                 }
@@ -305,11 +315,26 @@ class OllamaConfig(BaseConfig):
                         model_response.choices[0].finish_reason = "stop"
                 except json.JSONDecodeError:
                     # If JSON parsing fails, treat as regular text response
-                    message = litellm.Message(content=response_text)
+                    ## output parse reasoning content from response_text
+                    reasoning_content: Optional[str] = None
+                    content: Optional[str] = None
+                    if response_text is not None:
+                        reasoning_content, content = _parse_content_for_reasoning(
+                            response_text
+                        )
+                    message = litellm.Message(
+                        content=content, reasoning_content=reasoning_content
+                    )
                     model_response.choices[0].message = message  # type: ignore
                     model_response.choices[0].finish_reason = "stop"
         else:
-            model_response.choices[0].message.content = response_json["response"]  # type: ignore
+            response_text = response_json.get("response", "")
+            content = None
+            reasoning_content = None
+            if response_text is not None:
+                reasoning_content, content = _parse_content_for_reasoning(response_text)
+            model_response.choices[0].message.content = content  # type: ignore
+            model_response.choices[0].message.reasoning_content = reasoning_content  # type: ignore
         model_response.created = int(time.time())
         model_response.model = "ollama/" + model
         _prompt = request_data.get("prompt", "")
@@ -434,12 +459,21 @@ class OllamaConfig(BaseConfig):
 
 
 class OllamaTextCompletionResponseIterator(BaseModelResponseIterator):
+    def __init__(
+        self, streaming_response, sync_stream: bool, json_mode: Optional[bool] = False
+    ):
+        super().__init__(streaming_response, sync_stream, json_mode)
+        self.started_reasoning_content: bool = False
+        self.finished_reasoning_content: bool = False
+
     def _handle_string_chunk(
         self, str_line: str
     ) -> Union[GenericStreamingChunk, ModelResponseStream]:
         return self.chunk_parser(json.loads(str_line))
 
-    def chunk_parser(self, chunk: dict) -> Union[GenericStreamingChunk, ModelResponseStream]:
+    def chunk_parser(
+        self, chunk: dict
+    ) -> Union[GenericStreamingChunk, ModelResponseStream]:
         try:
             if "error" in chunk:
                 raise Exception(f"Ollama Error - {chunk}")
@@ -469,12 +503,42 @@ class OllamaTextCompletionResponseIterator(BaseModelResponseIterator):
                 )
             elif chunk["response"]:
                 text = chunk["response"]
-                return GenericStreamingChunk(
-                    text=text,
-                    is_finished=is_finished,
-                    finish_reason="stop",
+                reasoning_content: Optional[str] = None
+                content: Optional[str] = None
+                if text is not None:
+                    if "<think>" in text:
+                        text = text.replace("<think>", "")
+                        self.started_reasoning_content = True
+                    elif "</think>" in text:
+                        text = text.replace("</think>", "")
+                        self.finished_reasoning_content = True
+
+                    if (
+                        self.started_reasoning_content
+                        and not self.finished_reasoning_content
+                    ):
+                        reasoning_content = text
+                    else:
+                        content = text
+
+                return ModelResponseStream(
+                    choices=[
+                        StreamingChoices(
+                            index=0,
+                            delta=Delta(
+                                reasoning_content=reasoning_content, content=content
+                            ),
+                        )
+                    ],
+                    finish_reason=finish_reason,
                     usage=None,
                 )
+                # return GenericStreamingChunk(
+                #     text=text,
+                #     is_finished=is_finished,
+                #     finish_reason="stop",
+                #     usage=None,
+                # )
             elif "thinking" in chunk and not chunk["response"]:
                 # Return reasoning content as ModelResponseStream so UIs can render it
                 thinking_content = chunk.get("thinking") or ""

litellm/llms/openai/chat/gpt_5_transformation.py

@@ -28,7 +28,18 @@ class OpenAIGPT5Config(OpenAIGPTConfig):
         base_gpt_series_params.extend(gpt_5_only_params)
         if not supports_tool_choice(model=model):
             base_gpt_series_params.remove("tool_choice")
-        return base_gpt_series_params
+
+        non_supported_params = [
+            "logprobs",
+            "top_p",
+            "presence_penalty",
+            "frequency_penalty",
+            "top_logprobs",
+        ]
+
+        return [
+            param for param in base_gpt_series_params if param not in non_supported_params
+        ]
 
     def map_openai_params(
         self,

litellm/llms/openai_like/chat/handler.py

@@ -49,15 +49,8 @@ async def make_call(
         model_response = ModelResponse(**response.json())
         completion_stream = MockResponseIterator(model_response=model_response)
     else:
-        # Use aiter_text with explicit UTF-8 encoding to avoid ASCII encoding errors
-        async def utf8_aiter_lines():
-            async for line in response.aiter_text(encoding='utf-8'):
-                for line_part in line.splitlines(keepends=True):
-                    if line_part.strip():
-                        yield line_part.rstrip('\r\n')
-        
         completion_stream = ModelResponseIterator(
-            streaming_response=utf8_aiter_lines(), sync_stream=False
+            streaming_response=response.aiter_lines(), sync_stream=False
         )
     # LOGGING
     logging_obj.post_call(
@@ -100,15 +93,8 @@ def make_sync_call(
         model_response = ModelResponse(**response.json())
         completion_stream = MockResponseIterator(model_response=model_response)
     else:
-        # Use iter_text with explicit UTF-8 encoding to avoid ASCII encoding errors
-        def utf8_iter_lines():
-            for line in response.iter_text(encoding='utf-8'):
-                for line_part in line.splitlines(keepends=True):
-                    if line_part.strip():
-                        yield line_part.rstrip('\r\n')
-        
         completion_stream = ModelResponseIterator(
-            streaming_response=utf8_iter_lines(), sync_stream=True
+            streaming_response=response.iter_lines(), sync_stream=True
         )
 
     # LOGGING

litellm/llms/vercel_ai_gateway/chat/transformation.py

@@ -0,0 +1,112 @@
+"""
+Support for OpenAI's `/v1/chat/completions` endpoint.
+
+Calls done in OpenAI/openai.py as Vercel AI Gateway is openai-compatible.
+
+Docs: https://vercel.com/docs/ai-gateway
+"""
+
+from typing import List, Optional, Tuple, Union
+
+import httpx
+
+from litellm.llms.base_llm.chat.transformation import BaseLLMException
+from litellm.types.llms.openai import AllMessageValues
+from litellm.secret_managers.main import get_secret_str
+import litellm
+
+from ...openai.chat.gpt_transformation import OpenAIGPTConfig
+from ..common_utils import VercelAIGatewayException
+
+
+class VercelAIGatewayConfig(OpenAIGPTConfig):
+    @property
+    def custom_llm_provider(self) -> Optional[str]:
+        return "vercel_ai_gateway"
+
+    def get_supported_openai_params(self, model: str) -> list:
+        base_params = super().get_supported_openai_params(model)
+        if "extra_body" not in base_params:
+            base_params.append("extra_body")
+        return base_params
+
+    def _get_openai_compatible_provider_info(
+        self, api_base: Optional[str], api_key: Optional[str]
+    ) -> Tuple[Optional[str], Optional[str]]:
+
+        api_base = (
+            api_base
+            or get_secret_str("VERCEL_AI_GATEWAY_API_BASE")
+            or "https://ai-gateway.vercel.sh/v1"
+        )
+        user_api_key = (
+            api_key 
+            or get_secret_str("VERCEL_AI_GATEWAY_API_KEY")
+            or get_secret_str("VERCEL_OIDC_TOKEN")
+        )
+        return api_base, user_api_key
+
+    def map_openai_params(
+        self,
+        non_default_params: dict,
+        optional_params: dict,
+        model: str,
+        drop_params: bool,
+    ) -> dict:
+        mapped_openai_params = super().map_openai_params(
+            non_default_params, optional_params, model, drop_params
+        )
+
+        # Vercel AI Gateway-only parameters
+        extra_body = {}
+        provider_options = non_default_params.pop("providerOptions", None)
+        
+        if provider_options is not None:
+            extra_body["providerOptions"] = provider_options
+        
+        mapped_openai_params["extra_body"] = extra_body  # openai client supports `extra_body` param
+        return mapped_openai_params
+
+    def transform_request(
+        self,
+        model: str,
+        messages: List[AllMessageValues],
+        optional_params: dict,
+        litellm_params: dict,
+        headers: dict,
+    ) -> dict:
+        """
+        Transform the overall request to be sent to the API.
+
+        Returns:
+            dict: The transformed request. Sent as the body of the API call.
+        """
+        return super().transform_request(
+            model, messages, optional_params, litellm_params, headers
+        )
+
+    def get_error_class(
+        self, error_message: str, status_code: int, headers: Union[dict, httpx.Headers]
+    ) -> BaseLLMException:
+        return VercelAIGatewayException(
+            message=error_message,
+            status_code=status_code,
+            headers=headers,
+        )
+
+    def get_models(
+        self, api_key: Optional[str] = None, api_base: Optional[str] = None
+    ) -> List[str]:
+        api_base, _ = self._get_openai_compatible_provider_info(api_base, api_key)
+        
+        if api_base is None:
+            api_base = "https://ai-gateway.vercel.sh/v1"
+            
+        models_url = f"{api_base}/models"
+        response = litellm.module_level_client.get(url=models_url)
+
+        if response.status_code != 200:
+            raise Exception(f"Failed to get models: {response.text}")
+
+        models = response.json()["data"]
+        return [model["id"] for model in models]

litellm/llms/vercel_ai_gateway/common_utils.py

@@ -0,0 +1,5 @@
+from litellm.llms.base_llm.chat.transformation import BaseLLMException
+
+
+class VercelAIGatewayException(BaseLLMException):
+    pass

litellm/llms/vertex_ai/common_utils.py

@@ -187,6 +187,25 @@ def _check_text_in_content(parts: List[PartType]) -> bool:
     return has_text_param
 
 
+def _fix_enum_empty_strings(schema, depth=0):
+    """Fix empty strings in enum values by replacing them with None. Gemini doesn't accept empty strings in enums."""
+    if depth > DEFAULT_MAX_RECURSE_DEPTH:
+        raise ValueError(f"Max depth of {DEFAULT_MAX_RECURSE_DEPTH} exceeded while processing schema.")
+    
+    if "enum" in schema and isinstance(schema["enum"], list):
+        schema["enum"] = [None if value == "" else value for value in schema["enum"]]
+
+    # Reuse existing recursion pattern from convert_anyof_null_to_nullable
+    properties = schema.get("properties", None)
+    if properties is not None:
+        for _, value in properties.items():
+            _fix_enum_empty_strings(value, depth=depth + 1)
+
+    items = schema.get("items", None)
+    if items is not None:
+        _fix_enum_empty_strings(items, depth=depth + 1)
+
+
 def _build_vertex_schema(parameters: dict, add_property_ordering: bool = False):
     """
     This is a modified version of https://github.com/google-gemini/generative-ai-python/blob/8f77cc6ac99937cd3a81299ecf79608b91b06bbb/google/generativeai/types/content_types.py#L419
@@ -215,6 +234,11 @@ def _build_vertex_schema(parameters: dict, add_property_ordering: bool = False):
     #     * https://github.com/pydantic/pydantic/discussions/4872
     convert_anyof_null_to_nullable(parameters)
 
+    _convert_schema_types(parameters)
+
+    # Handle empty strings in enum values - Gemini doesn't accept empty strings in enums
+    _fix_enum_empty_strings(parameters)
+
     # Handle empty items objects
     process_items(parameters)
     add_object_type(parameters)
@@ -254,9 +278,7 @@ def _filter_anyof_fields(schema_dict: Dict[str, Any]) -> Dict[str, Any]:
                     item["title"] = title
                 if description:
                     item["description"] = description
-            return {"anyOf": any_of}
-        else:
-            return schema_dict
+        return {"anyOf": any_of}
     return schema_dict
 
 
@@ -441,6 +463,47 @@ def _convert_vertex_datetime_to_openai_datetime(vertex_datetime: str) -> int:
     return int(dt.timestamp())
 
 
+def _convert_schema_types(schema, depth=0):
+    """
+    Convert type arrays and lowercase types for Vertex AI compatibility.
+    
+    Transforms OpenAI-style schemas to Vertex AI format by converting type arrays 
+    like ["string", "number"] to anyOf format and converting all types to uppercase.
+    """
+    if depth > DEFAULT_MAX_RECURSE_DEPTH:
+        raise ValueError(
+            f"Max depth of {DEFAULT_MAX_RECURSE_DEPTH} exceeded while processing schema. Please check the schema for excessive nesting."
+        )
+    
+    if not isinstance(schema, dict):
+        return
+
+    
+    # Handle type field
+    if "type" in schema:
+        type_val = schema["type"]
+        if isinstance(type_val, list) and len(type_val) > 1:
+            # Convert ["string", "number"] -> {"anyOf": [{"type": "STRING"}, {"type": "NUMBER"}]}
+            schema["anyOf"] = [{"type": t} for t in type_val if isinstance(t, str)]
+            schema.pop("type")
+        elif isinstance(type_val, list) and len(type_val) == 1:
+            schema["type"] = type_val[0]
+        elif isinstance(type_val, str):
+            schema["type"] = type_val
+    
+    # Recursively process nested properties, items, and anyOf
+    for key in ["properties", "items", "anyOf"]:
+        if key in schema:
+            value = schema[key]
+            if key == "properties" and isinstance(value, dict):
+                for prop_schema in value.values():
+                    _convert_schema_types(prop_schema, depth + 1)
+            elif key == "items":
+                _convert_schema_types(value, depth + 1)
+            elif key == "anyOf" and isinstance(value, list):
+                for anyof_schema in value:
+                    _convert_schema_types(anyof_schema, depth + 1)
+
 def get_vertex_project_id_from_url(url: str) -> Optional[str]:
     """
     Get the vertex project id from the url

litellm/llms/vertex_ai/gemini/transformation.py

@@ -35,6 +35,7 @@ from litellm.types.llms.openai import (
     ChatCompletionFileObject,
     ChatCompletionImageObject,
     ChatCompletionTextObject,
+    ChatCompletionUserMessage,
 )
 from litellm.types.llms.vertex_ai import *
 from litellm.types.llms.vertex_ai import (
@@ -104,6 +105,64 @@ def _process_gemini_image(image_url: str, format: Optional[str] = None) -> PartT
         raise e
 
 
+def _snake_to_camel(snake_str: str) -> str:
+    """Convert snake_case to camelCase"""
+    components = snake_str.split("_")
+    return components[0] + "".join(x.capitalize() for x in components[1:])
+
+
+def _camel_to_snake(camel_str: str) -> str:
+    """Convert camelCase to snake_case"""
+    import re
+
+    return re.sub(r"(?<!^)(?=[A-Z])", "_", camel_str).lower()
+
+
+def _get_equivalent_key(key: str, available_keys: set) -> Optional[str]:
+    """
+    Get the equivalent key from available keys, checking both camelCase and snake_case variants
+    """
+    if key in available_keys:
+        return key
+
+    # Try camelCase version
+    camel_key = _snake_to_camel(key)
+    if camel_key in available_keys:
+        return camel_key
+
+    # Try snake_case version
+    snake_key = _camel_to_snake(key)
+    if snake_key in available_keys:
+        return snake_key
+
+    return None
+
+
+def check_if_part_exists_in_parts(
+    parts: List[PartType], part: PartType, excluded_keys: List[str] = []
+) -> bool:
+    """
+    Check if a part exists in a list of parts
+    Handles both camelCase and snake_case key variations (e.g., function_call vs functionCall)
+    """
+    keys_to_compare = set(part.keys()) - set(excluded_keys)
+    for p in parts:
+        p_keys = set(p.keys())
+        # Check if all keys in part have equivalent values in p
+        match_found = True
+        for key in keys_to_compare:
+            equivalent_key = _get_equivalent_key(key, p_keys)
+            if equivalent_key is None or p.get(equivalent_key, None) != part.get(
+                key, None
+            ):
+                match_found = False
+                break
+
+        if match_found:
+            return True
+    return False
+
+
 def _gemini_convert_messages_with_history(  # noqa: PLR0915
     messages: List[AllMessageValues],
 ) -> List[ContentType]:
@@ -235,10 +294,33 @@ def _gemini_convert_messages_with_history(  # noqa: PLR0915
                 assistant_msg = ChatCompletionAssistantMessage(**msg_dict)  # type: ignore
                 _message_content = assistant_msg.get("content", None)
                 reasoning_content = assistant_msg.get("reasoning_content", None)
+                thinking_blocks = assistant_msg.get("thinking_blocks")
                 if reasoning_content is not None:
                     assistant_content.append(
                         PartType(thought=True, text=reasoning_content)
                     )
+                if thinking_blocks is not None:
+                    for block in thinking_blocks:
+                        block_thinking_str = block.get("thinking")
+                        block_signature = block.get("signature")
+                        if (
+                            block_thinking_str is not None
+                            and block_signature is not None
+                        ):
+                            try:
+                                assistant_content.append(
+                                    PartType(
+                                        thoughtSignature=block_signature,
+                                        **json.loads(block_thinking_str),
+                                    )
+                                )
+                            except Exception:
+                                assistant_content.append(
+                                    PartType(
+                                        thoughtSignature=block_signature,
+                                        text=block_thinking_str,
+                                    )
+                                )
                 if _message_content is not None and isinstance(_message_content, list):
                     _parts = []
                     for element in _message_content:
@@ -261,9 +343,17 @@ def _gemini_convert_messages_with_history(  # noqa: PLR0915
                     assistant_msg.get("tool_calls", []) is not None
                     or assistant_msg.get("function_call") is not None
                 ):  # support assistant tool invoke conversion
-                    assistant_content.extend(
-                        convert_to_gemini_tool_call_invoke(assistant_msg)
+                    gemini_tool_call_parts = convert_to_gemini_tool_call_invoke(
+                        assistant_msg
                     )
+                    ## check if gemini_tool_call already exists in assistant_content
+                    for gemini_tool_call_part in gemini_tool_call_parts:
+                        if not check_if_part_exists_in_parts(
+                            assistant_content,
+                            gemini_tool_call_part,
+                            excluded_keys=["thoughtSignature"],
+                        ):
+                            assistant_content.append(gemini_tool_call_part)
                     last_message_with_tool_calls = assistant_msg
 
                 msg_i += 1
@@ -476,6 +566,15 @@ async def async_transform_request_body(
     )
 
 
+def _default_user_message_when_system_message_passed() -> ChatCompletionUserMessage:
+    """
+    Returns a default user message when a "system" message is passed in gemini fails.
+
+    This adds a blank user message to the messages list, to ensure that gemini doesn't fail the request.
+    """
+    return ChatCompletionUserMessage(content=".", role="user")
+
+
 def _transform_system_message(
     supports_system_message: bool, messages: List[AllMessageValues]
 ) -> Tuple[Optional[SystemInstructions], List[AllMessageValues]]:
@@ -510,6 +609,13 @@ def _transform_system_message(
                 messages.pop(idx)
 
     if len(system_content_blocks) > 0:
+        #########################################################
+        # If no messages are passed in, add a blank user message
+        # Relevant Issue - https://github.com/BerriAI/litellm/issues/13769
+        #########################################################
+        if len(messages) == 0:
+            messages.append(_default_user_message_when_system_message_passed())
+        #########################################################
         return SystemInstructions(parts=system_content_blocks), messages
 
     return None, messages

litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py

@@ -43,9 +43,11 @@ from litellm.types.llms.gemini import BidiGenerateContentServerMessage
 from litellm.types.llms.openai import (
     AllMessageValues,
     ChatCompletionResponseMessage,
+    ChatCompletionThinkingBlock,
     ChatCompletionToolCallChunk,
     ChatCompletionToolCallFunctionChunk,
     ChatCompletionToolParamFunctionChunk,
+    ImageURLObject,
     OpenAIChatCompletionFinishReason,
 )
 from litellm.types.llms.vertex_ai import (
@@ -89,11 +91,12 @@ from .transformation import (
 
 if TYPE_CHECKING:
     from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
-    from litellm.types.utils import ModelResponseStream
+    from litellm.types.utils import ModelResponseStream, StreamingChoices
 
     LoggingClass = LiteLLMLoggingObj
 else:
     LoggingClass = Any
+    StreamingChoices = Any
 
 
 class VertexAIBaseConfig:
@@ -461,7 +464,6 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
             params["includeThoughts"] = True
         if thinking_budget is not None and isinstance(thinking_budget, int):
             params["thinkingBudget"] = thinking_budget
-
         return params
 
     def map_response_modalities(self, value: list) -> list:
@@ -774,8 +776,9 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
             elif "inlineData" in part:
                 mime_type = part["inlineData"]["mimeType"]
                 data = part["inlineData"]["data"]
-                # Check if inline data is audio - if so, exclude from text content
-                if mime_type.startswith("audio/"):
+                # Check if inline data is audio or image - if so, exclude from text content
+                # Images and audio are now handled separately in their respective response fields
+                if mime_type.startswith("audio/") or mime_type.startswith("image/"):
                     continue
                 _content_str += "data:{};base64,{}".format(mime_type, data)
 
@@ -791,6 +794,38 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
 
         return content_str, reasoning_content_str
 
+    def _extract_thinking_blocks_from_parts(
+        self, parts: List[HttpxPartType]
+    ) -> List[ChatCompletionThinkingBlock]:
+        """Extract thinking blocks from parts if present"""
+        thinking_blocks: List[ChatCompletionThinkingBlock] = []
+        for part in parts:
+            if "thoughtSignature" in part:
+                part_copy = part.copy()
+                part_copy.pop("thoughtSignature")
+                thinking_blocks.append(
+                    ChatCompletionThinkingBlock(
+                        type="thinking",
+                        thinking=json.dumps(part_copy),
+                        signature=part["thoughtSignature"],
+                    )
+                )
+        return thinking_blocks
+
+    def _extract_image_response_from_parts(
+        self, parts: List[HttpxPartType]
+    ) -> Optional[ImageURLObject]:
+        """Extract image response from parts if present"""
+        for part in parts:
+            if "inlineData" in part:
+                mime_type = part["inlineData"]["mimeType"]
+                data = part["inlineData"]["data"]
+                if mime_type.startswith("image/"):
+                    # Convert base64 data to data URI format
+                    data_uri = f"data:{mime_type};base64,{data}"
+                    return ImageURLObject(url=data_uri, detail="auto")
+        return None
+
     def _extract_audio_response_from_parts(
         self, parts: List[HttpxPartType]
     ) -> Optional[ChatCompletionAudioResponse]:
@@ -1109,6 +1144,80 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
                     web_search_requests = len(grounding_metadata)
         return web_search_requests
 
+    @staticmethod
+    def _create_streaming_choice(
+        chat_completion_message: ChatCompletionResponseMessage,
+        candidate: Candidates,
+        idx: int,
+        tools: Optional[List[ChatCompletionToolCallChunk]],
+        functions: Optional[ChatCompletionToolCallFunctionChunk],
+        chat_completion_logprobs: Optional[ChoiceLogprobs],
+        image_response: Optional[ImageURLObject],
+    ) -> StreamingChoices:
+        """
+        Helper method to create a streaming choice object for Vertex AI
+        """
+        from litellm.types.utils import Delta, StreamingChoices
+
+        # create a streaming choice object
+        choice = StreamingChoices(
+            finish_reason=VertexGeminiConfig._check_finish_reason(
+                chat_completion_message, candidate.get("finishReason")
+            ),
+            index=candidate.get("index", idx),
+            delta=Delta(
+                content=chat_completion_message.get("content"),
+                reasoning_content=chat_completion_message.get("reasoning_content"),
+                tool_calls=tools,
+                image=image_response,
+                function_call=functions,
+            ),
+            logprobs=chat_completion_logprobs,
+            enhancements=None,
+        )
+        return choice
+
+    @staticmethod
+    def _extract_candidate_metadata(
+        candidate: Candidates,
+    ) -> Tuple[List[dict], List[dict], List, List]:
+        """
+        Extract metadata from a single candidate response.
+
+        Returns:
+            grounding_metadata: List[dict]
+            url_context_metadata: List[dict]
+            safety_ratings: List
+            citation_metadata: List
+        """
+        grounding_metadata: List[dict] = []
+        url_context_metadata: List[dict] = []
+        safety_ratings: List = []
+        citation_metadata: List = []
+
+        if "groundingMetadata" in candidate:
+            if isinstance(candidate["groundingMetadata"], list):
+                grounding_metadata.extend(candidate["groundingMetadata"])  # type: ignore
+            else:
+                grounding_metadata.append(candidate["groundingMetadata"])  # type: ignore
+
+        if "safetyRatings" in candidate:
+            safety_ratings.append(candidate["safetyRatings"])
+
+        if "citationMetadata" in candidate:
+            citation_metadata.append(candidate["citationMetadata"])
+
+        if "urlContextMetadata" in candidate:
+            # Add URL context metadata to grounding metadata
+            url_context_metadata.append(cast(dict, candidate["urlContextMetadata"]))
+
+        return (
+            grounding_metadata,
+            url_context_metadata,
+            safety_ratings,
+            citation_metadata,
+        )
+
     @staticmethod
     def _process_candidates(
         _candidates: List[Candidates],
@@ -1131,6 +1240,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
 
         grounding_metadata: List[dict] = []
         url_context_metadata: List[dict] = []
+        image_response: Optional[ImageURLObject] = None
         safety_ratings: List = []
         citation_metadata: List = []
         chat_completion_message: ChatCompletionResponseMessage = {"role": "assistant"}
@@ -1138,26 +1248,24 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
         tools: Optional[List[ChatCompletionToolCallChunk]] = []
         functions: Optional[ChatCompletionToolCallFunctionChunk] = None
         cumulative_tool_call_index: int = 0
+        thinking_blocks: Optional[List[ChatCompletionThinkingBlock]] = None
 
         for idx, candidate in enumerate(_candidates):
             if "content" not in candidate:
                 continue
 
-            if "groundingMetadata" in candidate:
-                if isinstance(candidate["groundingMetadata"], list):
-                    grounding_metadata.extend(candidate["groundingMetadata"])  # type: ignore
-                else:
-                    grounding_metadata.append(candidate["groundingMetadata"])  # type: ignore
+            # Extract metadata using helper function
+            (
+                candidate_grounding_metadata,
+                candidate_url_context_metadata,
+                candidate_safety_ratings,
+                candidate_citation_metadata,
+            ) = VertexGeminiConfig._extract_candidate_metadata(candidate)
 
-            if "safetyRatings" in candidate:
-                safety_ratings.append(candidate["safetyRatings"])
-
-            if "citationMetadata" in candidate:
-                citation_metadata.append(candidate["citationMetadata"])
-
-            if "urlContextMetadata" in candidate:
-                # Add URL context metadata to grounding metadata
-                url_context_metadata.append(cast(dict, candidate["urlContextMetadata"]))
+            grounding_metadata.extend(candidate_grounding_metadata)
+            url_context_metadata.extend(candidate_url_context_metadata)
+            safety_ratings.extend(candidate_safety_ratings)
+            citation_metadata.extend(candidate_citation_metadata)
 
             if "parts" in candidate["content"]:
                 (
@@ -1172,18 +1280,33 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
                         parts=candidate["content"]["parts"]
                     )
                 )
+                image_response = (
+                    VertexGeminiConfig()._extract_image_response_from_parts(
+                        parts=candidate["content"]["parts"]
+                    )
+                )
+
+                thinking_blocks = (
+                    VertexGeminiConfig()._extract_thinking_blocks_from_parts(
+                        parts=candidate["content"]["parts"]
+                    )
+                )
 
                 if audio_response is not None:
                     cast(Dict[str, Any], chat_completion_message)[
                         "audio"
                     ] = audio_response
                     chat_completion_message["content"] = None  # OpenAI spec
-                elif content is not None:
+                if image_response is not None:
+                    # Handle image response - combine with text content into structured format
+                    cast(Dict[str, Any], chat_completion_message)[
+                        "image"
+                    ] = image_response
+                if content is not None:
                     chat_completion_message["content"] = content
 
                 if reasoning_content is not None:
                     chat_completion_message["reasoning_content"] = reasoning_content
-
                 (
                     functions,
                     tools,
@@ -1205,25 +1328,18 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
             if functions is not None:
                 chat_completion_message["function_call"] = functions
 
-            if isinstance(model_response, ModelResponseStream):
-                from litellm.types.utils import Delta, StreamingChoices
+            if thinking_blocks is not None:
+                chat_completion_message["thinking_blocks"] = thinking_blocks  # type: ignore
 
-                # create a streaming choice object
-                choice = StreamingChoices(
-                    finish_reason=VertexGeminiConfig._check_finish_reason(
-                        chat_completion_message, candidate.get("finishReason")
-                    ),
-                    index=candidate.get("index", idx),
-                    delta=Delta(
-                        content=chat_completion_message.get("content"),
-                        reasoning_content=chat_completion_message.get(
-                            "reasoning_content"
-                        ),
-                        tool_calls=tools,
-                        function_call=functions,
-                    ),
-                    logprobs=chat_completion_logprobs,
-                    enhancements=None,
+            if isinstance(model_response, ModelResponseStream):
+                choice = VertexGeminiConfig._create_streaming_choice(
+                    chat_completion_message=chat_completion_message,
+                    candidate=candidate,
+                    idx=idx,
+                    tools=tools,
+                    functions=functions,
+                    chat_completion_logprobs=chat_completion_logprobs,
+                    image_response=image_response,
                 )
                 model_response.choices.append(choice)
             elif isinstance(model_response, ModelResponse):

litellm/llms/xai/chat/transformation.py

@@ -31,7 +31,6 @@ class XAIChatConfig(OpenAIGPTConfig):
 
     def get_supported_openai_params(self, model: str) -> list:
         base_openai_params = [
-            "frequency_penalty",
             "logit_bias",
             "logprobs",
             "max_tokens",
@@ -50,8 +49,22 @@ class XAIChatConfig(OpenAIGPTConfig):
             "web_search_options",
         ]
         # for some reason, grok-3-mini does not support stop tokens
+        #########################################################
+        # stop tokens check
+        #########################################################
         if self._supports_stop_reason(model):
             base_openai_params.append("stop")
+        
+
+        #########################################################
+        # frequency penalty check
+        #########################################################
+        if self._supports_frequency_penalty(model):
+            base_openai_params.append("frequency_penalty")
+        
+        #########################################################
+        # reasoning check
+        #########################################################
         try:
             if litellm.supports_reasoning(
                 model=model, custom_llm_provider=self.custom_llm_provider
@@ -68,6 +81,18 @@ class XAIChatConfig(OpenAIGPTConfig):
         elif "grok-4" in model:
             return False
         return True
+    
+    def _supports_frequency_penalty(self, model: str) -> bool:
+        """
+        From manual testing grok-4 does not support `frequency_penalty`
+
+        When sent the model fails from xAI API
+        """
+        if "grok-4" in model:
+            return False
+        if "grok-code-fast" in model:
+            return False
+        return True
 
     def map_openai_params(
         self,

litellm/llms/xai/cost_calculator.py

@@ -0,0 +1,54 @@
+"""
+Helper util for handling XAI-specific cost calculation
+- e.g.: reasoning tokens for grok models
+"""
+
+from typing import Tuple, Union
+
+from litellm.types.utils import Usage
+from litellm.utils import get_model_info
+
+
+def cost_per_token(model: str, usage: Usage) -> Tuple[float, float]:
+    """
+    Calculates the cost per token for a given XAI model, prompt tokens, and completion tokens.
+
+    Input:
+        - model: str, the model name without provider prefix
+        - usage: LiteLLM Usage block, containing XAI-specific usage information
+
+    Returns:
+        Tuple[float, float] - prompt_cost_in_usd, completion_cost_in_usd
+    """
+    ## GET MODEL INFO
+    model_info = get_model_info(model=model, custom_llm_provider="xai")
+
+    def _safe_float_cast(
+        value: Union[str, int, float, None, object], default: float = 0.0
+    ) -> float:
+        """Safely cast a value to float with proper type handling for mypy."""
+        if value is None:
+            return default
+        try:
+            return float(value)  # type: ignore
+        except (ValueError, TypeError):
+            return default
+
+    ## CALCULATE INPUT COST
+    input_cost_per_token = _safe_float_cast(model_info.get("input_cost_per_token"))
+    prompt_cost: float = (usage.prompt_tokens or 0) * input_cost_per_token
+
+    ## CALCULATE OUTPUT COST
+    output_cost_per_token = _safe_float_cast(model_info.get("output_cost_per_token"))
+
+    # For XAI models, completion is billed as (visible completion tokens + reasoning tokens)
+    completion_tokens = int(getattr(usage, "completion_tokens", 0) or 0)
+    reasoning_tokens = 0
+    if hasattr(usage, "completion_tokens_details") and usage.completion_tokens_details:
+        reasoning_tokens = int(
+            getattr(usage.completion_tokens_details, "reasoning_tokens", 0) or 0
+        )
+
+    completion_cost = (completion_tokens + reasoning_tokens) * output_cost_per_token
+
+    return prompt_cost, completion_cost

litellm/main.py

@@ -61,6 +61,9 @@ from litellm.exceptions import LiteLLMUnknownProvider
 from litellm.integrations.custom_logger import CustomLogger
 from litellm.litellm_core_utils.audio_utils.utils import get_audio_file_for_health_check
 from litellm.litellm_core_utils.dd_tracing import tracer
+from litellm.litellm_core_utils.get_provider_specific_headers import (
+    ProviderSpecificHeaderUtils,
+)
 from litellm.litellm_core_utils.health_check_utils import (
     _create_health_check_response,
     _filter_model_params,
@@ -1107,11 +1110,11 @@ def completion(  # type: ignore # noqa: PLR0915
             api_key=api_key,
         )
 
-        if (
-            provider_specific_header is not None
-            and provider_specific_header["custom_llm_provider"] == custom_llm_provider
-        ):
-            headers.update(provider_specific_header["extra_headers"])
+        if provider_specific_header is not None:
+            headers.update(ProviderSpecificHeaderUtils.get_provider_specific_headers(
+                provider_specific_header=provider_specific_header,
+                custom_llm_provider=custom_llm_provider,
+            ))
 
         if model_response is not None and hasattr(model_response, "_hidden_params"):
             model_response._hidden_params["custom_llm_provider"] = custom_llm_provider
@@ -1253,6 +1256,7 @@ def completion(  # type: ignore # noqa: PLR0915
             additional_drop_params=kwargs.get("additional_drop_params"),
             remove_sensitive_keys=True,
             add_provider_specific_params=True,
+            provider_config=provider_config,
         )
 
         if litellm.add_function_to_prompt and optional_params.get(
@@ -2169,8 +2173,18 @@ def completion(  # type: ignore # noqa: PLR0915
                 or "https://api.anthropic.com/v1/complete"
             )
 
-            if api_base is not None and not api_base.endswith("/v1/complete"):
+            # Check if we should disable automatic URL suffix appending
+            disable_url_suffix = get_secret_bool("LITELLM_ANTHROPIC_DISABLE_URL_SUFFIX")
+            if (
+                api_base is not None
+                and not disable_url_suffix
+                and not api_base.endswith("/v1/complete")
+            ):
                 api_base += "/v1/complete"
+            elif disable_url_suffix:
+                verbose_logger.debug(
+                    "LITELLM_ANTHROPIC_DISABLE_URL_SUFFIX is set, skipping /v1/complete suffix"
+                )
 
             response = base_llm_http_handler.completion(
                 model=model,
@@ -2206,8 +2220,18 @@ def completion(  # type: ignore # noqa: PLR0915
                 or "https://api.anthropic.com/v1/messages"
             )
 
-            if api_base is not None and not api_base.endswith("/v1/messages"):
+            # Check if we should disable automatic URL suffix appending
+            disable_url_suffix = get_secret_bool("LITELLM_ANTHROPIC_DISABLE_URL_SUFFIX")
+            if (
+                api_base is not None
+                and not disable_url_suffix
+                and not api_base.endswith("/v1/messages")
+            ):
                 api_base += "/v1/messages"
+            elif disable_url_suffix:
+                verbose_logger.debug(
+                    "LITELLM_ANTHROPIC_DISABLE_URL_SUFFIX is set, skipping /v1/messages suffix"
+                )
 
             response = anthropic_chat_completions.completion(
                 model=model,
@@ -2644,6 +2668,70 @@ def completion(  # type: ignore # noqa: PLR0915
             logging.post_call(
                 input=messages, api_key=openai.api_key, original_response=response
             )
+        elif custom_llm_provider == "vercel_ai_gateway":
+            api_base = (
+                api_base
+                or litellm.api_base
+                or get_secret_str("VERCEL_AI_GATEWAY_API_BASE")
+                or "https://ai-gateway.vercel.sh/v1"
+            )
+
+            api_key = (
+                api_key
+                or litellm.api_key
+                or get_secret("VERCEL_AI_GATEWAY_API_KEY")
+            )
+
+            vercel_site_url = get_secret("VERCEL_SITE_URL") or "https://litellm.ai"
+            vercel_app_name = get_secret("VERCEL_APP_NAME") or "liteLLM"
+
+            vercel_headers = {
+                "http-referer": vercel_site_url,
+                "x-title": vercel_app_name,
+            }
+
+            _headers = headers or litellm.headers
+            if _headers:
+                vercel_headers.update(_headers)
+
+            headers = vercel_headers
+            
+            ## Load Config
+            config = litellm.VercelAIGatewayConfig.get_config()
+            for k, v in config.items():
+                if k == "extra_body":
+                    # we use openai 'extra_body' to pass vercel specific params - providerOptions
+                    if "extra_body" in optional_params:
+                        optional_params[k].update(v)
+                    else:
+                        optional_params[k] = v
+                elif k not in optional_params:
+                    optional_params[k] = v
+
+            data = {"model": model, "messages": messages, **optional_params}
+
+            ## COMPLETION CALL
+            response = base_llm_http_handler.completion(
+                model=model,
+                stream=stream,
+                messages=messages,
+                acompletion=acompletion,
+                api_base=api_base,
+                model_response=model_response,
+                optional_params=optional_params,
+                litellm_params=litellm_params,
+                custom_llm_provider="vercel_ai_gateway",
+                timeout=timeout,
+                headers=headers,
+                encoding=encoding,
+                api_key=api_key,
+                logging_obj=logging,  # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements
+                client=client,
+            )
+            ## LOGGING
+            logging.post_call(
+                input=messages, api_key=openai.api_key, original_response=response
+            )
         elif (
             custom_llm_provider == "together_ai"
             or ("togethercomputer" in model)

litellm/proxy/_experimental/mcp_server/rest_endpoints.py

@@ -1,5 +1,5 @@
 import importlib
-from typing import Optional, Dict
+from typing import Dict, List, Optional
 
 from fastapi import APIRouter, Depends, Query, Request
 
@@ -21,9 +21,10 @@ router = APIRouter(
 )
 
 if MCP_AVAILABLE:
+    from litellm.experimental_mcp_client.client import MCPTool
     from litellm.proxy._experimental.mcp_server.mcp_server_manager import (
-        global_mcp_server_manager,
         _convert_protocol_version_to_enum,
+        global_mcp_server_manager,
     )
     from litellm.proxy._experimental.mcp_server.server import (
         ListMCPToolsRestAPIResponseObject,
@@ -100,7 +101,9 @@ if MCP_AVAILABLE:
             "message": "Successfully retrieved tools"
         }
         """
-        from litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp import MCPRequestHandler
+        from litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp import (
+            MCPRequestHandler,
+        )
         
         try:
             # Extract auth headers from request
@@ -172,10 +175,11 @@ if MCP_AVAILABLE:
         """
         REST API to call a specific MCP tool with the provided arguments
         """
-        from litellm.proxy.proxy_server import add_litellm_data_to_request, proxy_config
-        from litellm.exceptions import BlockedPiiEntityError, GuardrailRaisedException
         from fastapi import HTTPException
 
+        from litellm.exceptions import BlockedPiiEntityError, GuardrailRaisedException
+        from litellm.proxy.proxy_server import add_litellm_data_to_request, proxy_config
+
         try:
             data = await request.json()
             data = await add_litellm_data_to_request(
@@ -230,13 +234,19 @@ if MCP_AVAILABLE:
     from litellm.proxy.management_endpoints.mcp_management_endpoints import (
         NewMCPServerRequest,
     )
-    @router.post("/test/connection")
-    async def test_connection(
-        request: NewMCPServerRequest,
-    ):
+    
+    async def _execute_with_mcp_client(request: NewMCPServerRequest, operation):
         """
-        Test if we can connect to the provided MCP server before adding it
+        Common helper to create MCP client, execute operation, and ensure proper cleanup.
+        
+        Args:
+            request: MCP server configuration
+            operation: Async function that takes a client and returns the operation result
+            
+        Returns:
+            Operation result or error response
         """
+        client = None
         try:
             client = global_mcp_server_manager._create_mcp_client(
                 server=MCPServer(
@@ -250,12 +260,31 @@ if MCP_AVAILABLE:
                 ),
                 mcp_auth_header=None,
             )
-
-            await client.connect()
+            
+            return await operation(client)
+            
         except Exception as e:
-            verbose_logger.error(f"Error in test_connection: {e}", exc_info=True)
+            verbose_logger.error(f"Error in MCP operation: {e}", exc_info=True)
             return {"status": "error", "message": "An internal error has occurred."}
-        return {"status": "ok"}
+        finally:
+            # Ensure client is properly disconnected before response is sent
+            if client is not None:
+                try:
+                    await client.disconnect()
+                except Exception as e:
+                    verbose_logger.warning(f"Error disconnecting MCP client: {e}")
+    @router.post("/test/connection")
+    async def test_connection(
+        request: NewMCPServerRequest,
+    ):
+        """
+        Test if we can connect to the provided MCP server before adding it
+        """
+        async def _test_connection_operation(client):
+            await client.connect()
+            return {"status": "ok"}
+        
+        return await _execute_with_mcp_client(request, _test_connection_operation)
         
     
     @router.post("/test/tools/list")
@@ -266,25 +295,13 @@ if MCP_AVAILABLE:
         """
         Preview tools available from MCP server before adding it
         """
-        try:
-            client = global_mcp_server_manager._create_mcp_client(
-                server=MCPServer(
-                    server_id=request.server_id or "",
-                    name=request.alias or request.server_name or "",
-                    url=request.url,
-                    transport=request.transport,
-                    spec_version=_convert_protocol_version_to_enum(request.spec_version),
-                    auth_type=request.auth_type,
-                    mcp_info=request.mcp_info,
-                ),
-                mcp_auth_header=None,
-            )
-            list_tools_result = await client.list_tools()
-        except Exception as e:
-            verbose_logger.error(f"Error in test_tools_list: {e}", exc_info=True)
-            return {"status": "error", "message": "An internal error has occurred."}
-        return {
-            "tools": list_tools_result,
-            "error": None,
-            "message": "Successfully retrieved tools"
-        }
+        async def _list_tools_operation(client):
+            list_tools_result: List[MCPTool] = await client.list_tools()
+            model_dumped_tools: List[dict] = [tool.model_dump() for tool in list_tools_result]
+            return {
+                "tools": model_dumped_tools,
+                "error": None,
+                "message": "Successfully retrieved tools"
+            }
+        
+        return await _execute_with_mcp_client(request, _list_tools_operation)