Merge pull request #19491 from BerriAI/main

merge main 20 1 25

.github/workflows/ghcr_deploy.yml

@@ -320,72 +320,36 @@ jobs:
         run: |
           echo "REPO_OWNER=`echo ${{github.repository_owner}} | tr '[:upper:]' '[:lower:]'`" >>${GITHUB_ENV}
     
-      - name: Get LiteLLM Latest Tag
-        id: current_app_tag
-        shell: bash
-        run: |
-          LATEST_TAG=$(git describe --tags --exclude "*dev*" --abbrev=0)
-          if [ -z "${LATEST_TAG}" ]; then
-            echo "latest_tag=latest" | tee -a $GITHUB_OUTPUT
-          else
-            echo "latest_tag=${LATEST_TAG}" | tee -a $GITHUB_OUTPUT
-          fi
-
-      - name: Get last published chart version
-        id: current_version
-        shell: bash
-        run: |
-          CHART_LIST=$(helm show chart oci://${{ env.REGISTRY }}/${{ env.REPO_OWNER }}/${{ env.CHART_NAME }} 2>/dev/null || true)
-          if [ -z "${CHART_LIST}" ]; then
-            echo "current-version=1.0.0" | tee -a $GITHUB_OUTPUT
-          else
-            # Extract version and strip any prerelease suffix (e.g., 1.0.5-latest -> 1.0.5)
-            VERSION=$(printf '%s' "${CHART_LIST}" | grep '^version:' | awk 'BEGIN{FS=":"}{print $2}' | tr -d " " | cut -d'-' -f1)
-            echo "current-version=${VERSION}" | tee -a $GITHUB_OUTPUT
-          fi
-        env:
-          HELM_EXPERIMENTAL_OCI: '1'
-
-      # Automatically update the helm chart version one "patch" level
-      - name: Bump release version
-        id: bump_version
-        uses: christian-draeger/increment-semantic-version@1.1.0
-        with:
-          current-version: ${{ steps.current_version.outputs.current-version || '1.0.0' }}
-          version-fragment: 'bug'
-
-      # Add suffix for non-stable releases (semantic versioning)
+      # Sync Helm chart version with LiteLLM release version (1-1 versioning)
+      # This allows users to easily map Helm chart versions to LiteLLM versions
+      # See: https://codefresh.io/docs/docs/ci-cd-guides/helm-best-practices/
       - name: Calculate chart and app versions
         id: chart_version
         shell: bash
         run: |
-          BASE_VERSION="${{ steps.bump_version.outputs.next-version || '1.0.0' }}"
-          RELEASE_TYPE="${{ github.event.inputs.release_type }}"
           INPUT_TAG="${{ github.event.inputs.tag }}"
+          RELEASE_TYPE="${{ github.event.inputs.release_type }}"
 
-          # Chart version (independent Helm chart versioning with release type suffix)
-          if [ "$RELEASE_TYPE" = "stable" ]; then
-            echo "version=${BASE_VERSION}" | tee -a $GITHUB_OUTPUT
-          else
-            echo "version=${BASE_VERSION}-${RELEASE_TYPE}" | tee -a $GITHUB_OUTPUT
+          # Chart version = LiteLLM version without 'v' prefix (Helm semver convention)
+          # v1.81.0 -> 1.81.0, v1.81.0.rc.1 -> 1.81.0.rc.1
+          CHART_VERSION="${INPUT_TAG#v}"
+
+          # Add suffix for 'latest' releases (rc already has suffix in tag)
+          if [ "$RELEASE_TYPE" = "latest" ]; then
+            CHART_VERSION="${CHART_VERSION}-latest"
           fi
 
-          # App version (must match Docker tags)
-          # stable/rc releases: Docker creates main-{tag}, so use the tag
-          # latest/dev releases: Docker only creates main-{release_type}, so use release_type
-          if [ "$RELEASE_TYPE" = "stable" ] || [ "$RELEASE_TYPE" = "rc" ]; then
-            APP_VERSION="${INPUT_TAG}"
-          else
-            APP_VERSION="${RELEASE_TYPE}"
-          fi
+          # App version = Docker tag (keeps 'v' prefix to match Docker image tags)
+          APP_VERSION="${INPUT_TAG}"
 
+          echo "version=${CHART_VERSION}" | tee -a $GITHUB_OUTPUT
           echo "app_version=${APP_VERSION}" | tee -a $GITHUB_OUTPUT
 
       - uses: ./.github/actions/helm-oci-chart-releaser
         with:
           name: ${{ env.CHART_NAME }}
           repository: ${{ env.REPO_OWNER }}
-          tag: ${{ github.event.inputs.chartVersion || steps.chart_version.outputs.version || '1.0.0' }}
+          tag: ${{ steps.chart_version.outputs.version }}
           app_version: ${{ steps.chart_version.outputs.app_version }}
           path: deploy/charts/${{ env.CHART_NAME }}
           registry: ${{ env.REGISTRY }}

.github/workflows/ghcr_helm_deploy.yml

@@ -1,10 +1,12 @@
-# this workflow is triggered by an API call when there is a new PyPI release of LiteLLM
+# Standalone workflow to publish LiteLLM Helm Chart
+# Note: The main ghcr_deploy.yml workflow also publishes the Helm chart as part of a full release
 name: Build, Publish LiteLLM Helm Chart. New Release
 on:
   workflow_dispatch:
     inputs:
-      chartVersion:
-        description: "Update the helm chart's version to this"
+      tag:
+        description: "LiteLLM version tag (e.g., v1.81.0)"
+        required: true
 
 # Defines two custom environment variables for the workflow. Used for the Container registry domain, and a name for the Docker image that this workflow builds.
 env:
@@ -31,24 +33,22 @@ jobs:
         run: |
           echo "REPO_OWNER=`echo ${{github.repository_owner}} | tr '[:upper:]' '[:lower:]'`" >>${GITHUB_ENV}
 
-      - name: Get LiteLLM Latest Tag
-        id: current_app_tag
-        uses: WyriHaximus/github-action-get-previous-tag@v1.3.0
-
-      - name: Get last published chart version
-        id: current_version
+      # Sync Helm chart version with LiteLLM release version (1-1 versioning)
+      - name: Calculate chart and app versions
+        id: chart_version
         shell: bash
-        run: helm show chart oci://${{ env.REGISTRY }}/${{ env.REPO_OWNER }}/litellm-helm | grep '^version:' | awk 'BEGIN{FS=":"}{print "current-version="$2}' | tr -d " " | tee -a $GITHUB_OUTPUT
-        env:
-          HELM_EXPERIMENTAL_OCI: '1'
+        run: |
+          INPUT_TAG="${{ github.event.inputs.tag }}"
 
-      # Automatically update the helm chart version one "patch" level
-      - name: Bump release version
-        id: bump_version
-        uses: christian-draeger/increment-semantic-version@1.1.0
-        with:
-          current-version: ${{ steps.current_version.outputs.current-version || '0.1.0' }}
-          version-fragment: 'bug'
+          # Chart version = LiteLLM version without 'v' prefix
+          # v1.81.0 -> 1.81.0
+          CHART_VERSION="${INPUT_TAG#v}"
+
+          # App version = Docker tag (keeps 'v' prefix)
+          APP_VERSION="${INPUT_TAG}"
+
+          echo "version=${CHART_VERSION}" | tee -a $GITHUB_OUTPUT
+          echo "app_version=${APP_VERSION}" | tee -a $GITHUB_OUTPUT
 
       - name: Lint helm chart
         run: helm lint deploy/charts/litellm-helm
@@ -57,8 +57,8 @@ jobs:
         with:
           name: litellm-helm
           repository: ${{ env.REPO_OWNER }}
-          tag: ${{ github.event.inputs.chartVersion || steps.bump_version.outputs.next-version || '0.1.0' }}
-          app_version: ${{ steps.current_app_tag.outputs.tag || 'latest' }}
+          tag: ${{ steps.chart_version.outputs.version }}
+          app_version: ${{ steps.chart_version.outputs.app_version }}
           path: deploy/charts/litellm-helm
           registry: ${{ env.REGISTRY }}
           registry_username: ${{ github.actor }}

docker/prod_entrypoint.sh

@@ -2,6 +2,7 @@
 
 if [ "$SEPARATE_HEALTH_APP" = "1" ]; then
     export LITELLM_ARGS="$@"
+    export SUPERVISORD_STOPWAITSECS="${SUPERVISORD_STOPWAITSECS:-3600}"
     exec supervisord -c /etc/supervisord.conf
 fi
 

docker/supervisord.conf

@@ -16,6 +16,7 @@ priority=1
 exitcodes=0
 stopasgroup=true
 killasgroup=true
+stopwaitsecs=%(ENV_SUPERVISORD_STOPWAITSECS)s
 stdout_logfile=/dev/stdout
 stderr_logfile=/dev/stderr
 stdout_logfile_maxbytes = 0
@@ -31,6 +32,7 @@ priority=2
 exitcodes=0
 stopasgroup=true
 killasgroup=true
+stopwaitsecs=%(ENV_SUPERVISORD_STOPWAITSECS)s
 stdout_logfile=/dev/stdout
 stderr_logfile=/dev/stderr
 stdout_logfile_maxbytes = 0

docs/my-website/docs/completion/json_mode.md

@@ -341,4 +341,90 @@ curl http://0.0.0.0:4000/v1/chat/completions \
 ```
 
 </TabItem>
-</Tabs>
+</Tabs>
+
+## Gemini - Native JSON Schema Format (Gemini 2.0+)
+
+Gemini 2.0+ models automatically use the native `responseJsonSchema` parameter, which provides better compatibility with standard JSON Schema format.
+
+### Benefits (Gemini 2.0+):
+- Standard JSON Schema format (lowercase types like `string`, `object`)
+- Supports `additionalProperties: false` for stricter validation
+- Better compatibility with Pydantic's `model_json_schema()`
+- No `propertyOrdering` required
+
+### Usage
+
+<Tabs>
+<TabItem value="sdk" label="SDK">
+
+```python
+from litellm import completion
+from pydantic import BaseModel
+
+class UserInfo(BaseModel):
+    name: str
+    age: int
+
+response = completion(
+    model="gemini/gemini-2.0-flash",
+    messages=[{"role": "user", "content": "Extract: John is 25 years old"}],
+    response_format={
+        "type": "json_schema",
+        "json_schema": {
+            "name": "user_info",
+            "schema": {
+                "type": "object",
+                "properties": {
+                    "name": {"type": "string"},
+                    "age": {"type": "integer"}
+                },
+                "required": ["name", "age"],
+                "additionalProperties": False  # Supported on Gemini 2.0+
+            }
+        }
+    }
+)
+```
+
+</TabItem>
+<TabItem value="proxy" label="PROXY">
+
+```bash
+curl http://0.0.0.0:4000/v1/chat/completions \
+  -H "Content-Type: application/json" \
+  -H "Authorization: Bearer $LITELLM_API_KEY" \
+  -d '{
+    "model": "gemini-2.0-flash",
+    "messages": [
+        {"role": "user", "content": "Extract: John is 25 years old"}
+    ],
+    "response_format": {
+        "type": "json_schema",
+        "json_schema": {
+            "name": "user_info",
+            "schema": {
+                "type": "object",
+                "properties": {
+                    "name": {"type": "string"},
+                    "age": {"type": "integer"}
+                },
+                "required": ["name", "age"],
+                "additionalProperties": false
+            }
+        }
+    }
+  }'
+```
+
+</TabItem>
+</Tabs>
+
+### Model Behavior
+
+| Model | Format Used | `additionalProperties` Support |
+|-------|-------------|-------------------------------|
+| Gemini 2.0+ | `responseJsonSchema` (JSON Schema) | ✅ Yes |
+| Gemini 1.5 | `responseSchema` (OpenAPI) | ❌ No |
+
+LiteLLM automatically selects the appropriate format based on the model version.

docs/my-website/docs/providers/chatgpt.md

@@ -0,0 +1,84 @@
+# ChatGPT Subscription
+
+Use ChatGPT Pro/Max subscription models through LiteLLM with OAuth device flow authentication.
+
+| Property | Details |
+|-------|-------|
+| Description | ChatGPT subscription access (Codex + GPT-5.2 family) via ChatGPT backend API |
+| Provider Route on LiteLLM | `chatgpt/` |
+| Supported Endpoints | `/responses`, `/chat/completions` (bridged to Responses for supported models) |
+| API Reference | https://chatgpt.com |
+
+ChatGPT subscription access is native to the Responses API. Chat Completions requests are bridged to Responses for supported models (for example `chatgpt/gpt-5.2`).
+
+Notes:
+- The ChatGPT subscription backend rejects token limit fields (`max_tokens`, `max_output_tokens`, `max_completion_tokens`) and `metadata`. LiteLLM strips these fields for this provider.
+- `/v1/chat/completions` honors `stream`. When `stream` is false (default), LiteLLM aggregates the Responses stream into a single JSON response.
+
+## Authentication
+
+ChatGPT subscription access uses an OAuth device code flow:
+
+1. LiteLLM prints a device code and verification URL
+2. Open the URL, sign in, and enter the code
+3. Tokens are stored locally for reuse
+
+## Usage - LiteLLM Python SDK
+
+### Responses (recommended for Codex models)
+
+```python showLineNumbers title="ChatGPT Responses"
+import litellm
+
+response = litellm.responses(
+    model="chatgpt/gpt-5.2-codex",
+    input="Write a Python hello world"
+)
+
+print(response)
+```
+
+### Chat Completions (bridged to Responses)
+
+```python showLineNumbers title="ChatGPT Chat Completions"
+import litellm
+
+response = litellm.completion(
+    model="chatgpt/gpt-5.2",
+    messages=[{"role": "user", "content": "Write a Python hello world"}]
+)
+
+print(response)
+```
+
+## Usage - LiteLLM Proxy
+
+```yaml showLineNumbers title="config.yaml"
+model_list:
+  - model_name: chatgpt/gpt-5.2
+    model_info:
+      mode: responses
+    litellm_params:
+      model: chatgpt/gpt-5.2
+  - model_name: chatgpt/gpt-5.2-codex
+    model_info:
+      mode: responses
+    litellm_params:
+      model: chatgpt/gpt-5.2-codex
+```
+
+```bash showLineNumbers title="Start LiteLLM Proxy"
+litellm --config config.yaml
+```
+
+## Configuration
+
+### Environment Variables
+
+- `CHATGPT_TOKEN_DIR`: Custom token storage directory
+- `CHATGPT_AUTH_FILE`: Auth file name (default: `auth.json`)
+- `CHATGPT_API_BASE`: Override API base (default: `https://chatgpt.com/backend-api/codex`)
+- `OPENAI_CHATGPT_API_BASE`: Alias for `CHATGPT_API_BASE`
+- `CHATGPT_ORIGINATOR`: Override the `originator` header value
+- `CHATGPT_USER_AGENT`: Override the `User-Agent` header value
+- `CHATGPT_USER_AGENT_SUFFIX`: Optional suffix appended to the `User-Agent` header

docs/my-website/docs/providers/gemini.md

@@ -15,6 +15,17 @@ import TabItem from '@theme/TabItem';
 
 <br />
 
+:::tip Gemini API vs Vertex AI
+| Model Format | Provider | Auth Required |
+|-------------|----------|---------------|
+| `gemini/gemini-2.0-flash` | Gemini API | `GEMINI_API_KEY` (simple API key) |
+| `vertex_ai/gemini-2.0-flash` | Vertex AI | GCP credentials + project |
+| `gemini-2.0-flash` (no prefix) | Vertex AI | GCP credentials + project |
+
+**If you just want to use an API key** (like OpenAI), use the `gemini/` prefix.
+
+Models without a prefix default to Vertex AI which requires full GCP authentication.
+:::
 
 ## API Keys
 

docs/my-website/docs/providers/vertex.md

@@ -14,6 +14,17 @@ import TabItem from '@theme/TabItem';
 | Base URL | 1. Regional endpoints<br/>`https://{vertex_location}-aiplatform.googleapis.com/`<br/>2. Global endpoints (limited availability)<br/>`https://aiplatform.googleapis.com/`|
 | Supported Operations | [`/chat/completions`](#sample-usage), `/completions`, [`/embeddings`](#embedding-models), [`/audio/speech`](#text-to-speech-apis), [`/fine_tuning`](#fine-tuning-apis), [`/batches`](#batch-apis), [`/files`](#batch-apis), [`/images`](#image-generation-models), [`/rerank`](#rerank-api) |
 
+:::tip Vertex AI vs Gemini API
+| Model Format | Provider | Auth Required |
+|-------------|----------|---------------|
+| `vertex_ai/gemini-2.0-flash` | Vertex AI | GCP credentials + project |
+| `gemini-2.0-flash` (no prefix) | Vertex AI | GCP credentials + project |
+| `gemini/gemini-2.0-flash` | Gemini API | `GEMINI_API_KEY` (simple API key) |
+
+**If you just want to use an API key** (like OpenAI), use the `gemini/` prefix instead. See [Gemini - Google AI Studio](./gemini.md).
+
+Models without a prefix default to Vertex AI which requires GCP authentication.
+:::
 
 <br />
 <br />

docs/my-website/docs/proxy/config_settings.md

@@ -397,6 +397,7 @@ router_settings:
 | AUDIO_SPEECH_CHUNK_SIZE | Chunk size for audio speech processing. Default is 1024
 | ANTHROPIC_API_KEY | API key for Anthropic service
 | ANTHROPIC_API_BASE | Base URL for Anthropic API. Default is https://api.anthropic.com
+| ANTHROPIC_TOKEN_COUNTING_BETA_VERSION | Beta version header for Anthropic token counting API. Default is `token-counting-2024-11-01`
 | AWS_ACCESS_KEY_ID | Access Key ID for AWS services
 | AWS_BATCH_ROLE_ARN | ARN of the AWS IAM role for batch operations
 | AWS_DEFAULT_REGION | Default AWS region for service interactions when AWS_REGION is not set
@@ -412,6 +413,8 @@ router_settings:
 | AWS_WEB_IDENTITY_TOKEN | Web identity token for AWS
 | AWS_WEB_IDENTITY_TOKEN_FILE | Path to file containing web identity token for AWS
 | AZURE_API_VERSION | Version of the Azure API being used
+| AZURE_AI_API_BASE | Base URL for Azure AI services (e.g., Azure AI Anthropic)
+| AZURE_AI_API_KEY | API key for Azure AI services (e.g., Azure AI Anthropic)
 | AZURE_AUTHORITY_HOST | Azure authority host URL
 | AZURE_CERTIFICATE_PASSWORD | Password for Azure OpenAI certificate
 | AZURE_CLIENT_ID | Client ID for Azure services
@@ -449,6 +452,13 @@ router_settings:
 | BRAINTRUST_API_KEY | API key for Braintrust integration
 | BRAINTRUST_API_BASE | Base URL for Braintrust API. Default is https://api.braintrustdata.com/v1
 | CACHED_STREAMING_CHUNK_DELAY | Delay in seconds for cached streaming chunks. Default is 0.02
+| CHATGPT_API_BASE | Base URL for ChatGPT API. Default is https://chatgpt.com/backend-api/codex
+| CHATGPT_AUTH_FILE | Filename for ChatGPT authentication data. Default is "auth.json"
+| CHATGPT_DEFAULT_INSTRUCTIONS | Default system instructions for ChatGPT provider
+| CHATGPT_ORIGINATOR | Originator identifier for ChatGPT API requests. Default is "codex_cli_rs"
+| CHATGPT_TOKEN_DIR | Directory to store ChatGPT authentication tokens. Default is "~/.config/litellm/chatgpt"
+| CHATGPT_USER_AGENT | Custom user agent string for ChatGPT API requests
+| CHATGPT_USER_AGENT_SUFFIX | Suffix to append to the ChatGPT user agent string
 | CIRCLE_OIDC_TOKEN | OpenID Connect token for CircleCI
 | CIRCLE_OIDC_TOKEN_V2 | Version 2 of the OpenID Connect token for CircleCI
 | CLOUDZERO_API_KEY | CloudZero API key for authentication
@@ -794,6 +804,7 @@ router_settings:
 | OPENAI_BASE_URL | Base URL for OpenAI API
 | OPENAI_API_BASE | Base URL for OpenAI API. Default is https://api.openai.com/
 | OPENAI_API_KEY | API key for OpenAI services
+| OPENAI_CHATGPT_API_BASE | Alternative to CHATGPT_API_BASE. Base URL for ChatGPT API
 | OPENAI_FILE_SEARCH_COST_PER_1K_CALLS | Cost per 1000 calls for OpenAI file search. Default is 0.0025
 | OPENAI_ORGANIZATION | Organization identifier for OpenAI
 | OPENID_BASE_URL | Base URL for OpenID Connect services
@@ -867,6 +878,7 @@ router_settings:
 | SECRET_MANAGER_REFRESH_INTERVAL | Refresh interval in seconds for secret manager. Default is 86400 (24 hours)
 | SEPARATE_HEALTH_APP | If set to '1', runs health endpoints on a separate ASGI app and port. Default: '0'.
 | SEPARATE_HEALTH_PORT | Port for the separate health endpoints app. Only used if SEPARATE_HEALTH_APP=1. Default: 4001.
+| SUPERVISORD_STOPWAITSECS | Upper bound timeout in seconds for graceful shutdown when SEPARATE_HEALTH_APP=1. Default: 3600 (1 hour).
 | SERVER_ROOT_PATH | Root path for the server application
 | SEND_USER_API_KEY_ALIAS | Flag to send user API key alias to Zscaler AI Guard. Default is False
 | SEND_USER_API_KEY_TEAM_ID | Flag to send user API key team ID to Zscaler AI Guard. Default is False

docs/my-website/docs/proxy/prod.md

@@ -277,8 +277,13 @@ Set the following environment variable(s):
 ```bash
 SEPARATE_HEALTH_APP="1" # Default "0" 
 SEPARATE_HEALTH_PORT="8001" # Default "4001", Works only if `SEPARATE_HEALTH_APP` is "1"
+SUPERVISORD_STOPWAITSECS="3600" # Optional: Upper bound timeout in seconds for graceful shutdown. Default: 3600 (1 hour). Only used when SEPARATE_HEALTH_APP=1.
 ```
 
+**Graceful Shutdown:**
+
+Previously, `stopwaitsecs` was not set, defaulting to 10 seconds and causing in-flight requests to fail. `SUPERVISORD_STOPWAITSECS` (default: 3600) provides an upper bound for graceful shutdown, allowing uvicorn to wait for all in-flight requests to complete.
+
 <video controls width="100%" style={{ borderRadius: '8px', marginBottom: '1em' }}>
   <source src="https://cdn.loom.com/sessions/thumbnails/b08be303331246b88fdc053940d03281-1718990992822.mp4" type="video/mp4" />
   Your browser does not support the video tag.

docs/my-website/docs/proxy/users.md

@@ -545,6 +545,26 @@ You can set:
 - max parallel requests
 - rpm / tpm limits per model for a given key
 
+### TPM Rate Limit Type (Input/Output/Total)
+
+By default, TPM (tokens per minute) rate limits count **total tokens** (input + output). You can configure this to count only input tokens or only output tokens instead.
+
+Set `token_rate_limit_type` in your `config.yaml`:
+
+```yaml
+general_settings:
+  master_key: sk-1234
+  token_rate_limit_type: "output"  # Options: "input", "output", "total" (default)
+```
+
+| Value | Description |
+|-------|-------------|
+| `total` | Count total tokens (prompt + completion). **Default behavior.** |
+| `input` | Count only prompt/input tokens |
+| `output` | Count only completion/output tokens |
+
+This setting applies globally to all TPM rate limit checks (keys, users, teams, etc.).
+
 
 <Tabs>
 <TabItem value="per-team" label="Per Team">

docs/my-website/docs/tutorials/claude_code_max_subscription.md

@@ -0,0 +1,357 @@
+import Image from '@theme/IdealImage';
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+
+# Using Claude Code Max Subscription
+
+<div style={{ textAlign: 'center' }}>
+<Image img={require('../../img/claude_code_max.png')} style={{ width: '100%', maxWidth: '800px', height: 'auto' }} />
+
+Route Claude Code Max subscription traffic through LiteLLM AI Gateway.
+</div>
+
+**Why Claude Code Max over direct API?**
+- **Lower costs** — Claude Code Max subscriptions are cheaper for Claude Code power users than per-token API pricing
+
+**Why route through LiteLLM?**
+- **Cost attribution** — Track spend per user, team, or key
+- **Budgets & rate limits** — Set spending caps and request limits
+- **Guardrails** — Apply content filtering and safety controls to all requests
+
+
+
+## Quick Start Video
+
+Watch the end-to-end walkthrough of setting up Claude Code with LiteLLM Gateway:
+
+<iframe width="840" height="500" src="https://www.loom.com/embed/2d069b9e3bcc4cecaa5eb27a72ba7b3c" frameborder="0" webkitallowfullscreen mozallowfullscreen allowfullscreen></iframe>
+
+## Prerequisites
+
+- [Claude Code](https://docs.anthropic.com/en/docs/claude-code/overview) installed
+- Claude Max subscription
+- LiteLLM Gateway running
+
+## Step 1: Configure LiteLLM Proxy
+
+Create a `config.yaml` with the critical `forward_client_headers_to_llm_api: true` setting:
+
+```yaml showLineNumbers title="config.yaml"
+model_list:
+  - model_name: anthropic-claude
+    litellm_params:
+      model: anthropic/claude-sonnet-4-20250514
+
+  - model_name: claude-3-5-sonnet-20241022
+    litellm_params:
+      model: anthropic/claude-3-5-sonnet-20241022
+
+  - model_name: claude-3-5-haiku-20241022
+    litellm_params:
+      model: anthropic/claude-3-5-haiku-20241022
+
+general_settings:
+  forward_client_headers_to_llm_api: true  # Required: forwards OAuth token to Anthropic
+
+litellm_settings:
+  master_key: os.environ/LITELLM_MASTER_KEY
+```
+
+:::info Why `forward_client_headers_to_llm_api`?
+
+This setting forwards the user's OAuth token (in the `Authorization` header) through LiteLLM to the Anthropic API, enabling per-user authentication with their Max subscription while LiteLLM handles tracking and controls.
+
+:::
+
+## Step 2: Start LiteLLM Proxy
+
+```bash showLineNumbers title="Start LiteLLM Proxy"
+litellm --config /path/to/config.yaml
+
+# RUNNING on http://0.0.0.0:4000
+```
+
+## Walkthrough
+
+### Part 1: Create a Virtual Key in LiteLLM
+
+Navigate to the LiteLLM Dashboard and create a new virtual key for Claude Code usage.
+
+#### 1.1 Open Virtual Keys Page
+
+Navigate to the Virtual Keys section in the LiteLLM Dashboard.
+
+<Image img={require('../../img/claude_code_max/step1.jpeg')} style={{ width: '800px', height: 'auto' }} />
+
+#### 1.2 Click "Create New Key"
+
+<Image img={require('../../img/claude_code_max/step2.jpeg')} style={{ width: '800px', height: 'auto' }} />
+
+#### 1.3 Configure Key Details
+
+Enter a key name (e.g., `claude-code-test`) and select the models you want to allow access to.
+
+<Image img={require('../../img/claude_code_max/step3.jpeg')} style={{ width: '800px', height: 'auto' }} />
+
+#### 1.4 Select Models
+
+Choose the Anthropic models that should be accessible via this key (e.g., `anthropic-claude`, `claude-4.5-haiku`).
+
+<Image img={require('../../img/claude_code_max/step5.jpeg')} style={{ width: '800px', height: 'auto' }} />
+
+#### 1.5 Confirm Model Selection
+
+<Image img={require('../../img/claude_code_max/step7.jpeg')} style={{ width: '800px', height: 'auto' }} />
+
+#### 1.6 Create the Key
+
+Click "Create Key" to generate your virtual key. Copy the generated key value (e.g., `sk-otsclFlEblQ-6D60ua2IZg`).
+
+<Image img={require('../../img/claude_code_max/step8.jpeg')} style={{ width: '800px', height: 'auto' }} />
+
+---
+
+### Part 2: Sign into Claude Code Max Plan (Client Side)
+
+Set up Claude Code environment variables and authenticate with your Max subscription.
+
+#### 2.1 Set Environment Variables
+
+Configure Claude Code to use LiteLLM Gateway with your virtual key:
+
+```bash showLineNumbers title="Configure Claude Code Environment Variables"
+export ANTHROPIC_BASE_URL=http://localhost:4000
+export ANTHROPIC_MODEL="anthropic-claude"
+export ANTHROPIC_CUSTOM_HEADERS="x-litellm-api-key: Bearer sk-otsclFlEblQ-6D60ua2IZg"
+```
+
+<Image img={require('../../img/claude_code_max/step15.jpeg')} style={{ width: '800px', height: 'auto' }} />
+
+#### Environment Variables Explained
+
+| Variable | Description |
+|----------|-------------|
+| `ANTHROPIC_BASE_URL` | Points Claude Code to your LiteLLM Gateway endpoint |
+| `ANTHROPIC_MODEL` | The model name configured in your LiteLLM `config.yaml` |
+| `ANTHROPIC_CUSTOM_HEADERS` | The `x-litellm-api-key` header for LiteLLM authentication |
+
+#### 2.2 Launch Claude Code
+
+Start Claude Code:
+
+```bash showLineNumbers title="Launch Claude Code"
+claude
+```
+
+<Image img={require('../../img/claude_code_max/step16.jpeg')} style={{ width: '800px', height: 'auto' }} />
+
+#### 2.3 Select Login Method
+
+Choose "Claude account with subscription" (Pro, Max, Team, or Enterprise).
+
+<Image img={require('../../img/claude_code_max/step17.jpeg')} style={{ width: '800px', height: 'auto' }} />
+
+#### 2.4 Authorize in Browser
+
+Claude Code opens your browser to authenticate. Click "Authorize" to connect your Claude Max account.
+
+<Image img={require('../../img/claude_code_max/step19.jpeg')} style={{ width: '800px', height: 'auto' }} />
+
+#### 2.5 Login Successful
+
+After authorization, you'll see the login success confirmation.
+
+<Image img={require('../../img/claude_code_max/step20.jpeg')} style={{ width: '800px', height: 'auto' }} />
+
+#### 2.6 Complete Setup
+
+Press Enter to continue past the security notes and complete the setup.
+
+<Image img={require('../../img/claude_code_max/step21.jpeg')} style={{ width: '800px', height: 'auto' }} />
+
+---
+
+### Part 3: Use Claude Code with LiteLLM
+
+Now you can use Claude Code normally, and all requests will be tracked in LiteLLM.
+
+#### 3.1 Make a Request in Claude Code
+
+Start using Claude Code - requests will flow through LiteLLM Gateway.
+
+<Image img={require('../../img/claude_code_max/step24.jpeg')} style={{ width: '800px', height: 'auto' }} />
+
+#### 3.2 View Logs in LiteLLM Dashboard
+
+Navigate to the Logs page in LiteLLM Dashboard to see all Claude Code requests.
+
+<Image img={require('../../img/claude_code_max/step25.jpeg')} style={{ width: '800px', height: 'auto' }} />
+
+#### 3.3 View Request Details
+
+Click on a request to see detailed information including tokens, cost, duration, and model used.
+
+<Image img={require('../../img/claude_code_max/step27.jpeg')} style={{ width: '800px', height: 'auto' }} />
+
+The logs show:
+- **Key Name**: `claude-code-test` (the virtual key you created)
+- **Model**: `anthropic/claude-sonnet-4-20250514`
+- **Tokens**: 65012 (64679 prompt + 333 completion)
+- **Cost**: $0.249754
+- **Status**: Success
+
+<Image img={require('../../img/claude_code_max/step28.jpeg')} style={{ width: '800px', height: 'auto' }} />
+
+---
+
+## How It Works
+
+LiteLLM Gateway handles two types of authentication:
+1. **`x-litellm-api-key`**: Authenticates the request with LiteLLM (usage tracking, budgets, rate limits)
+2. **OAuth Token (via `Authorization` header)**: Forwarded to Anthropic API for Claude Max authentication
+
+```mermaid
+sequenceDiagram
+    participant User as Claude Code User
+    participant LiteLLM as LiteLLM AI Gateway
+    participant Anthropic as Anthropic API
+
+    User->>LiteLLM: Request with:<br/>- x-litellm-api-key (LiteLLM auth)<br/>- Authorization: Bearer {oauth_token}
+
+    Note over LiteLLM: 1. Validate x-litellm-api-key<br/>2. Check budgets/rate limits<br/>3. Log request for tracking
+
+    LiteLLM->>Anthropic: Forward request with:<br/>- Authorization: Bearer {oauth_token}<br/>(User's Claude Max OAuth token)
+
+    Note over Anthropic: Authenticate user via<br/>OAuth token from Max plan
+
+    Anthropic-->>LiteLLM: Response
+
+    Note over LiteLLM: Log usage, tokens, cost
+
+    LiteLLM-->>User: Response
+```
+
+### Header Flow
+
+| Header | Purpose | Handled By |
+|--------|---------|------------|
+| `x-litellm-api-key` | LiteLLM Gateway authentication, budget tracking, rate limits | LiteLLM |
+| `Authorization: Bearer {oauth_token}` | Claude Max subscription authentication | Anthropic API |
+
+### Complete Request Flow Example
+
+Here's what a typical request looks like when Claude Code makes a call through LiteLLM:
+
+```bash showLineNumbers title="Example Request from Claude Code to LiteLLM"
+curl -X POST "http://localhost:4000/v1/messages" \
+  -H "x-litellm-api-key: Bearer sk-otsclFlEblQ-6D60ua2IZg" \
+  -H "Authorization: Bearer oauth_token_from_max_plan" \
+  -H "Content-Type: application/json" \
+  -d '{
+    "model": "anthropic-claude",
+    "max_tokens": 1024,
+    "messages": [{"role": "user", "content": "Hello, Claude!"}]
+  }'
+```
+
+LiteLLM then:
+1. Validates `x-litellm-api-key` for gateway access
+2. Logs the request for usage tracking
+3. Forwards the request to Anthropic with the OAuth `Authorization` header (because of `forward_client_headers_to_llm_api: true`)
+
+## Advanced Configuration
+
+### Per-Model Header Forwarding
+
+For more granular control, you can enable header forwarding only for specific models:
+
+```yaml showLineNumbers title="config.yaml - Per-Model Header Forwarding"
+model_list:
+  - model_name: anthropic-claude
+    litellm_params:
+      model: anthropic/claude-sonnet-4-20250514
+
+  - model_name: claude-3-5-haiku-20241022
+    litellm_params:
+      model: anthropic/claude-3-5-haiku-20241022
+
+litellm_settings:
+  master_key: os.environ/LITELLM_MASTER_KEY
+  model_group_settings:
+    forward_client_headers_to_llm_api:
+      - anthropic-claude
+      - claude-3-5-haiku-20241022
+```
+
+### Budget Controls
+
+Set up per-user budgets while using Max subscriptions:
+
+```yaml showLineNumbers title="config.yaml - With Database for Budget Tracking"
+model_list:
+  - model_name: anthropic-claude
+    litellm_params:
+      model: anthropic/claude-sonnet-4-20250514
+
+general_settings:
+  forward_client_headers_to_llm_api: true
+  database_url: "postgresql://..."
+
+litellm_settings:
+  master_key: os.environ/LITELLM_MASTER_KEY
+```
+
+Then create virtual keys with budgets:
+
+```bash showLineNumbers title="Create Virtual Key with Budget"
+curl -X POST "http://localhost:4000/key/generate" \
+  -H "Authorization: Bearer $LITELLM_MASTER_KEY" \
+  -H "Content-Type: application/json" \
+  -d '{
+    "key_alias": "developer-1",
+    "max_budget": 100.00,
+    "budget_duration": "monthly"
+  }'
+```
+
+## Troubleshooting
+
+### OAuth Token Not Being Forwarded
+
+**Symptom**: Authentication errors from Anthropic API
+
+**Solution**: Ensure `forward_client_headers_to_llm_api: true` is set in your config:
+
+```yaml showLineNumbers title="config.yaml - Enable Header Forwarding"
+general_settings:
+  forward_client_headers_to_llm_api: true
+```
+
+### LiteLLM Authentication Failing
+
+**Symptom**: 401 errors from LiteLLM Gateway
+
+**Solution**: Verify `x-litellm-api-key` header is set correctly in `ANTHROPIC_CUSTOM_HEADERS`:
+
+```bash showLineNumbers title="Verify Key Info"
+curl -X GET "http://localhost:4000/key/info" \
+  -H "Authorization: Bearer sk-otsclFlEblQ-6D60ua2IZg"
+```
+
+### Model Not Found
+
+**Symptom**: Model not found errors
+
+**Solution**: Ensure the `ANTHROPIC_MODEL` matches a model name in your config:
+
+```bash showLineNumbers title="List Available Models"
+curl "http://localhost:4000/v1/models" \
+  -H "Authorization: Bearer sk-otsclFlEblQ-6D60ua2IZg"
+```
+
+## Related Documentation
+
+- [Forward Client Headers](/docs/proxy/forward_client_headers) - Detailed header forwarding configuration
+- [Claude Code Quickstart](/docs/tutorials/claude_responses_api) - Basic Claude Code + LiteLLM setup
+- [Virtual Keys](/docs/proxy/virtual_keys) - Creating and managing API keys
+- [Budgets & Rate Limits](/docs/proxy/users) - Setting up usage controls

docs/my-website/release_notes/v1.81.0/index.md

@@ -27,7 +27,7 @@ import TabItem from '@theme/TabItem';
 docker run \
 -e STORE_MODEL_IN_DB=True \
 -p 4000:4000 \
-docker.litellm.ai/berriai/litellm:v1.81.0
+docker.litellm.ai/berriai/litellm:v1.81.0.rc.1
 ```
 
 </TabItem>

docs/my-website/sidebars.js

@@ -121,6 +121,7 @@ const sidebars = {
           label: "Claude Code",
           items: [
             "tutorials/claude_responses_api",
+            "tutorials/claude_code_max_subscription",
             "tutorials/claude_code_customer_tracking",
             "tutorials/claude_code_websearch",
             "tutorials/claude_mcp",
@@ -718,6 +719,7 @@ const sidebars = {
         "providers/galadriel",
         "providers/github",
         "providers/github_copilot",
+        "providers/chatgpt",
         "providers/gradient_ai",
         "providers/groq",
         "providers/helicone",

litellm-proxy-extras/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "litellm-proxy-extras"
-version = "0.4.23"
+version = "0.4.25"
 description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package."
 authors = ["BerriAI"]
 readme = "README.md"
@@ -22,7 +22,7 @@ requires = ["poetry-core"]
 build-backend = "poetry.core.masonry.api"
 
 [tool.commitizen]
-version = "0.4.23"
+version = "0.4.25"
 version_files = [
     "pyproject.toml:version",
     "../requirements.txt:litellm-proxy-extras==",

litellm/__init__.py

@@ -557,6 +557,7 @@ docker_model_runner_models: Set = set()
 amazon_nova_models: Set = set()
 stability_models: Set = set()
 github_copilot_models: Set = set()
+chatgpt_models: Set = set()
 minimax_models: Set = set()
 aws_polly_models: Set = set()
 gigachat_models: Set = set()
@@ -812,6 +813,8 @@ def add_known_models():
             stability_models.add(key)
         elif value.get("litellm_provider") == "github_copilot":
             github_copilot_models.add(key)
+        elif value.get("litellm_provider") == "chatgpt":
+            chatgpt_models.add(key)
         elif value.get("litellm_provider") == "minimax":
             minimax_models.add(key)
         elif value.get("litellm_provider") == "aws_polly":
@@ -1025,6 +1028,7 @@ models_by_provider: dict = {
     "amazon_nova": amazon_nova_models,
     "stability": stability_models,
     "github_copilot": github_copilot_models,
+    "chatgpt": chatgpt_models,
     "minimax": minimax_models,
     "aws_polly": aws_polly_models,
     "gigachat": gigachat_models,
@@ -1459,6 +1463,8 @@ if TYPE_CHECKING:
     from .llms.github_copilot.chat.transformation import GithubCopilotConfig as GithubCopilotConfig
     from .llms.github_copilot.responses.transformation import GithubCopilotResponsesAPIConfig as GithubCopilotResponsesAPIConfig
     from .llms.github_copilot.embedding.transformation import GithubCopilotEmbeddingConfig as GithubCopilotEmbeddingConfig
+    from .llms.chatgpt.chat.transformation import ChatGPTConfig as ChatGPTConfig
+    from .llms.chatgpt.responses.transformation import ChatGPTResponsesAPIConfig as ChatGPTResponsesAPIConfig
     from .llms.gigachat.chat.transformation import GigaChatConfig as GigaChatConfig
     from .llms.gigachat.embedding.transformation import GigaChatEmbeddingConfig as GigaChatEmbeddingConfig
     from .llms.nebius.chat.transformation import NebiusConfig as NebiusConfig

litellm/_lazy_imports_registry.py

@@ -254,6 +254,8 @@ LLM_CONFIG_NAMES = (
     "IBMWatsonXAudioTranscriptionConfig",
     "GithubCopilotConfig",
     "GithubCopilotResponsesAPIConfig",
+    "ChatGPTConfig",
+    "ChatGPTResponsesAPIConfig",
     "ManusResponsesAPIConfig",
     "GithubCopilotEmbeddingConfig",
     "NebiusConfig",
@@ -650,6 +652,8 @@ _LLM_CONFIGS_IMPORT_MAP = {
     "GithubCopilotConfig": (".llms.github_copilot.chat.transformation", "GithubCopilotConfig"),
     "GithubCopilotResponsesAPIConfig": (".llms.github_copilot.responses.transformation", "GithubCopilotResponsesAPIConfig"),
     "GithubCopilotEmbeddingConfig": (".llms.github_copilot.embedding.transformation", "GithubCopilotEmbeddingConfig"),
+    "ChatGPTConfig": (".llms.chatgpt.chat.transformation", "ChatGPTConfig"),
+    "ChatGPTResponsesAPIConfig": (".llms.chatgpt.responses.transformation", "ChatGPTResponsesAPIConfig"),
     "NebiusConfig": (".llms.nebius.chat.transformation", "NebiusConfig"),
     "WandbConfig": (".llms.wandb.chat.transformation", "WandbConfig"),
     "GigaChatConfig": (".llms.gigachat.chat.transformation", "GigaChatConfig"),

litellm/_service_logger.py

@@ -145,16 +145,19 @@ class ServiceLogging(CustomLogger):
                     event_metadata=event_metadata,
                 )
             elif callback == "otel" or isinstance(callback, OpenTelemetry):
-                from litellm.proxy.proxy_server import open_telemetry_logger
+                _otel_logger_to_use: Optional[OpenTelemetry] = None
+                if isinstance(callback, OpenTelemetry):
+                    _otel_logger_to_use = callback
+                else:
+                    from litellm.proxy.proxy_server import open_telemetry_logger
 
-                await self.init_otel_logger_if_none()
+                    if open_telemetry_logger is not None and isinstance(
+                        open_telemetry_logger, OpenTelemetry
+                    ):
+                        _otel_logger_to_use = open_telemetry_logger
 
-                if (
-                    parent_otel_span is not None
-                    and open_telemetry_logger is not None
-                    and isinstance(open_telemetry_logger, OpenTelemetry)
-                ):
-                    await self.otel_logger.async_service_success_hook(
+                if _otel_logger_to_use is not None and parent_otel_span is not None:
+                    await _otel_logger_to_use.async_service_success_hook(
                         payload=payload,
                         parent_otel_span=parent_otel_span,
                         start_time=start_time,
@@ -253,20 +256,24 @@ class ServiceLogging(CustomLogger):
                     event_metadata=event_metadata,
                 )
             elif callback == "otel" or isinstance(callback, OpenTelemetry):
-                from litellm.proxy.proxy_server import open_telemetry_logger
+                _otel_logger_to_use: Optional[OpenTelemetry] = None
+                if isinstance(callback, OpenTelemetry):
+                    _otel_logger_to_use = callback
+                else:
+                    from litellm.proxy.proxy_server import open_telemetry_logger
 
-                await self.init_otel_logger_if_none()
+                    if open_telemetry_logger is not None and isinstance(
+                        open_telemetry_logger, OpenTelemetry
+                    ):
+                        _otel_logger_to_use = open_telemetry_logger
 
                 if not isinstance(error, str):
                     error = str(error)
 
-                if (
-                    parent_otel_span is not None
-                    and open_telemetry_logger is not None
-                    and isinstance(open_telemetry_logger, OpenTelemetry)
-                ):
-                    await self.otel_logger.async_service_success_hook(
+                if _otel_logger_to_use is not None and parent_otel_span is not None:
+                    await _otel_logger_to_use.async_service_failure_hook(
                         payload=payload,
+                        error=error,
                         parent_otel_span=parent_otel_span,
                         start_time=start_time,
                         end_time=end_time,

litellm/completion_extras/litellm_responses_transformation/handler.py

@@ -2,10 +2,12 @@
 Handler for transforming /chat/completions api requests to litellm.responses requests
 """
 
-from typing import TYPE_CHECKING, Any, Coroutine, Union
+from typing import TYPE_CHECKING, Any, Coroutine, Optional, Union
 
 from typing_extensions import TypedDict
 
+from litellm.types.llms.openai import ResponsesAPIResponse
+
 if TYPE_CHECKING:
     from litellm import CustomStreamWrapper, LiteLLMLoggingObj, ModelResponse
 
@@ -28,6 +30,71 @@ class ResponsesToCompletionBridgeHandler:
         super().__init__()
         self.transformation_handler = LiteLLMResponsesTransformationHandler()
 
+    @staticmethod
+    def _resolve_stream_flag(optional_params: dict, litellm_params: dict) -> bool:
+        stream = optional_params.get("stream")
+        if stream is None:
+            stream = litellm_params.get("stream", False)
+        return bool(stream)
+
+    @staticmethod
+    def _coerce_response_object(
+        response_obj: Any,
+        hidden_params: Optional[dict],
+    ) -> "ResponsesAPIResponse":
+        if isinstance(response_obj, ResponsesAPIResponse):
+            response = response_obj
+        elif isinstance(response_obj, dict):
+            try:
+                response = ResponsesAPIResponse(**response_obj)
+            except Exception:
+                response = ResponsesAPIResponse.model_construct(**response_obj)
+        else:
+            raise ValueError("Unexpected responses stream payload")
+
+        if hidden_params:
+            existing = getattr(response, "_hidden_params", None)
+            if not isinstance(existing, dict) or not existing:
+                setattr(response, "_hidden_params", dict(hidden_params))
+            else:
+                for key, value in hidden_params.items():
+                    existing.setdefault(key, value)
+        return response
+
+    def _collect_response_from_stream(
+        self, stream_iter: Any
+    ) -> "ResponsesAPIResponse":
+        for _ in stream_iter:
+            pass
+
+        completed = getattr(stream_iter, "completed_response", None)
+        response_obj = getattr(completed, "response", None) if completed else None
+        if response_obj is None:
+            raise ValueError("Stream ended without a completed response")
+
+        hidden_params = getattr(stream_iter, "_hidden_params", None)
+        response = self._coerce_response_object(response_obj, hidden_params)
+        if not isinstance(response, ResponsesAPIResponse):
+            raise ValueError("Stream completed response is invalid")
+        return response
+
+    async def _collect_response_from_stream_async(
+        self, stream_iter: Any
+    ) -> "ResponsesAPIResponse":
+        async for _ in stream_iter:
+            pass
+
+        completed = getattr(stream_iter, "completed_response", None)
+        response_obj = getattr(completed, "response", None) if completed else None
+        if response_obj is None:
+            raise ValueError("Stream ended without a completed response")
+
+        hidden_params = getattr(stream_iter, "_hidden_params", None)
+        response = self._coerce_response_object(response_obj, hidden_params)
+        if not isinstance(response, ResponsesAPIResponse):
+            raise ValueError("Stream completed response is invalid")
+        return response
+
     def validate_input_kwargs(
         self, kwargs: dict
     ) -> ResponsesToCompletionBridgeHandlerInputKwargs:
@@ -87,7 +154,6 @@ class ResponsesToCompletionBridgeHandler:
 
         from litellm import responses
         from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper
-        from litellm.types.llms.openai import ResponsesAPIResponse
 
         validated_kwargs = self.validate_input_kwargs(kwargs)
         model = validated_kwargs["model"]
@@ -113,6 +179,7 @@ class ResponsesToCompletionBridgeHandler:
             **request_data,
         )
 
+        stream = self._resolve_stream_flag(optional_params, litellm_params)
         if isinstance(result, ResponsesAPIResponse):
             return self.transformation_handler.transform_response(
                 model=model,
@@ -127,6 +194,21 @@ class ResponsesToCompletionBridgeHandler:
                 api_key=kwargs.get("api_key"),
                 json_mode=kwargs.get("json_mode"),
             )
+        elif not stream:
+            responses_api_response = self._collect_response_from_stream(result)
+            return self.transformation_handler.transform_response(
+                model=model,
+                raw_response=responses_api_response,
+                model_response=model_response,
+                logging_obj=logging_obj,
+                request_data=request_data,
+                messages=messages,
+                optional_params=optional_params,
+                litellm_params=litellm_params,
+                encoding=kwargs.get("encoding"),
+                api_key=kwargs.get("api_key"),
+                json_mode=kwargs.get("json_mode"),
+            )
         else:
             completion_stream = self.transformation_handler.get_model_response_iterator(
                 streaming_response=result,  # type: ignore
@@ -146,7 +228,6 @@ class ResponsesToCompletionBridgeHandler:
     ) -> Union["ModelResponse", "CustomStreamWrapper"]:
         from litellm import aresponses
         from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper
-        from litellm.types.llms.openai import ResponsesAPIResponse
 
         validated_kwargs = self.validate_input_kwargs(kwargs)
         model = validated_kwargs["model"]
@@ -175,6 +256,7 @@ class ResponsesToCompletionBridgeHandler:
             aresponses=True,
         )
 
+        stream = self._resolve_stream_flag(optional_params, litellm_params)
         if isinstance(result, ResponsesAPIResponse):
             return self.transformation_handler.transform_response(
                 model=model,
@@ -189,6 +271,23 @@ class ResponsesToCompletionBridgeHandler:
                 api_key=kwargs.get("api_key"),
                 json_mode=kwargs.get("json_mode"),
             )
+        elif not stream:
+            responses_api_response = await self._collect_response_from_stream_async(
+                result
+            )
+            return self.transformation_handler.transform_response(
+                model=model,
+                raw_response=responses_api_response,
+                model_response=model_response,
+                logging_obj=logging_obj,
+                request_data=request_data,
+                messages=messages,
+                optional_params=optional_params,
+                litellm_params=litellm_params,
+                encoding=kwargs.get("encoding"),
+                api_key=kwargs.get("api_key"),
+                json_mode=kwargs.get("json_mode"),
+            )
         else:
             completion_stream = self.transformation_handler.get_model_response_iterator(
                 streaming_response=result,  # type: ignore

litellm/completion_extras/litellm_responses_transformation/transformation.py

@@ -779,10 +779,10 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
     @staticmethod
     def _convert_annotations_to_chat_format(
         annotations: Optional[List[Any]],
-    ) -> Optional[List["ChatCompletionAnnotation"]]:
+    ) -> Optional[List[ChatCompletionAnnotation]]:
         """
         Convert annotations from Responses API to Chat Completions format.
-        
+
         Annotations are already in compatible format between both APIs,
         so we just need to convert Pydantic models to dicts.
         """

litellm/constants.py

@@ -323,6 +323,9 @@ EMAIL_BUDGET_ALERT_TTL = int(os.getenv("EMAIL_BUDGET_ALERT_TTL", 24 * 60 * 60))
 EMAIL_BUDGET_ALERT_MAX_SPEND_ALERT_PERCENTAGE = float(os.getenv("EMAIL_BUDGET_ALERT_MAX_SPEND_ALERT_PERCENTAGE", 0.8))  # 80% of max budget
 ############### LLM Provider Constants ###############
 ### ANTHROPIC CONSTANTS ###
+ANTHROPIC_TOKEN_COUNTING_BETA_VERSION = os.getenv(
+    "ANTHROPIC_TOKEN_COUNTING_BETA_VERSION", "token-counting-2024-11-01"
+)
 ANTHROPIC_SKILLS_API_BETA_VERSION = "skills-2025-10-02"
 ANTHROPIC_WEB_SEARCH_TOOL_MAX_USES = {
     "low": 1,
@@ -415,6 +418,7 @@ LITELLM_CHAT_PROVIDERS = [
     "galadriel",
     "gradient_ai",
     "github_copilot",  # GitHub Copilot Chat API
+    "chatgpt",  # ChatGPT subscription API
     "novita",
     "meta_llama",
     "featherless_ai",
@@ -617,6 +621,7 @@ openai_compatible_providers: List = [
     "lm_studio",
     "galadriel",
     "github_copilot",  # GitHub Copilot Chat API
+    "chatgpt",  # ChatGPT subscription API
     "novita",
     "meta_llama",
     "publicai",  # PublicAI - JSON-configured provider

litellm/integrations/langfuse/langfuse_otel.py

@@ -156,7 +156,11 @@ class LangfuseOtelLogger(OpenTelemetry):
                         "arguments": arguments_obj,
                     }
                     transformed_tool_calls.append(langfuse_tool_call)
-                safe_set_attribute(span, LangfuseSpanAttributes.OBSERVATION_OUTPUT.value, safe_dumps(transformed_tool_calls))
+                safe_set_attribute(
+                    span,
+                    LangfuseSpanAttributes.OBSERVATION_OUTPUT.value,
+                    safe_dumps(transformed_tool_calls),
+                )
             else:
                 output_data = {}
                 if message.get("role"):
@@ -164,7 +168,11 @@ class LangfuseOtelLogger(OpenTelemetry):
                 if message.get("content") is not None:
                     output_data["content"] = message.get("content")
                 if output_data:
-                    safe_set_attribute(span, LangfuseSpanAttributes.OBSERVATION_OUTPUT.value, safe_dumps(output_data))
+                    safe_set_attribute(
+                        span,
+                        LangfuseSpanAttributes.OBSERVATION_OUTPUT.value,
+                        safe_dumps(output_data),
+                    )
 
         output = response_obj.get("output", [])
         if output:
@@ -175,15 +183,28 @@ class LangfuseOtelLogger(OpenTelemetry):
                     if item_type == "reasoning" and hasattr(item, "summary"):
                         for summary in item.summary:
                             if hasattr(summary, "text"):
-                                output_items_data.append({"role": "reasoning_summary", "content": summary.text})
+                                output_items_data.append(
+                                    {
+                                        "role": "reasoning_summary",
+                                        "content": summary.text,
+                                    }
+                                )
                     elif item_type == "message":
-                        output_items_data.append({
-                            "role": getattr(item, "role", "assistant"),
-                            "content": getattr(getattr(item, "content", [{}])[0], "text", "")
-                        })
+                        output_items_data.append(
+                            {
+                                "role": getattr(item, "role", "assistant"),
+                                "content": getattr(
+                                    getattr(item, "content", [{}])[0], "text", ""
+                                ),
+                            }
+                        )
                     elif item_type == "function_call":
                         arguments_str = getattr(item, "arguments", "{}")
-                        arguments_obj = json.loads(arguments_str) if isinstance(arguments_str, str) else arguments_str
+                        arguments_obj = (
+                            json.loads(arguments_str)
+                            if isinstance(arguments_str, str)
+                            else arguments_str
+                        )
                         langfuse_tool_call = {
                             "id": getattr(item, "id", ""),
                             "name": getattr(item, "name", ""),
@@ -193,7 +214,11 @@ class LangfuseOtelLogger(OpenTelemetry):
                         }
                         output_items_data.append(langfuse_tool_call)
             if output_items_data:
-                safe_set_attribute(span, LangfuseSpanAttributes.OBSERVATION_OUTPUT.value, safe_dumps(output_items_data))
+                safe_set_attribute(
+                    span,
+                    LangfuseSpanAttributes.OBSERVATION_OUTPUT.value,
+                    safe_dumps(output_items_data),
+                )
 
     @staticmethod
     def _set_langfuse_specific_attributes(span: Span, kwargs, response_obj):
@@ -210,14 +235,22 @@ class LangfuseOtelLogger(OpenTelemetry):
 
         langfuse_environment = os.environ.get("LANGFUSE_TRACING_ENVIRONMENT")
         if langfuse_environment:
-            safe_set_attribute(span, LangfuseSpanAttributes.LANGFUSE_ENVIRONMENT.value, langfuse_environment)
+            safe_set_attribute(
+                span,
+                LangfuseSpanAttributes.LANGFUSE_ENVIRONMENT.value,
+                langfuse_environment,
+            )
 
         metadata = LangfuseOtelLogger._extract_langfuse_metadata(kwargs)
         LangfuseOtelLogger._set_metadata_attributes(span=span, metadata=metadata)
 
         messages = kwargs.get("messages")
         if messages:
-            safe_set_attribute(span, LangfuseSpanAttributes.OBSERVATION_INPUT.value, safe_dumps(messages))
+            safe_set_attribute(
+                span,
+                LangfuseSpanAttributes.OBSERVATION_INPUT.value,
+                safe_dumps(messages),
+            )
 
         LangfuseOtelLogger._set_observation_output(span=span, response_obj=response_obj)
 
@@ -319,3 +352,15 @@ class LangfuseOtelLogger(OpenTelemetry):
             dynamic_headers["Authorization"] = auth_header
 
         return dynamic_headers
+
+    async def async_service_success_hook(self, *args, **kwargs):
+        """
+        Langfuse should not receive service success logs.
+        """
+        pass
+
+    async def async_service_failure_hook(self, *args, **kwargs):
+        """
+        Langfuse should not receive service failure logs.
+        """
+        pass

litellm/litellm_core_utils/get_llm_provider_logic.py

@@ -768,6 +768,14 @@ def _get_openai_compatible_provider_info(  # noqa: PLR0915
         ) = litellm.GithubCopilotConfig()._get_openai_compatible_provider_info(
             model, api_base, api_key, custom_llm_provider
         )
+    elif custom_llm_provider == "chatgpt":
+        (
+            api_base,
+            dynamic_api_key,
+            custom_llm_provider,
+        ) = litellm.ChatGPTConfig()._get_openai_compatible_provider_info(
+            model, api_base, api_key, custom_llm_provider
+        )
     elif custom_llm_provider == "novita":
         api_base = (
             api_base

litellm/litellm_core_utils/litellm_logging.py

@@ -1897,6 +1897,14 @@ class Logging(LiteLLMLoggingBaseClass):
                     status="success",
                     standard_built_in_tools_params=self.standard_built_in_tools_params,
                 )
+                if (
+                    standard_logging_payload := self.model_call_details.get(
+                        "standard_logging_object"
+                    )
+                ) is not None:
+                    # Only emit for sync requests (async_success_handler handles async)
+                    if is_sync_request:
+                        emit_standard_logging_payload(standard_logging_payload)
             callbacks = self.get_combined_callback_list(
                 dynamic_success_callbacks=self.dynamic_success_callbacks,
                 global_callbacks=litellm.success_callback,
@@ -2190,10 +2198,7 @@ class Logging(LiteLLMLoggingBaseClass):
                                 print_verbose=print_verbose,
                             )
 
-                    if (
-                        callback == "openmeter"
-                        and is_sync_request
-                    ):
+                    if callback == "openmeter" and is_sync_request:
                         global openMeterLogger
                         if openMeterLogger is None:
                             print_verbose("Instantiates openmeter client")
@@ -2405,6 +2410,14 @@ class Logging(LiteLLMLoggingBaseClass):
                 status="success",
                 standard_built_in_tools_params=self.standard_built_in_tools_params,
             )
+
+            # print standard logging payload
+            if (
+                standard_logging_payload := self.model_call_details.get(
+                    "standard_logging_object"
+                )
+            ) is not None:
+                emit_standard_logging_payload(standard_logging_payload)
         callbacks = self.get_combined_callback_list(
             dynamic_success_callbacks=self.dynamic_async_success_callbacks,
             global_callbacks=litellm._async_success_callback,
@@ -2799,8 +2812,7 @@ class Logging(LiteLLMLoggingBaseClass):
                             callback_func=callback,
                         )
                     if (
-                        isinstance(callback, CustomLogger)
-                        and is_sync_request
+                        isinstance(callback, CustomLogger) and is_sync_request
                     ):  # custom logger class
                         callback.log_failure_event(
                             start_time=start_time,
@@ -3743,10 +3755,13 @@ def _init_custom_logger_compatible_class(  # noqa: PLR0915
                 OpenTelemetry,
                 OpenTelemetryConfig,
             )
-            logfire_base_url = os.getenv("LOGFIRE_BASE_URL", "https://logfire-api.pydantic.dev")
+
+            logfire_base_url = os.getenv(
+                "LOGFIRE_BASE_URL", "https://logfire-api.pydantic.dev"
+            )
             otel_config = OpenTelemetryConfig(
                 exporter="otlp_http",
-                endpoint = f"{logfire_base_url.rstrip('/')}/v1/traces",
+                endpoint=f"{logfire_base_url.rstrip('/')}/v1/traces",
                 headers=f"Authorization={os.getenv('LOGFIRE_TOKEN')}",
             )
             for callback in _in_memory_loggers:
@@ -4342,32 +4357,38 @@ class StandardLoggingPayloadSetup:
     def merge_litellm_metadata(litellm_params: dict) -> dict:
         """
         Merge both litellm_metadata and metadata from litellm_params.
-        
+
         litellm_metadata contains model-related fields, metadata contains user API key fields.
         We need both for complete standard logging payload.
-        
+
         Args:
             litellm_params: Dictionary containing metadata and litellm_metadata
-            
+
         Returns:
             dict: Merged metadata with user API key fields taking precedence
         """
         merged_metadata: dict = {}
-        
+
         # Start with metadata (user API key fields) - but skip non-serializable objects
-        if litellm_params.get("metadata") and isinstance(litellm_params.get("metadata"), dict):
+        if litellm_params.get("metadata") and isinstance(
+            litellm_params.get("metadata"), dict
+        ):
             for key, value in litellm_params["metadata"].items():
                 # Skip non-serializable objects like UserAPIKeyAuth
                 if key == "user_api_key_auth":
                     continue
                 merged_metadata[key] = value
-        
+
         # Then merge litellm_metadata (model-related fields) - this will NOT overwrite existing keys
-        if litellm_params.get("litellm_metadata") and isinstance(litellm_params.get("litellm_metadata"), dict):
+        if litellm_params.get("litellm_metadata") and isinstance(
+            litellm_params.get("litellm_metadata"), dict
+        ):
             for key, value in litellm_params["litellm_metadata"].items():
-                if key not in merged_metadata:  # Don't overwrite existing keys from metadata
+                if (
+                    key not in merged_metadata
+                ):  # Don't overwrite existing keys from metadata
                     merged_metadata[key] = value
-        
+
         return merged_metadata
 
     @staticmethod
@@ -4810,7 +4831,9 @@ class StandardLoggingPayloadSetup:
         """
         Extract additional header tags for spend tracking based on config.
         """
-        extra_headers: List[str] = getattr(litellm, "extra_spend_tag_headers", None) or []
+        extra_headers: List[str] = (
+            getattr(litellm, "extra_spend_tag_headers", None) or []
+        )
         if not extra_headers:
             return None
 
@@ -4959,7 +4982,9 @@ def get_standard_logging_object_payload(
         proxy_server_request = litellm_params.get("proxy_server_request") or {}
 
         # Merge both litellm_metadata and metadata to get complete metadata
-        metadata: dict = StandardLoggingPayloadSetup.merge_litellm_metadata(litellm_params)
+        metadata: dict = StandardLoggingPayloadSetup.merge_litellm_metadata(
+            litellm_params
+        )
 
         completion_start_time = kwargs.get("completion_start_time", end_time)
         call_type = kwargs.get("call_type")
@@ -5129,7 +5154,8 @@ def get_standard_logging_object_payload(
             standard_built_in_tools_params=standard_built_in_tools_params,
         )
 
-        emit_standard_logging_payload(payload)
+        # emit_standard_logging_payload(payload) - Moved to success_handler to prevent double emitting
+
         return payload
     except Exception as e:
         verbose_logger.exception(

litellm/litellm_core_utils/llm_cost_calc/utils.py

@@ -354,7 +354,7 @@ class PromptTokensDetailsResult(TypedDict):
     image_tokens: int
     character_count: int
     image_count: int
-    video_length_seconds: int
+    video_length_seconds: float
 
 
 def _parse_prompt_tokens_details(usage: Usage) -> PromptTokensDetailsResult:
@@ -400,10 +400,10 @@ def _parse_prompt_tokens_details(usage: Usage) -> PromptTokensDetailsResult:
     )
     video_length_seconds = (
         cast(
-            Optional[int],
+            Optional[float],
             getattr(usage.prompt_tokens_details, "video_length_seconds", 0),
         )
-        or 0
+        or 0.0
     )
 
     return PromptTokensDetailsResult(
@@ -415,7 +415,7 @@ def _parse_prompt_tokens_details(usage: Usage) -> PromptTokensDetailsResult:
         image_tokens=image_tokens,
         character_count=character_count,
         image_count=image_count,
-        video_length_seconds=video_length_seconds,
+        video_length_seconds=float(video_length_seconds),
     )
 
 
@@ -561,7 +561,7 @@ def generic_cost_per_token(  # noqa: PLR0915
         image_tokens=0,
         character_count=0,
         image_count=0,
-        video_length_seconds=0,
+        video_length_seconds=0.0,
     )
     if usage.prompt_tokens_details:
         prompt_tokens_details = _parse_prompt_tokens_details(usage)

litellm/litellm_core_utils/prompt_templates/factory.py

@@ -6,7 +6,7 @@ import mimetypes
 import re
 import xml.etree.ElementTree as ET
 from enum import Enum
-from typing import Any, Dict, List, Optional, Tuple, Union, cast, overload
+from typing import Any, Dict, List, Optional, Set, Tuple, Union, cast, overload
 
 from jinja2.sandbox import ImmutableSandboxedEnvironment
 
@@ -2143,6 +2143,9 @@ def anthropic_messages_pt(  # noqa: PLR0915
         if user_content:
             new_messages.append({"role": "user", "content": user_content})
 
+        # Track unique tool IDs in this merge block to avoid duplication
+        unique_tool_ids: Set[str] = set()
+
         assistant_content: List[AnthropicMessagesAssistantMessageValues] = []
         ## MERGE CONSECUTIVE ASSISTANT CONTENT ##
         while msg_i < len(messages) and messages[msg_i]["role"] == "assistant":
@@ -2236,13 +2239,25 @@ def anthropic_messages_pt(  # noqa: PLR0915
                     assistant_tool_calls,
                     web_search_results=_web_search_results,
                 )
-                # AnthropicMessagesAssistantMessageValues includes AnthropicMessagesToolUseParam
-                assistant_content.extend(
-                    cast(
-                        List[AnthropicMessagesAssistantMessageValues],
-                        tool_invoke_results,
+
+                # Prevent "tool_use ids must be unique" errors by filtering duplicates
+                # This can happen when merging history that already contains the tool calls
+                for item in tool_invoke_results:
+                    # tool_use items are typically dicts, but handle objects just in case
+                    item_id = (
+                        item.get("id")
+                        if isinstance(item, dict)
+                        else getattr(item, "id", None)
+                    )
+
+                    if item_id:
+                        if item_id in unique_tool_ids:
+                            continue
+                        unique_tool_ids.add(item_id)
+
+                    assistant_content.append(
+                        cast(AnthropicMessagesAssistantMessageValues, item)
                     )
-                )
 
             assistant_function_call = assistant_content_block.get("function_call")
 

litellm/llms/anthropic/chat/handler.py

@@ -317,6 +317,7 @@ class AnthropicChatCompletion(BaseLLM):
         stream = optional_params.pop("stream", None)
         json_mode: bool = optional_params.pop("json_mode", False)
         is_vertex_request: bool = optional_params.pop("is_vertex_request", False)
+        optional_params.pop("vertex_count_tokens_location", None)
         _is_function_call = False
         messages = copy.deepcopy(messages)
         headers = AnthropicConfig().validate_environment(

litellm/llms/anthropic/common_utils.py

@@ -2,7 +2,7 @@
 This file contains common utils for anthropic calls.
 """
 
-from typing import Any, Dict, List, Optional, Union
+from typing import Dict, List, Optional, Union
 
 import httpx
 
@@ -14,11 +14,36 @@ from litellm.llms.base_llm.base_utils import BaseLLMModelInfo, BaseTokenCounter
 from litellm.llms.base_llm.chat.transformation import BaseLLMException
 from litellm.types.llms.anthropic import (
     ANTHROPIC_HOSTED_TOOLS,
+    ANTHROPIC_OAUTH_BETA_HEADER,
+    ANTHROPIC_OAUTH_TOKEN_PREFIX,
     AllAnthropicToolsValues,
     AnthropicMcpServerTool,
 )
 from litellm.types.llms.openai import AllMessageValues
-from litellm.types.utils import TokenCountResponse
+
+
+def optionally_handle_anthropic_oauth(
+    headers: dict, api_key: Optional[str]
+) -> tuple[dict, Optional[str]]:
+    """
+    Handle Anthropic OAuth token detection and header setup.
+
+    If an OAuth token is detected in the Authorization header, extracts it
+    and sets the required OAuth headers.
+
+    Args:
+        headers: Request headers dict
+        api_key: Current API key (may be None)
+
+    Returns:
+        Tuple of (updated headers, api_key)
+    """
+    auth_header = headers.get("authorization", "")
+    if auth_header and auth_header.startswith(f"Bearer {ANTHROPIC_OAUTH_TOKEN_PREFIX}"):
+        api_key = auth_header.replace("Bearer ", "")
+        headers["anthropic-beta"] = ANTHROPIC_OAUTH_BETA_HEADER
+        headers["anthropic-dangerous-direct-browser-access"] = "true"
+    return headers, api_key
 
 
 class AnthropicError(BaseLLMException):
@@ -372,6 +397,8 @@ class AnthropicModelInfo(BaseLLMModelInfo):
         api_key: Optional[str] = None,
         api_base: Optional[str] = None,
     ) -> Dict:
+        # Check for Anthropic OAuth token in headers
+        headers, api_key = optionally_handle_anthropic_oauth(headers=headers, api_key=api_key)
         if api_key is None:
             raise litellm.AuthenticationError(
                 message="Missing Anthropic API Key - A call is being made to anthropic but no key is set either in the environment variables or via params. Please set `ANTHROPIC_API_KEY` in your environment vars",
@@ -476,45 +503,11 @@ class AnthropicModelInfo(BaseLLMModelInfo):
         Returns:
             AnthropicTokenCounter instance for this provider.
         """
-        return AnthropicTokenCounter()
-
-
-class AnthropicTokenCounter(BaseTokenCounter):
-    """Token counter implementation for Anthropic provider."""
-
-    def should_use_token_counting_api(
-        self, 
-        custom_llm_provider: Optional[str] = None,
-    ) -> bool:
-        from litellm.types.utils import LlmProviders
-        return custom_llm_provider == LlmProviders.ANTHROPIC.value
-    
-    async def count_tokens(
-        self,
-        model_to_use: str,
-        messages: Optional[List[Dict[str, Any]]],
-        contents: Optional[List[Dict[str, Any]]],
-        deployment: Optional[Dict[str, Any]] = None,
-        request_model: str = "",
-    ) -> Optional[TokenCountResponse]:
-        from litellm.proxy.utils import count_tokens_with_anthropic_api
-        
-        result = await count_tokens_with_anthropic_api(
-            model_to_use=model_to_use,
-            messages=messages,
-            deployment=deployment,
+        from litellm.llms.anthropic.count_tokens.token_counter import (
+            AnthropicTokenCounter,
         )
-        
-        if result is not None:
-            return TokenCountResponse(
-                total_tokens=result.get("total_tokens", 0),
-                request_model=request_model,
-                model_used=model_to_use,
-                tokenizer_type=result.get("tokenizer_used", ""),
-                original_response=result,
-            )
-        
-        return None
+
+        return AnthropicTokenCounter()
 
 
 def process_anthropic_headers(headers: Union[httpx.Headers, dict]) -> dict:

litellm/llms/anthropic/count_tokens/__init__.py

@@ -0,0 +1,15 @@
+"""
+Anthropic CountTokens API implementation.
+"""
+
+from litellm.llms.anthropic.count_tokens.handler import AnthropicCountTokensHandler
+from litellm.llms.anthropic.count_tokens.token_counter import AnthropicTokenCounter
+from litellm.llms.anthropic.count_tokens.transformation import (
+    AnthropicCountTokensConfig,
+)
+
+__all__ = [
+    "AnthropicCountTokensHandler",
+    "AnthropicCountTokensConfig",
+    "AnthropicTokenCounter",
+]

litellm/llms/anthropic/count_tokens/handler.py

@@ -0,0 +1,122 @@
+"""
+Anthropic CountTokens API handler.
+
+Uses httpx for HTTP requests instead of the Anthropic SDK.
+"""
+
+from typing import Any, Dict, List, Optional, Union
+
+import httpx
+
+import litellm
+from litellm._logging import verbose_logger
+from litellm.llms.anthropic.common_utils import AnthropicError
+from litellm.llms.anthropic.count_tokens.transformation import (
+    AnthropicCountTokensConfig,
+)
+from litellm.llms.custom_httpx.http_handler import get_async_httpx_client
+
+
+class AnthropicCountTokensHandler(AnthropicCountTokensConfig):
+    """
+    Handler for Anthropic CountTokens API requests.
+
+    Uses httpx for HTTP requests, following the same pattern as BedrockCountTokensHandler.
+    """
+
+    async def handle_count_tokens_request(
+        self,
+        model: str,
+        messages: List[Dict[str, Any]],
+        api_key: str,
+        api_base: Optional[str] = None,
+        timeout: Optional[Union[float, httpx.Timeout]] = None,
+    ) -> Dict[str, Any]:
+        """
+        Handle a CountTokens request using httpx.
+
+        Args:
+            model: The model identifier (e.g., "claude-3-5-sonnet-20241022")
+            messages: The messages to count tokens for
+            api_key: The Anthropic API key
+            api_base: Optional custom API base URL
+            timeout: Optional timeout for the request (defaults to litellm.request_timeout)
+
+        Returns:
+            Dictionary containing token count response
+
+        Raises:
+            AnthropicError: If the API request fails
+        """
+        try:
+            # Validate the request
+            self.validate_request(model, messages)
+
+            verbose_logger.debug(
+                f"Processing Anthropic CountTokens request for model: {model}"
+            )
+
+            # Transform request to Anthropic format
+            request_body = self.transform_request_to_count_tokens(
+                model=model,
+                messages=messages,
+            )
+
+            verbose_logger.debug(f"Transformed request: {request_body}")
+
+            # Get endpoint URL
+            endpoint_url = api_base or self.get_anthropic_count_tokens_endpoint()
+
+            verbose_logger.debug(f"Making request to: {endpoint_url}")
+
+            # Get required headers
+            headers = self.get_required_headers(api_key)
+
+            # Use LiteLLM's async httpx client
+            async_client = get_async_httpx_client(
+                llm_provider=litellm.LlmProviders.ANTHROPIC
+            )
+
+            # Use provided timeout or fall back to litellm.request_timeout
+            request_timeout = timeout if timeout is not None else litellm.request_timeout
+
+            response = await async_client.post(
+                endpoint_url,
+                headers=headers,
+                json=request_body,
+                timeout=request_timeout,
+            )
+
+            verbose_logger.debug(f"Response status: {response.status_code}")
+
+            if response.status_code != 200:
+                error_text = response.text
+                verbose_logger.error(f"Anthropic API error: {error_text}")
+                raise AnthropicError(
+                    status_code=response.status_code,
+                    message=error_text,
+                )
+
+            anthropic_response = response.json()
+
+            verbose_logger.debug(f"Anthropic response: {anthropic_response}")
+
+            # Return Anthropic response directly - no transformation needed
+            return anthropic_response
+
+        except AnthropicError:
+            # Re-raise Anthropic exceptions as-is
+            raise
+        except httpx.HTTPStatusError as e:
+            # HTTP errors - preserve the actual status code
+            verbose_logger.error(f"HTTP error in CountTokens handler: {str(e)}")
+            raise AnthropicError(
+                status_code=e.response.status_code,
+                message=e.response.text,
+            )
+        except Exception as e:
+            verbose_logger.error(f"Error in CountTokens handler: {str(e)}")
+            raise AnthropicError(
+                status_code=500,
+                message=f"CountTokens processing error: {str(e)}",
+            )

litellm/llms/anthropic/count_tokens/token_counter.py

@@ -0,0 +1,104 @@
+"""
+Anthropic Token Counter implementation using the CountTokens API.
+"""
+
+import os
+from typing import Any, Dict, List, Optional
+
+from litellm._logging import verbose_logger
+from litellm.llms.anthropic.count_tokens.handler import AnthropicCountTokensHandler
+from litellm.llms.base_llm.base_utils import BaseTokenCounter
+from litellm.types.utils import LlmProviders, TokenCountResponse
+
+# Global handler instance - reuse across all token counting requests
+anthropic_count_tokens_handler = AnthropicCountTokensHandler()
+
+
+class AnthropicTokenCounter(BaseTokenCounter):
+    """Token counter implementation for Anthropic provider using the CountTokens API."""
+
+    def should_use_token_counting_api(
+        self,
+        custom_llm_provider: Optional[str] = None,
+    ) -> bool:
+        return custom_llm_provider == LlmProviders.ANTHROPIC.value
+
+    async def count_tokens(
+        self,
+        model_to_use: str,
+        messages: Optional[List[Dict[str, Any]]],
+        contents: Optional[List[Dict[str, Any]]],
+        deployment: Optional[Dict[str, Any]] = None,
+        request_model: str = "",
+    ) -> Optional[TokenCountResponse]:
+        """
+        Count tokens using Anthropic's CountTokens API.
+
+        Args:
+            model_to_use: The model identifier
+            messages: The messages to count tokens for
+            contents: Alternative content format (not used for Anthropic)
+            deployment: Deployment configuration containing litellm_params
+            request_model: The original request model name
+
+        Returns:
+            TokenCountResponse with token count, or None if counting fails
+        """
+        from litellm.llms.anthropic.common_utils import AnthropicError
+
+        if not messages:
+            return None
+
+        deployment = deployment or {}
+        litellm_params = deployment.get("litellm_params", {})
+
+        # Get Anthropic API key from deployment config or environment
+        api_key = litellm_params.get("api_key")
+        if not api_key:
+            api_key = os.getenv("ANTHROPIC_API_KEY")
+
+        if not api_key:
+            verbose_logger.warning("No Anthropic API key found for token counting")
+            return None
+
+        try:
+            result = await anthropic_count_tokens_handler.handle_count_tokens_request(
+                model=model_to_use,
+                messages=messages,
+                api_key=api_key,
+            )
+
+            if result is not None:
+                return TokenCountResponse(
+                    total_tokens=result.get("input_tokens", 0),
+                    request_model=request_model,
+                    model_used=model_to_use,
+                    tokenizer_type="anthropic_api",
+                    original_response=result,
+                )
+        except AnthropicError as e:
+            verbose_logger.warning(
+                f"Anthropic CountTokens API error: status={e.status_code}, message={e.message}"
+            )
+            return TokenCountResponse(
+                total_tokens=0,
+                request_model=request_model,
+                model_used=model_to_use,
+                tokenizer_type="anthropic_api",
+                error=True,
+                error_message=e.message,
+                status_code=e.status_code,
+            )
+        except Exception as e:
+            verbose_logger.warning(f"Error calling Anthropic CountTokens API: {e}")
+            return TokenCountResponse(
+                total_tokens=0,
+                request_model=request_model,
+                model_used=model_to_use,
+                tokenizer_type="anthropic_api",
+                error=True,
+                error_message=str(e),
+                status_code=500,
+            )
+
+        return None

litellm/llms/anthropic/count_tokens/transformation.py

@@ -0,0 +1,103 @@
+"""
+Anthropic CountTokens API transformation logic.
+
+This module handles the transformation of requests to Anthropic's CountTokens API format.
+"""
+
+from typing import Any, Dict, List
+
+from litellm.constants import ANTHROPIC_TOKEN_COUNTING_BETA_VERSION
+
+
+class AnthropicCountTokensConfig:
+    """
+    Configuration and transformation logic for Anthropic CountTokens API.
+
+    Anthropic CountTokens API Specification:
+    - Endpoint: POST https://api.anthropic.com/v1/messages/count_tokens
+    - Beta header required: anthropic-beta: token-counting-2024-11-01
+    - Response: {"input_tokens": <number>}
+    """
+
+    def get_anthropic_count_tokens_endpoint(self) -> str:
+        """
+        Get the Anthropic CountTokens API endpoint.
+
+        Returns:
+            The endpoint URL for the CountTokens API
+        """
+        return "https://api.anthropic.com/v1/messages/count_tokens"
+
+    def transform_request_to_count_tokens(
+        self,
+        model: str,
+        messages: List[Dict[str, Any]],
+    ) -> Dict[str, Any]:
+        """
+        Transform request to Anthropic CountTokens format.
+
+        Input:
+        {
+            "model": "claude-3-5-sonnet-20241022",
+            "messages": [{"role": "user", "content": "Hello!"}]
+        }
+
+        Output (Anthropic CountTokens format):
+        {
+            "model": "claude-3-5-sonnet-20241022",
+            "messages": [{"role": "user", "content": "Hello!"}]
+        }
+        """
+        return {
+            "model": model,
+            "messages": messages,
+        }
+
+    def get_required_headers(self, api_key: str) -> Dict[str, str]:
+        """
+        Get the required headers for the CountTokens API.
+
+        Args:
+            api_key: The Anthropic API key
+
+        Returns:
+            Dictionary of required headers
+        """
+        return {
+            "Content-Type": "application/json",
+            "x-api-key": api_key,
+            "anthropic-version": "2023-06-01",
+            "anthropic-beta": ANTHROPIC_TOKEN_COUNTING_BETA_VERSION,
+        }
+
+    def validate_request(
+        self, model: str, messages: List[Dict[str, Any]]
+    ) -> None:
+        """
+        Validate the incoming count tokens request.
+
+        Args:
+            model: The model name
+            messages: The messages to count tokens for
+
+        Raises:
+            ValueError: If the request is invalid
+        """
+        if not model:
+            raise ValueError("model parameter is required")
+
+        if not messages:
+            raise ValueError("messages parameter is required")
+
+        if not isinstance(messages, list):
+            raise ValueError("messages must be a list")
+
+        for i, message in enumerate(messages):
+            if not isinstance(message, dict):
+                raise ValueError(f"Message {i} must be a dictionary")
+
+            if "role" not in message:
+                raise ValueError(f"Message {i} must have a 'role' field")
+
+            if "content" not in message:
+                raise ValueError(f"Message {i} must have a 'content' field")

litellm/llms/anthropic/experimental_pass_through/messages/transformation.py

@@ -17,7 +17,11 @@ from litellm.types.llms.anthropic_messages.anthropic_response import (
 from litellm.types.llms.anthropic_tool_search import get_tool_search_beta_header
 from litellm.types.router import GenericLiteLLMParams
 
-from ...common_utils import AnthropicError, AnthropicModelInfo
+from ...common_utils import (
+    AnthropicError,
+    AnthropicModelInfo,
+    optionally_handle_anthropic_oauth,
+)
 
 DEFAULT_ANTHROPIC_API_BASE = "https://api.anthropic.com"
 DEFAULT_ANTHROPIC_API_VERSION = "2023-06-01"
@@ -68,8 +72,11 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
     ) -> Tuple[dict, Optional[str]]:
         import os
 
+        # Check for Anthropic OAuth token in Authorization header
+        headers, api_key = optionally_handle_anthropic_oauth(headers=headers, api_key=api_key)
         if api_key is None:
             api_key = os.getenv("ANTHROPIC_API_KEY")
+
         if "x-api-key" not in headers and api_key:
             headers["x-api-key"] = api_key
         if "anthropic-version" not in headers:

litellm/llms/azure/azure.py

@@ -4,7 +4,13 @@ import time
 from typing import Any, Callable, Coroutine, Dict, List, Optional, Union
 
 import httpx  # type: ignore
-from openai import APITimeoutError, AsyncAzureOpenAI, AzureOpenAI
+from openai import (
+    APITimeoutError,
+    AsyncAzureOpenAI,
+    AsyncOpenAI,
+    AzureOpenAI,
+    OpenAI,
+)
 
 import litellm
 from litellm.constants import AZURE_OPERATION_POLLING_TIMEOUT, DEFAULT_MAX_RETRIES
@@ -128,7 +134,7 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM):
 
     def make_sync_azure_openai_chat_completion_request(
         self,
-        azure_client: AzureOpenAI,
+        azure_client: Union[AzureOpenAI, OpenAI],
         data: dict,
         timeout: Union[float, httpx.Timeout],
     ):
@@ -151,7 +157,7 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM):
     @track_llm_api_timing()
     async def make_azure_openai_chat_completion_request(
         self,
-        azure_client: AsyncAzureOpenAI,
+        azure_client: Union[AsyncAzureOpenAI, AsyncOpenAI],
         data: dict,
         timeout: Union[float, httpx.Timeout],
         logging_obj: LiteLLMLoggingObj,
@@ -328,10 +334,10 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM):
                     _is_async=False,
                     litellm_params=litellm_params,
                 )
-                if not isinstance(azure_client, AzureOpenAI):
+                if not isinstance(azure_client, (AzureOpenAI, OpenAI)):
                     raise AzureOpenAIError(
                         status_code=500,
-                        message="azure_client is not an instance of AzureOpenAI",
+                        message="azure_client is not an instance of AzureOpenAI or OpenAI",
                     )
 
                 headers, response = self.make_sync_azure_openai_chat_completion_request(
@@ -401,8 +407,8 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM):
                 _is_async=True,
                 litellm_params=litellm_params,
             )
-            if not isinstance(azure_client, AsyncAzureOpenAI):
-                raise ValueError("Azure client is not an instance of AsyncAzureOpenAI")
+            if not isinstance(azure_client, (AsyncAzureOpenAI, AsyncOpenAI)):
+                raise ValueError("Azure client is not an instance of AsyncAzureOpenAI or AsyncOpenAI")
             ## LOGGING
             logging_obj.pre_call(
                 input=data["messages"],
@@ -412,7 +418,7 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM):
                         "api_key": api_key,
                         "azure_ad_token": azure_ad_token,
                     },
-                    "api_base": azure_client._base_url._uri_reference,
+                    "api_base": api_base,
                     "acompletion": True,
                     "complete_input_dict": data,
                 },
@@ -520,10 +526,10 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM):
             _is_async=False,
             litellm_params=litellm_params,
         )
-        if not isinstance(azure_client, AzureOpenAI):
+        if not isinstance(azure_client, (AzureOpenAI, OpenAI)):
             raise AzureOpenAIError(
                 status_code=500,
-                message="azure_client is not an instance of AzureOpenAI",
+                message="azure_client is not an instance of AzureOpenAI or OpenAI",
             )
         ## LOGGING
         logging_obj.pre_call(
@@ -534,7 +540,7 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM):
                     "api_key": api_key,
                     "azure_ad_token": azure_ad_token,
                 },
-                "api_base": azure_client._base_url._uri_reference,
+                "api_base": api_base,
                 "acompletion": True,
                 "complete_input_dict": data,
             },
@@ -578,8 +584,8 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM):
                 _is_async=True,
                 litellm_params=litellm_params,
             )
-            if not isinstance(azure_client, AsyncAzureOpenAI):
-                raise ValueError("Azure client is not an instance of AsyncAzureOpenAI")
+            if not isinstance(azure_client, (AsyncAzureOpenAI, AsyncOpenAI)):
+                raise ValueError("Azure client is not an instance of AsyncAzureOpenAI or AsyncOpenAI")
 
             ## LOGGING
             logging_obj.pre_call(
@@ -590,7 +596,7 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM):
                         "api_key": api_key,
                         "azure_ad_token": azure_ad_token,
                     },
-                    "api_base": azure_client._base_url._uri_reference,
+                    "api_base": api_base,
                     "acompletion": True,
                     "complete_input_dict": data,
                 },
@@ -657,8 +663,8 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM):
                 client=client,
                 litellm_params=litellm_params,
             )
-            if not isinstance(openai_aclient, AsyncAzureOpenAI):
-                raise ValueError("Azure client is not an instance of AsyncAzureOpenAI")
+            if not isinstance(openai_aclient, (AsyncAzureOpenAI, AsyncOpenAI)):
+                raise ValueError("Azure client is not an instance of AsyncAzureOpenAI or AsyncOpenAI")
 
             raw_response = await openai_aclient.embeddings.with_raw_response.create(
                 **data, timeout=timeout
@@ -776,10 +782,10 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM):
                 client=client,
                 litellm_params=litellm_params,
             )
-            if not isinstance(azure_client, AzureOpenAI):
+            if not isinstance(azure_client, (AzureOpenAI, OpenAI)):
                 raise AzureOpenAIError(
                     status_code=500,
-                    message="azure_client is not an instance of AzureOpenAI",
+                    message="azure_client is not an instance of AzureOpenAI or OpenAI",
                 )
 
             ## COMPLETION CALL

litellm/llms/azure/batches/handler.py

@@ -6,6 +6,8 @@ from typing import Any, Coroutine, Optional, Union, cast
 
 import httpx
 
+from openai import AsyncOpenAI, OpenAI
+
 from litellm.llms.azure.azure import AsyncAzureOpenAI, AzureOpenAI
 from litellm.types.llms.openai import (
     Batch,
@@ -33,7 +35,7 @@ class AzureBatchesAPI(BaseAzureLLM):
     async def acreate_batch(
         self,
         create_batch_data: CreateBatchRequest,
-        azure_client: AsyncAzureOpenAI,
+        azure_client: Union[AsyncAzureOpenAI, AsyncOpenAI],
     ) -> LiteLLMBatch:
         response = await azure_client.batches.create(**create_batch_data)
         return LiteLLMBatch(**response.model_dump())
@@ -47,11 +49,11 @@ class AzureBatchesAPI(BaseAzureLLM):
         api_version: Optional[str],
         timeout: Union[float, httpx.Timeout],
         max_retries: Optional[int],
-        client: Optional[Union[AzureOpenAI, AsyncAzureOpenAI]] = None,
+        client: Optional[Union[AzureOpenAI, AsyncAzureOpenAI, OpenAI, AsyncOpenAI]] = None,
         litellm_params: Optional[dict] = None,
     ) -> Union[LiteLLMBatch, Coroutine[Any, Any, LiteLLMBatch]]:
         azure_client: Optional[
-            Union[AzureOpenAI, AsyncAzureOpenAI]
+            Union[AzureOpenAI, AsyncAzureOpenAI, OpenAI, AsyncOpenAI]
         ] = self.get_azure_openai_client(
             api_key=api_key,
             api_base=api_base,
@@ -66,20 +68,20 @@ class AzureBatchesAPI(BaseAzureLLM):
             )
 
         if _is_async is True:
-            if not isinstance(azure_client, AsyncAzureOpenAI):
+            if not isinstance(azure_client, (AsyncAzureOpenAI, AsyncOpenAI)):
                 raise ValueError(
                     "OpenAI client is not an instance of AsyncOpenAI. Make sure you passed an AsyncOpenAI client."
                 )
             return self.acreate_batch(  # type: ignore
                 create_batch_data=create_batch_data, azure_client=azure_client
             )
-        response = cast(AzureOpenAI, azure_client).batches.create(**create_batch_data)
+        response = cast(Union[AzureOpenAI, OpenAI], azure_client).batches.create(**create_batch_data)
         return LiteLLMBatch(**response.model_dump())
 
     async def aretrieve_batch(
         self,
         retrieve_batch_data: RetrieveBatchRequest,
-        client: AsyncAzureOpenAI,
+        client: Union[AsyncAzureOpenAI, AsyncOpenAI],
     ) -> LiteLLMBatch:
         response = await client.batches.retrieve(**retrieve_batch_data)
         return LiteLLMBatch(**response.model_dump())
@@ -93,11 +95,11 @@ class AzureBatchesAPI(BaseAzureLLM):
         api_version: Optional[str],
         timeout: Union[float, httpx.Timeout],
         max_retries: Optional[int],
-        client: Optional[AzureOpenAI] = None,
+        client: Optional[Union[AzureOpenAI, AsyncAzureOpenAI, OpenAI, AsyncOpenAI]] = None,
         litellm_params: Optional[dict] = None,
     ):
         azure_client: Optional[
-            Union[AzureOpenAI, AsyncAzureOpenAI]
+            Union[AzureOpenAI, AsyncAzureOpenAI, OpenAI, AsyncOpenAI]
         ] = self.get_azure_openai_client(
             api_key=api_key,
             api_base=api_base,
@@ -112,14 +114,14 @@ class AzureBatchesAPI(BaseAzureLLM):
             )
 
         if _is_async is True:
-            if not isinstance(azure_client, AsyncAzureOpenAI):
+            if not isinstance(azure_client, (AsyncAzureOpenAI, AsyncOpenAI)):
                 raise ValueError(
                     "OpenAI client is not an instance of AsyncOpenAI. Make sure you passed an AsyncOpenAI client."
                 )
             return self.aretrieve_batch(  # type: ignore
                 retrieve_batch_data=retrieve_batch_data, client=azure_client
             )
-        response = cast(AzureOpenAI, azure_client).batches.retrieve(
+        response = cast(Union[AzureOpenAI, OpenAI], azure_client).batches.retrieve(
             **retrieve_batch_data
         )
         return LiteLLMBatch(**response.model_dump())
@@ -127,7 +129,7 @@ class AzureBatchesAPI(BaseAzureLLM):
     async def acancel_batch(
         self,
         cancel_batch_data: CancelBatchRequest,
-        client: AsyncAzureOpenAI,
+        client: Union[AsyncAzureOpenAI, AsyncOpenAI],
     ) -> Batch:
         response = await client.batches.cancel(**cancel_batch_data)
         return response
@@ -141,11 +143,11 @@ class AzureBatchesAPI(BaseAzureLLM):
         api_version: Optional[str],
         timeout: Union[float, httpx.Timeout],
         max_retries: Optional[int],
-        client: Optional[AzureOpenAI] = None,
+        client: Optional[Union[AzureOpenAI, AsyncAzureOpenAI, OpenAI, AsyncOpenAI]] = None,
         litellm_params: Optional[dict] = None,
     ):
         azure_client: Optional[
-            Union[AzureOpenAI, AsyncAzureOpenAI]
+            Union[AzureOpenAI, AsyncAzureOpenAI, OpenAI, AsyncOpenAI]
         ] = self.get_azure_openai_client(
             api_key=api_key,
             api_base=api_base,
@@ -163,7 +165,7 @@ class AzureBatchesAPI(BaseAzureLLM):
 
     async def alist_batches(
         self,
-        client: AsyncAzureOpenAI,
+        client: Union[AsyncAzureOpenAI, AsyncOpenAI],
         after: Optional[str] = None,
         limit: Optional[int] = None,
     ):
@@ -180,11 +182,11 @@ class AzureBatchesAPI(BaseAzureLLM):
         max_retries: Optional[int],
         after: Optional[str] = None,
         limit: Optional[int] = None,
-        client: Optional[AzureOpenAI] = None,
+        client: Optional[Union[AzureOpenAI, AsyncAzureOpenAI, OpenAI, AsyncOpenAI]] = None,
         litellm_params: Optional[dict] = None,
     ):
         azure_client: Optional[
-            Union[AzureOpenAI, AsyncAzureOpenAI]
+            Union[AzureOpenAI, AsyncAzureOpenAI, OpenAI, AsyncOpenAI]
         ] = self.get_azure_openai_client(
             api_key=api_key,
             api_base=api_base,
@@ -199,7 +201,7 @@ class AzureBatchesAPI(BaseAzureLLM):
             )
 
         if _is_async is True:
-            if not isinstance(azure_client, AsyncAzureOpenAI):
+            if not isinstance(azure_client, (AsyncAzureOpenAI, AsyncOpenAI)):
                 raise ValueError(
                     "OpenAI client is not an instance of AsyncOpenAI. Make sure you passed an AsyncOpenAI client."
                 )

litellm/llms/azure/common_utils.py

@@ -3,7 +3,7 @@ import os
 from typing import Any, Callable, Dict, Literal, Optional, Union, cast
 
 import httpx
-from openai import AsyncAzureOpenAI, AzureOpenAI
+from openai import AsyncAzureOpenAI, AsyncOpenAI, AzureOpenAI, OpenAI
 
 import litellm
 from litellm._logging import verbose_logger
@@ -439,12 +439,12 @@ class BaseAzureLLM(BaseOpenAILLM):
         api_key: Optional[str],
         api_base: Optional[str],
         api_version: Optional[str] = None,
-        client: Optional[Union[AzureOpenAI, AsyncAzureOpenAI]] = None,
+        client: Optional[Union[AzureOpenAI, AsyncAzureOpenAI, OpenAI, AsyncOpenAI]] = None,
         litellm_params: Optional[dict] = None,
         _is_async: bool = False,
         model: Optional[str] = None,
-    ) -> Optional[Union[AzureOpenAI, AsyncAzureOpenAI]]:
-        openai_client: Optional[Union[AzureOpenAI, AsyncAzureOpenAI]] = None
+    ) -> Optional[Union[AzureOpenAI, AsyncAzureOpenAI, OpenAI, AsyncOpenAI]]:
+        openai_client: Optional[Union[AzureOpenAI, AsyncAzureOpenAI, OpenAI, AsyncOpenAI]] = None
         client_initialization_params: dict = locals()
         client_initialization_params["is_async"] = _is_async
         if client is None:
@@ -453,9 +453,7 @@ class BaseAzureLLM(BaseOpenAILLM):
                 client_type="azure",
             )
             if cached_client:
-                if isinstance(cached_client, AzureOpenAI) or isinstance(
-                    cached_client, AsyncAzureOpenAI
-                ):
+                if isinstance(cached_client, (AzureOpenAI, AsyncAzureOpenAI, OpenAI, AsyncOpenAI)):
                     return cached_client
 
             azure_client_params = self.initialize_azure_sdk_client(
@@ -466,15 +464,40 @@ class BaseAzureLLM(BaseOpenAILLM):
                 api_version=api_version,
                 is_async=_is_async,
             )
-            if _is_async is True:
-                openai_client = AsyncAzureOpenAI(**azure_client_params)
+
+            # For Azure v1 API, use standard OpenAI client instead of AzureOpenAI
+            # See: https://learn.microsoft.com/en-us/azure/ai-services/openai/reference#api-specs
+            if self._is_azure_v1_api_version(api_version):
+                # Extract only params that OpenAI client accepts
+                # Always use /openai/v1/ regardless of whether user passed "v1", "latest", or "preview"
+                v1_params = {
+                    "api_key": azure_client_params.get("api_key"),
+                    "base_url": f"{api_base}/openai/v1/",
+                }
+                if "timeout" in azure_client_params:
+                    v1_params["timeout"] = azure_client_params["timeout"]
+                if "max_retries" in azure_client_params:
+                    v1_params["max_retries"] = azure_client_params["max_retries"]
+                if "http_client" in azure_client_params:
+                    v1_params["http_client"] = azure_client_params["http_client"]
+
+                verbose_logger.debug(f"Using Azure v1 API with base_url: {v1_params['base_url']}")
+
+                if _is_async is True:
+                    openai_client = AsyncOpenAI(**v1_params)  # type: ignore
+                else:
+                    openai_client = OpenAI(**v1_params)  # type: ignore
             else:
-                openai_client = AzureOpenAI(**azure_client_params)  # type: ignore
+                # Traditional Azure API uses AzureOpenAI client
+                if _is_async is True:
+                    openai_client = AsyncAzureOpenAI(**azure_client_params)
+                else:
+                    openai_client = AzureOpenAI(**azure_client_params)  # type: ignore
         else:
             openai_client = client
             if api_version is not None and isinstance(
-                openai_client._custom_query, dict
-            ):
+                openai_client, (AzureOpenAI, AsyncAzureOpenAI)
+            ) and isinstance(openai_client._custom_query, dict):
                 # set api_version to version passed by user
                 openai_client._custom_query.setdefault("api-version", api_version)
 

litellm/llms/azure/files/handler.py

@@ -1,7 +1,7 @@
 from typing import Any, Coroutine, Optional, Union, cast
 
 import httpx
-from openai import AsyncAzureOpenAI, AzureOpenAI
+from openai import AsyncAzureOpenAI, AsyncOpenAI, AzureOpenAI, OpenAI
 from openai.types.file_deleted import FileDeleted
 
 from litellm._logging import verbose_logger
@@ -40,7 +40,7 @@ class AzureOpenAIFilesAPI(BaseAzureLLM):
     async def acreate_file(
         self,
         create_file_data: CreateFileRequest,
-        openai_client: AsyncAzureOpenAI,
+        openai_client: Union[AsyncAzureOpenAI, AsyncOpenAI],
     ) -> OpenAIFileObject:
         verbose_logger.debug("create_file_data=%s", create_file_data)
         response = await openai_client.files.create(**self._prepare_create_file_data(create_file_data))  # type: ignore[arg-type]
@@ -56,11 +56,11 @@ class AzureOpenAIFilesAPI(BaseAzureLLM):
         api_version: Optional[str],
         timeout: Union[float, httpx.Timeout],
         max_retries: Optional[int],
-        client: Optional[Union[AzureOpenAI, AsyncAzureOpenAI]] = None,
+        client: Optional[Union[AzureOpenAI, AsyncAzureOpenAI, OpenAI, AsyncOpenAI]] = None,
         litellm_params: Optional[dict] = None,
     ) -> Union[OpenAIFileObject, Coroutine[Any, Any, OpenAIFileObject]]:
         openai_client: Optional[
-            Union[AzureOpenAI, AsyncAzureOpenAI]
+            Union[AzureOpenAI, AsyncAzureOpenAI, OpenAI, AsyncOpenAI]
         ] = self.get_azure_openai_client(
             litellm_params=litellm_params or {},
             api_key=api_key,
@@ -75,20 +75,20 @@ class AzureOpenAIFilesAPI(BaseAzureLLM):
             )
 
         if _is_async is True:
-            if not isinstance(openai_client, AsyncAzureOpenAI):
+            if not isinstance(openai_client, (AsyncAzureOpenAI, AsyncOpenAI)):
                 raise ValueError(
                     "AzureOpenAI client is not an instance of AsyncAzureOpenAI. Make sure you passed an AsyncAzureOpenAI client."
                 )
             return self.acreate_file(
                 create_file_data=create_file_data, openai_client=openai_client
             )
-        response = cast(AzureOpenAI, openai_client).files.create(**self._prepare_create_file_data(create_file_data))  # type: ignore[arg-type]
+        response = cast(Union[AzureOpenAI, OpenAI], openai_client).files.create(**self._prepare_create_file_data(create_file_data))  # type: ignore[arg-type]
         return OpenAIFileObject(**response.model_dump())
 
     async def afile_content(
         self,
         file_content_request: FileContentRequest,
-        openai_client: AsyncAzureOpenAI,
+        openai_client: Union[AsyncAzureOpenAI, AsyncOpenAI],
     ) -> HttpxBinaryResponseContent:
         response = await openai_client.files.content(**file_content_request)
         return HttpxBinaryResponseContent(response=response.response)
@@ -102,13 +102,13 @@ class AzureOpenAIFilesAPI(BaseAzureLLM):
         timeout: Union[float, httpx.Timeout],
         max_retries: Optional[int],
         api_version: Optional[str] = None,
-        client: Optional[Union[AzureOpenAI, AsyncAzureOpenAI]] = None,
+        client: Optional[Union[AzureOpenAI, AsyncAzureOpenAI, OpenAI, AsyncOpenAI]] = None,
         litellm_params: Optional[dict] = None,
     ) -> Union[
         HttpxBinaryResponseContent, Coroutine[Any, Any, HttpxBinaryResponseContent]
     ]:
         openai_client: Optional[
-            Union[AzureOpenAI, AsyncAzureOpenAI]
+            Union[AzureOpenAI, AsyncAzureOpenAI, OpenAI, AsyncOpenAI]
         ] = self.get_azure_openai_client(
             litellm_params=litellm_params or {},
             api_key=api_key,
@@ -123,7 +123,7 @@ class AzureOpenAIFilesAPI(BaseAzureLLM):
             )
 
         if _is_async is True:
-            if not isinstance(openai_client, AsyncAzureOpenAI):
+            if not isinstance(openai_client, (AsyncAzureOpenAI, AsyncOpenAI)):
                 raise ValueError(
                     "AzureOpenAI client is not an instance of AsyncAzureOpenAI. Make sure you passed an AsyncAzureOpenAI client."
                 )
@@ -131,7 +131,7 @@ class AzureOpenAIFilesAPI(BaseAzureLLM):
                 file_content_request=file_content_request,
                 openai_client=openai_client,
             )
-        response = cast(AzureOpenAI, openai_client).files.content(
+        response = cast(Union[AzureOpenAI, OpenAI], openai_client).files.content(
             **file_content_request
         )
 
@@ -140,7 +140,7 @@ class AzureOpenAIFilesAPI(BaseAzureLLM):
     async def aretrieve_file(
         self,
         file_id: str,
-        openai_client: AsyncAzureOpenAI,
+        openai_client: Union[AsyncAzureOpenAI, AsyncOpenAI],
     ) -> FileObject:
         response = await openai_client.files.retrieve(file_id=file_id)
         return response
@@ -154,11 +154,11 @@ class AzureOpenAIFilesAPI(BaseAzureLLM):
         timeout: Union[float, httpx.Timeout],
         max_retries: Optional[int],
         api_version: Optional[str] = None,
-        client: Optional[Union[AzureOpenAI, AsyncAzureOpenAI]] = None,
+        client: Optional[Union[AzureOpenAI, AsyncAzureOpenAI, OpenAI, AsyncOpenAI]] = None,
         litellm_params: Optional[dict] = None,
     ):
         openai_client: Optional[
-            Union[AzureOpenAI, AsyncAzureOpenAI]
+            Union[AzureOpenAI, AsyncAzureOpenAI, OpenAI, AsyncOpenAI]
         ] = self.get_azure_openai_client(
             litellm_params=litellm_params or {},
             api_key=api_key,
@@ -173,7 +173,7 @@ class AzureOpenAIFilesAPI(BaseAzureLLM):
             )
 
         if _is_async is True:
-            if not isinstance(openai_client, AsyncAzureOpenAI):
+            if not isinstance(openai_client, (AsyncAzureOpenAI, AsyncOpenAI)):
                 raise ValueError(
                     "AzureOpenAI client is not an instance of AsyncAzureOpenAI. Make sure you passed an AsyncAzureOpenAI client."
                 )
@@ -188,7 +188,7 @@ class AzureOpenAIFilesAPI(BaseAzureLLM):
     async def adelete_file(
         self,
         file_id: str,
-        openai_client: AsyncAzureOpenAI,
+        openai_client: Union[AsyncAzureOpenAI, AsyncOpenAI],
     ) -> FileDeleted:
         response = await openai_client.files.delete(file_id=file_id)
 
@@ -206,11 +206,11 @@ class AzureOpenAIFilesAPI(BaseAzureLLM):
         max_retries: Optional[int],
         organization: Optional[str] = None,
         api_version: Optional[str] = None,
-        client: Optional[Union[AzureOpenAI, AsyncAzureOpenAI]] = None,
+        client: Optional[Union[AzureOpenAI, AsyncAzureOpenAI, OpenAI, AsyncOpenAI]] = None,
         litellm_params: Optional[dict] = None,
     ):
         openai_client: Optional[
-            Union[AzureOpenAI, AsyncAzureOpenAI]
+            Union[AzureOpenAI, AsyncAzureOpenAI, OpenAI, AsyncOpenAI]
         ] = self.get_azure_openai_client(
             litellm_params=litellm_params or {},
             api_key=api_key,
@@ -225,7 +225,7 @@ class AzureOpenAIFilesAPI(BaseAzureLLM):
             )
 
         if _is_async is True:
-            if not isinstance(openai_client, AsyncAzureOpenAI):
+            if not isinstance(openai_client, (AsyncAzureOpenAI, AsyncOpenAI)):
                 raise ValueError(
                     "AzureOpenAI client is not an instance of AsyncAzureOpenAI. Make sure you passed an AsyncAzureOpenAI client."
                 )
@@ -242,7 +242,7 @@ class AzureOpenAIFilesAPI(BaseAzureLLM):
 
     async def alist_files(
         self,
-        openai_client: AsyncAzureOpenAI,
+        openai_client: Union[AsyncAzureOpenAI, AsyncOpenAI],
         purpose: Optional[str] = None,
     ):
         if isinstance(purpose, str):
@@ -260,11 +260,11 @@ class AzureOpenAIFilesAPI(BaseAzureLLM):
         max_retries: Optional[int],
         purpose: Optional[str] = None,
         api_version: Optional[str] = None,
-        client: Optional[Union[AzureOpenAI, AsyncAzureOpenAI]] = None,
+        client: Optional[Union[AzureOpenAI, AsyncAzureOpenAI, OpenAI, AsyncOpenAI]] = None,
         litellm_params: Optional[dict] = None,
     ):
         openai_client: Optional[
-            Union[AzureOpenAI, AsyncAzureOpenAI]
+            Union[AzureOpenAI, AsyncAzureOpenAI, OpenAI, AsyncOpenAI]
         ] = self.get_azure_openai_client(
             litellm_params=litellm_params or {},
             api_key=api_key,
@@ -279,7 +279,7 @@ class AzureOpenAIFilesAPI(BaseAzureLLM):
             )
 
         if _is_async is True:
-            if not isinstance(openai_client, AsyncAzureOpenAI):
+            if not isinstance(openai_client, (AsyncAzureOpenAI, AsyncOpenAI)):
                 raise ValueError(
                     "AzureOpenAI client is not an instance of AsyncAzureOpenAI. Make sure you passed an AsyncAzureOpenAI client."
                 )

litellm/llms/azure_ai/anthropic/count_tokens/__init__.py

@@ -0,0 +1,19 @@
+"""
+Azure AI Anthropic CountTokens API implementation.
+"""
+
+from litellm.llms.azure_ai.anthropic.count_tokens.handler import (
+    AzureAIAnthropicCountTokensHandler,
+)
+from litellm.llms.azure_ai.anthropic.count_tokens.token_counter import (
+    AzureAIAnthropicTokenCounter,
+)
+from litellm.llms.azure_ai.anthropic.count_tokens.transformation import (
+    AzureAIAnthropicCountTokensConfig,
+)
+
+__all__ = [
+    "AzureAIAnthropicCountTokensHandler",
+    "AzureAIAnthropicCountTokensConfig",
+    "AzureAIAnthropicTokenCounter",
+]

litellm/llms/azure_ai/anthropic/count_tokens/handler.py

@@ -0,0 +1,127 @@
+"""
+Azure AI Anthropic CountTokens API handler.
+
+Uses httpx for HTTP requests with Azure authentication.
+"""
+
+from typing import Any, Dict, List, Optional, Union
+
+import httpx
+
+import litellm
+from litellm._logging import verbose_logger
+from litellm.llms.anthropic.common_utils import AnthropicError
+from litellm.llms.azure_ai.anthropic.count_tokens.transformation import (
+    AzureAIAnthropicCountTokensConfig,
+)
+from litellm.llms.custom_httpx.http_handler import get_async_httpx_client
+
+
+class AzureAIAnthropicCountTokensHandler(AzureAIAnthropicCountTokensConfig):
+    """
+    Handler for Azure AI Anthropic CountTokens API requests.
+
+    Uses httpx for HTTP requests with Azure authentication.
+    """
+
+    async def handle_count_tokens_request(
+        self,
+        model: str,
+        messages: List[Dict[str, Any]],
+        api_key: str,
+        api_base: str,
+        litellm_params: Optional[Dict[str, Any]] = None,
+        timeout: Optional[Union[float, httpx.Timeout]] = None,
+    ) -> Dict[str, Any]:
+        """
+        Handle a CountTokens request using httpx with Azure authentication.
+
+        Args:
+            model: The model identifier (e.g., "claude-3-5-sonnet")
+            messages: The messages to count tokens for
+            api_key: The Azure AI API key
+            api_base: The Azure AI API base URL
+            litellm_params: Optional LiteLLM parameters
+            timeout: Optional timeout for the request (defaults to litellm.request_timeout)
+
+        Returns:
+            Dictionary containing token count response
+
+        Raises:
+            AnthropicError: If the API request fails
+        """
+        try:
+            # Validate the request
+            self.validate_request(model, messages)
+
+            verbose_logger.debug(
+                f"Processing Azure AI Anthropic CountTokens request for model: {model}"
+            )
+
+            # Transform request to Anthropic format
+            request_body = self.transform_request_to_count_tokens(
+                model=model,
+                messages=messages,
+            )
+
+            verbose_logger.debug(f"Transformed request: {request_body}")
+
+            # Get endpoint URL
+            endpoint_url = self.get_count_tokens_endpoint(api_base)
+
+            verbose_logger.debug(f"Making request to: {endpoint_url}")
+
+            # Get required headers with Azure authentication
+            headers = self.get_required_headers(
+                api_key=api_key,
+                litellm_params=litellm_params,
+            )
+
+            # Use LiteLLM's async httpx client
+            async_client = get_async_httpx_client(
+                llm_provider=litellm.LlmProviders.AZURE_AI
+            )
+
+            # Use provided timeout or fall back to litellm.request_timeout
+            request_timeout = timeout if timeout is not None else litellm.request_timeout
+
+            response = await async_client.post(
+                endpoint_url,
+                headers=headers,
+                json=request_body,
+                timeout=request_timeout,
+            )
+
+            verbose_logger.debug(f"Response status: {response.status_code}")
+
+            if response.status_code != 200:
+                error_text = response.text
+                verbose_logger.error(f"Azure AI Anthropic API error: {error_text}")
+                raise AnthropicError(
+                    status_code=response.status_code,
+                    message=error_text,
+                )
+
+            azure_response = response.json()
+
+            verbose_logger.debug(f"Azure AI Anthropic response: {azure_response}")
+
+            # Return Anthropic-compatible response directly - no transformation needed
+            return azure_response
+
+        except AnthropicError:
+            # Re-raise Anthropic exceptions as-is
+            raise
+        except httpx.HTTPStatusError as e:
+            # HTTP errors - preserve the actual status code
+            verbose_logger.error(f"HTTP error in CountTokens handler: {str(e)}")
+            raise AnthropicError(
+                status_code=e.response.status_code,
+                message=e.response.text,
+            )
+        except Exception as e:
+            verbose_logger.error(f"Error in CountTokens handler: {str(e)}")
+            raise AnthropicError(
+                status_code=500,
+                message=f"CountTokens processing error: {str(e)}",
+            )

litellm/llms/azure_ai/anthropic/count_tokens/token_counter.py

@@ -0,0 +1,119 @@
+"""
+Azure AI Anthropic Token Counter implementation using the CountTokens API.
+"""
+
+import os
+from typing import Any, Dict, List, Optional
+
+from litellm._logging import verbose_logger
+from litellm.llms.azure_ai.anthropic.count_tokens.handler import (
+    AzureAIAnthropicCountTokensHandler,
+)
+from litellm.llms.base_llm.base_utils import BaseTokenCounter
+from litellm.types.utils import LlmProviders, TokenCountResponse
+
+# Global handler instance - reuse across all token counting requests
+azure_ai_anthropic_count_tokens_handler = AzureAIAnthropicCountTokensHandler()
+
+
+class AzureAIAnthropicTokenCounter(BaseTokenCounter):
+    """Token counter implementation for Azure AI Anthropic provider using the CountTokens API."""
+
+    def should_use_token_counting_api(
+        self,
+        custom_llm_provider: Optional[str] = None,
+    ) -> bool:
+        return custom_llm_provider == LlmProviders.AZURE_AI.value
+
+    async def count_tokens(
+        self,
+        model_to_use: str,
+        messages: Optional[List[Dict[str, Any]]],
+        contents: Optional[List[Dict[str, Any]]],
+        deployment: Optional[Dict[str, Any]] = None,
+        request_model: str = "",
+    ) -> Optional[TokenCountResponse]:
+        """
+        Count tokens using Azure AI Anthropic's CountTokens API.
+
+        Args:
+            model_to_use: The model identifier
+            messages: The messages to count tokens for
+            contents: Alternative content format (not used for Anthropic)
+            deployment: Deployment configuration containing litellm_params
+            request_model: The original request model name
+
+        Returns:
+            TokenCountResponse with token count, or None if counting fails
+        """
+        from litellm.llms.anthropic.common_utils import AnthropicError
+
+        if not messages:
+            return None
+
+        deployment = deployment or {}
+        litellm_params = deployment.get("litellm_params", {})
+
+        # Get Azure AI API key from deployment config or environment
+        api_key = litellm_params.get("api_key")
+        if not api_key:
+            api_key = os.getenv("AZURE_AI_API_KEY")
+
+        # Get API base from deployment config or environment
+        api_base = litellm_params.get("api_base")
+        if not api_base:
+            api_base = os.getenv("AZURE_AI_API_BASE")
+
+        if not api_key:
+            verbose_logger.warning("No Azure AI API key found for token counting")
+            return None
+
+        if not api_base:
+            verbose_logger.warning("No Azure AI API base found for token counting")
+            return None
+
+        try:
+            result = await azure_ai_anthropic_count_tokens_handler.handle_count_tokens_request(
+                model=model_to_use,
+                messages=messages,
+                api_key=api_key,
+                api_base=api_base,
+                litellm_params=litellm_params,
+            )
+
+            if result is not None:
+                return TokenCountResponse(
+                    total_tokens=result.get("input_tokens", 0),
+                    request_model=request_model,
+                    model_used=model_to_use,
+                    tokenizer_type="azure_ai_anthropic_api",
+                    original_response=result,
+                )
+        except AnthropicError as e:
+            verbose_logger.warning(
+                f"Azure AI Anthropic CountTokens API error: status={e.status_code}, message={e.message}"
+            )
+            return TokenCountResponse(
+                total_tokens=0,
+                request_model=request_model,
+                model_used=model_to_use,
+                tokenizer_type="azure_ai_anthropic_api",
+                error=True,
+                error_message=e.message,
+                status_code=e.status_code,
+            )
+        except Exception as e:
+            verbose_logger.warning(
+                f"Error calling Azure AI Anthropic CountTokens API: {e}"
+            )
+            return TokenCountResponse(
+                total_tokens=0,
+                request_model=request_model,
+                model_used=model_to_use,
+                tokenizer_type="azure_ai_anthropic_api",
+                error=True,
+                error_message=str(e),
+                status_code=500,
+            )
+
+        return None

litellm/llms/azure_ai/anthropic/count_tokens/transformation.py

@@ -0,0 +1,88 @@
+"""
+Azure AI Anthropic CountTokens API transformation logic.
+
+Extends the base Anthropic CountTokens transformation with Azure authentication.
+"""
+
+from typing import Any, Dict, Optional
+
+from litellm.constants import ANTHROPIC_TOKEN_COUNTING_BETA_VERSION
+from litellm.llms.anthropic.count_tokens.transformation import (
+    AnthropicCountTokensConfig,
+)
+from litellm.llms.azure.common_utils import BaseAzureLLM
+from litellm.types.router import GenericLiteLLMParams
+
+
+class AzureAIAnthropicCountTokensConfig(AnthropicCountTokensConfig):
+    """
+    Configuration and transformation logic for Azure AI Anthropic CountTokens API.
+
+    Extends AnthropicCountTokensConfig with Azure authentication.
+    Azure AI Anthropic uses the same endpoint format but with Azure auth headers.
+    """
+
+    def get_required_headers(
+        self,
+        api_key: str,
+        litellm_params: Optional[Dict[str, Any]] = None,
+    ) -> Dict[str, str]:
+        """
+        Get the required headers for the Azure AI Anthropic CountTokens API.
+
+        Uses Azure authentication (api-key header) instead of Anthropic's x-api-key.
+
+        Args:
+            api_key: The Azure AI API key
+            litellm_params: Optional LiteLLM parameters for additional auth config
+
+        Returns:
+            Dictionary of required headers with Azure authentication
+        """
+        # Start with base headers
+        headers = {
+            "Content-Type": "application/json",
+            "anthropic-version": "2023-06-01",
+            "anthropic-beta": ANTHROPIC_TOKEN_COUNTING_BETA_VERSION,
+        }
+
+        # Use Azure authentication
+        litellm_params = litellm_params or {}
+        if "api_key" not in litellm_params:
+            litellm_params["api_key"] = api_key
+
+        litellm_params_obj = GenericLiteLLMParams(**litellm_params)
+
+        # Get Azure auth headers
+        azure_headers = BaseAzureLLM._base_validate_azure_environment(
+            headers={}, litellm_params=litellm_params_obj
+        )
+
+        # Merge Azure auth headers
+        headers.update(azure_headers)
+
+        return headers
+
+    def get_count_tokens_endpoint(self, api_base: str) -> str:
+        """
+        Get the Azure AI Anthropic CountTokens API endpoint.
+
+        Args:
+            api_base: The Azure AI API base URL 
+                      (e.g., https://my-resource.services.ai.azure.com or
+                       https://my-resource.services.ai.azure.com/anthropic)
+
+        Returns:
+            The endpoint URL for the CountTokens API
+        """
+        # Azure AI Anthropic endpoint format:
+        # https://<resource>.services.ai.azure.com/anthropic/v1/messages/count_tokens
+        api_base = api_base.rstrip("/")
+
+        # Ensure the URL has /anthropic path
+        if not api_base.endswith("/anthropic"):
+            if "/anthropic" not in api_base:
+                api_base = f"{api_base}/anthropic"
+
+        # Add the count_tokens path
+        return f"{api_base}/v1/messages/count_tokens"

litellm/llms/azure_ai/common_utils.py

@@ -1,17 +1,22 @@
 from typing import List, Literal, Optional
 
 import litellm
-from litellm.llms.base_llm.base_utils import BaseLLMModelInfo
+from litellm.llms.base_llm.base_utils import BaseLLMModelInfo, BaseTokenCounter
 from litellm.secret_managers.main import get_secret_str
 from litellm.types.llms.openai import AllMessageValues
 
 
 class AzureFoundryModelInfo(BaseLLMModelInfo):
+    """Model info for Azure AI / Azure Foundry models."""
+
+    def __init__(self, model: Optional[str] = None):
+        self._model = model
+
     @staticmethod
     def get_azure_ai_route(model: str) -> Literal["agents", "default"]:
         """
         Get the Azure AI route for the given model.
-        
+
         Similar to BedrockModelInfo.get_bedrock_route().
         """
         if "agents/" in model:
@@ -20,34 +25,54 @@ class AzureFoundryModelInfo(BaseLLMModelInfo):
 
     @staticmethod
     def get_api_base(api_base: Optional[str] = None) -> Optional[str]:
-        return (
-                api_base
-                or litellm.api_base
-                or get_secret_str("AZURE_AI_API_BASE")
-        )
-    
+        return api_base or litellm.api_base or get_secret_str("AZURE_AI_API_BASE")
+
     @staticmethod
     def get_api_key(api_key: Optional[str] = None) -> Optional[str]:
         return (
-                api_key
-                or litellm.api_key
-                or litellm.openai_key
-                or get_secret_str("AZURE_AI_API_KEY")
-            )
-    
+            api_key
+            or litellm.api_key
+            or litellm.openai_key
+            or get_secret_str("AZURE_AI_API_KEY")
+        )
+
     @property
     def api_version(self, api_version: Optional[str] = None) -> Optional[str]:
         api_version = (
-            api_version
-            or litellm.api_version
-            or get_secret_str("AZURE_API_VERSION")
+            api_version or litellm.api_version or get_secret_str("AZURE_API_VERSION")
         )
         return api_version
-    
+
+    def get_token_counter(self) -> Optional[BaseTokenCounter]:
+        """
+        Factory method to create a token counter for Azure AI.
+
+        Returns:
+            AzureAIAnthropicTokenCounter for Claude models, None otherwise.
+        """
+        # Only return token counter for Claude models
+        if self._model and "claude" in self._model.lower():
+            from litellm.llms.azure_ai.anthropic.count_tokens.token_counter import (
+                AzureAIAnthropicTokenCounter,
+            )
+
+            return AzureAIAnthropicTokenCounter()
+        return None
+
+    def get_models(
+        self, api_key: Optional[str] = None, api_base: Optional[str] = None
+    ) -> List[str]:
+        """
+        Returns a list of models supported by Azure AI.
+        
+        Azure AI doesn't have a standard model listing endpoint,
+        so this returns an empty list.
+        """
+        return []
+
     #########################################################
     # Not implemented methods
     #########################################################
-        
 
     @staticmethod
     def get_base_model(model: str) -> Optional[str]:
@@ -64,4 +89,6 @@ class AzureFoundryModelInfo(BaseLLMModelInfo):
         api_base: Optional[str] = None,
     ) -> dict:
         """Azure Foundry sends api key in query params"""
-        raise NotImplementedError("Azure Foundry does not support environment validation")
+        raise NotImplementedError(
+            "Azure Foundry does not support environment validation"
+        )

litellm/llms/bedrock/chat/agentcore/sse_iterator.py

@@ -1,252 +0,0 @@
-"""
-SSE Stream Iterator for Bedrock AgentCore.
-
-Handles Server-Sent Events (SSE) streaming responses from AgentCore.
-"""
-
-import json
-from typing import TYPE_CHECKING, Any, Optional
-
-import httpx
-
-from litellm._logging import verbose_logger
-from litellm._uuid import uuid
-from litellm.types.llms.bedrock_agentcore import AgentCoreUsage
-from litellm.types.utils import Delta, ModelResponse, StreamingChoices, Usage
-
-if TYPE_CHECKING:
-    pass
-
-
-class AgentCoreSSEStreamIterator:
-    """
-    Iterator for AgentCore SSE streaming responses.
-    Supports both sync and async iteration.
-
-    CRITICAL: The line iterators are created lazily on first access and reused.
-    We must NOT create new iterators in __aiter__/__iter__ because
-    CustomStreamWrapper calls __aiter__ on every call to its __anext__,
-    which would create new iterators and cause StreamConsumed errors.
-    """
-
-    def __init__(self, response: httpx.Response, model: str):
-        self.response = response
-        self.model = model
-        self.finished = False
-        self._sync_iter: Any = None
-        self._async_iter: Any = None
-        self._sync_iter_initialized = False
-        self._async_iter_initialized = False
-
-    def __iter__(self):
-        """Initialize sync iteration - create iterator lazily on first call only."""
-        if not self._sync_iter_initialized:
-            self._sync_iter = iter(self.response.iter_lines())
-            self._sync_iter_initialized = True
-        return self
-
-    def __aiter__(self):
-        """Initialize async iteration - create iterator lazily on first call only."""
-        if not self._async_iter_initialized:
-            self._async_iter = self.response.aiter_lines().__aiter__()
-            self._async_iter_initialized = True
-        return self
-
-    def _parse_sse_line(self, line: str) -> Optional[ModelResponse]:
-        """
-        Parse a single SSE line and return a ModelResponse chunk if applicable.
-
-        AgentCore SSE format:
-        - data: {"event": {"contentBlockDelta": {"delta": {"text": "..."}}}}
-        - data: {"event": {"metadata": {"usage": {...}}}}
-        - data: {"message": {...}}
-        """
-        line = line.strip()
-        if not line or not line.startswith("data:"):
-            return None
-
-        json_str = line[5:].strip()
-        if not json_str:
-            return None
-
-        try:
-            data = json.loads(json_str)
-
-            # Skip non-dict data (some lines contain Python repr strings)
-            if not isinstance(data, dict):
-                return None
-
-            # Process content delta events
-            if "event" in data and isinstance(data["event"], dict):
-                event_payload = data["event"]
-                content_block_delta = event_payload.get("contentBlockDelta")
-
-                if content_block_delta:
-                    delta = content_block_delta.get("delta", {})
-                    text = delta.get("text", "")
-
-                    if text:
-                        # Return chunk with text
-                        chunk = ModelResponse(
-                            id=f"chatcmpl-{uuid.uuid4()}",
-                            created=0,
-                            model=self.model,
-                            object="chat.completion.chunk",
-                        )
-
-                        chunk.choices = [
-                            StreamingChoices(
-                                finish_reason=None,
-                                index=0,
-                                delta=Delta(content=text, role="assistant"),
-                            )
-                        ]
-
-                        return chunk
-
-                # Check for metadata/usage - this signals the end
-                metadata = event_payload.get("metadata")
-                if metadata and "usage" in metadata:
-                    chunk = ModelResponse(
-                        id=f"chatcmpl-{uuid.uuid4()}",
-                        created=0,
-                        model=self.model,
-                        object="chat.completion.chunk",
-                    )
-
-                    chunk.choices = [
-                        StreamingChoices(
-                            finish_reason="stop",
-                            index=0,
-                            delta=Delta(),
-                        )
-                    ]
-
-                    usage_data: AgentCoreUsage = metadata["usage"]  # type: ignore
-                    setattr(
-                        chunk,
-                        "usage",
-                        Usage(
-                            prompt_tokens=usage_data.get("inputTokens", 0),
-                            completion_tokens=usage_data.get("outputTokens", 0),
-                            total_tokens=usage_data.get("totalTokens", 0),
-                        ),
-                    )
-
-                    self.finished = True
-                    return chunk
-
-            # Check for final message (alternative finish signal)
-            if "message" in data and isinstance(data["message"], dict):
-                if not self.finished:
-                    chunk = ModelResponse(
-                        id=f"chatcmpl-{uuid.uuid4()}",
-                        created=0,
-                        model=self.model,
-                        object="chat.completion.chunk",
-                    )
-
-                    chunk.choices = [
-                        StreamingChoices(
-                            finish_reason="stop",
-                            index=0,
-                            delta=Delta(),
-                        )
-                    ]
-
-                    self.finished = True
-                    return chunk
-
-        except json.JSONDecodeError:
-            verbose_logger.debug(f"Skipping non-JSON SSE line: {line[:100]}")
-
-        return None
-
-    def _create_final_chunk(self) -> ModelResponse:
-        """Create a final chunk to signal stream completion."""
-        chunk = ModelResponse(
-            id=f"chatcmpl-{uuid.uuid4()}",
-            created=0,
-            model=self.model,
-            object="chat.completion.chunk",
-        )
-
-        chunk.choices = [
-            StreamingChoices(
-                finish_reason="stop",
-                index=0,
-                delta=Delta(),
-            )
-        ]
-
-        return chunk
-
-    def __next__(self) -> ModelResponse:
-        """
-        Sync iteration - parse SSE events and yield ModelResponse chunks.
-        
-        Uses next() on the stored iterator to properly resume between calls.
-        """
-        try:
-            if self._sync_iter is None:
-                raise StopIteration
-
-            # Keep getting lines until we have a result to return
-            while True:
-                try:
-                    line = next(self._sync_iter)
-                except StopIteration:
-                    # Stream ended - send final chunk if not already finished
-                    if not self.finished:
-                        self.finished = True
-                        return self._create_final_chunk()
-                    raise
-
-                result = self._parse_sse_line(line)
-                if result is not None:
-                    return result
-
-        except StopIteration:
-            raise
-        except httpx.StreamConsumed:
-            raise StopIteration
-        except httpx.StreamClosed:
-            raise StopIteration
-        except Exception as e:
-            verbose_logger.error(f"Error in AgentCore SSE stream: {str(e)}")
-            raise StopIteration
-
-    async def __anext__(self) -> ModelResponse:
-        """
-        Async iteration - parse SSE events and yield ModelResponse chunks.
-
-        Uses __anext__() on the stored iterator to properly resume between calls.
-        """
-        try:
-            if self._async_iter is None:
-                raise StopAsyncIteration
-
-            # Keep getting lines until we have a result to return
-            while True:
-                try:
-                    line = await self._async_iter.__anext__()
-                except StopAsyncIteration:
-                    # Stream ended - send final chunk if not already finished
-                    if not self.finished:
-                        self.finished = True
-                        return self._create_final_chunk()
-                    raise
-
-                result = self._parse_sse_line(line)
-                if result is not None:
-                    return result
-
-        except StopAsyncIteration:
-            raise
-        except httpx.StreamConsumed:
-            raise StopAsyncIteration
-        except httpx.StreamClosed:
-            raise StopAsyncIteration
-        except Exception as e:
-            verbose_logger.error(f"Error in AgentCore SSE stream: {str(e)}")
-            raise StopAsyncIteration

litellm/llms/bedrock/chat/agentcore/transformation.py

@@ -5,6 +5,7 @@ https://docs.aws.amazon.com/bedrock/latest/APIReference/API_agentcore_InvokeAgen
 """
 
 import json
+from collections.abc import AsyncGenerator
 from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union, cast
 from urllib.parse import quote
 
@@ -15,9 +16,9 @@ from litellm._uuid import uuid
 from litellm.litellm_core_utils.prompt_templates.common_utils import (
     convert_content_list_to_str,
 )
+from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper
 from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException
 from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM
-from litellm.llms.bedrock.chat.agentcore.sse_iterator import AgentCoreSSEStreamIterator
 from litellm.llms.bedrock.common_utils import BedrockError
 from litellm.types.llms.bedrock_agentcore import (
     AgentCoreMessage,
@@ -25,19 +26,17 @@ from litellm.types.llms.bedrock_agentcore import (
     AgentCoreUsage,
 )
 from litellm.types.llms.openai import AllMessageValues
-from litellm.types.utils import Choices, Message, ModelResponse, Usage
+from litellm.types.utils import Choices, Delta, Message, ModelResponse, StreamingChoices, Usage
 
 if TYPE_CHECKING:
     from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
     from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
-    from litellm.utils import CustomStreamWrapper
 
     LiteLLMLoggingObj = _LiteLLMLoggingObj
 else:
     LiteLLMLoggingObj = Any
     HTTPHandler = Any
     AsyncHTTPHandler = Any
-    CustomStreamWrapper = Any
 
 
 class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM):
@@ -116,7 +115,8 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM):
         fake_stream: Optional[bool] = None,
     ) -> Tuple[dict, Optional[bytes]]:
         # Check if api_key (bearer token) is provided for Cognito authentication
-        jwt_token = optional_params.get("api_key")
+        # Priority: api_key parameter first, then optional_params
+        jwt_token = api_key or optional_params.get("api_key")
         if jwt_token:
             verbose_logger.debug(
                 f"AgentCore: Using Bearer token authentication (Cognito/JWT) - token: {jwt_token[:50]}..."
@@ -437,22 +437,104 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM):
             content=content, usage=usage_data, final_message=final_message
         )
 
-    def get_streaming_response(
+    def _stream_agentcore_response_sync(
         self,
+        response: httpx.Response,
         model: str,
-        raw_response: httpx.Response,
-    ) -> AgentCoreSSEStreamIterator:
+    ):
         """
-        Return a streaming iterator for SSE responses.
-
-        Args:
-            model: The model name
-            raw_response: Raw HTTP response with streaming data
-
-        Returns:
-            AgentCoreSSEStreamIterator: Iterator that yields ModelResponse chunks
+        Internal sync generator that parses SSE and yields ModelResponse chunks.
         """
-        return AgentCoreSSEStreamIterator(response=raw_response, model=model)
+        buffer = ""
+        for text_chunk in response.iter_text():
+            buffer += text_chunk
+
+            # Process complete lines
+            while '\n' in buffer:
+                line, buffer = buffer.split('\n', 1)
+                line = line.strip()
+
+                if not line or not line.startswith('data:'):
+                    continue
+
+                json_str = line[5:].strip()
+                if not json_str:
+                    continue
+
+                try:
+                    data_obj = json.loads(json_str)
+                    if not isinstance(data_obj, dict):
+                        continue
+
+                    # Process contentBlockDelta events
+                    if "event" in data_obj and isinstance(data_obj["event"], dict):
+                        event_payload = data_obj["event"]
+                        content_block_delta = event_payload.get("contentBlockDelta")
+
+                        if content_block_delta:
+                            delta = content_block_delta.get("delta", {})
+                            text = delta.get("text", "")
+
+                            if text:
+                                chunk = ModelResponse(
+                                    id=f"chatcmpl-{uuid.uuid4()}",
+                                    created=0,
+                                    model=model,
+                                    object="chat.completion.chunk",
+                                )
+                                chunk.choices = [
+                                    StreamingChoices(
+                                        finish_reason=None,
+                                        index=0,
+                                        delta=Delta(content=text, role="assistant"),
+                                    )
+                                ]
+                                yield chunk
+
+                        # Process metadata/usage
+                        metadata = event_payload.get("metadata")
+                        if metadata and "usage" in metadata:
+                            chunk = ModelResponse(
+                                id=f"chatcmpl-{uuid.uuid4()}",
+                                created=0,
+                                model=model,
+                                object="chat.completion.chunk",
+                            )
+                            chunk.choices = [
+                                StreamingChoices(
+                                    finish_reason="stop",
+                                    index=0,
+                                    delta=Delta(),
+                                )
+                            ]
+                            usage_data: AgentCoreUsage = metadata["usage"]  # type: ignore
+                            setattr(chunk, "usage", Usage(
+                                prompt_tokens=usage_data.get("inputTokens", 0),
+                                completion_tokens=usage_data.get("outputTokens", 0),
+                                total_tokens=usage_data.get("totalTokens", 0),
+                            ))
+                            yield chunk
+
+                    # Process final message
+                    if "message" in data_obj and isinstance(data_obj["message"], dict):
+                        chunk = ModelResponse(
+                            id=f"chatcmpl-{uuid.uuid4()}",
+                            created=0,
+                            model=model,
+                            object="chat.completion.chunk",
+                        )
+                        chunk.choices = [
+                            StreamingChoices(
+                                finish_reason="stop",
+                                index=0,
+                                delta=Delta(),
+                            )
+                        ]
+                        yield chunk
+
+                except json.JSONDecodeError:
+                    verbose_logger.debug(f"Skipping non-JSON SSE line: {line[:100]}")
+                    continue
 
     def get_sync_custom_stream_wrapper(
         self,
@@ -466,17 +548,14 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM):
         client: Optional[Union[HTTPHandler, "AsyncHTTPHandler"]] = None,
         json_mode: Optional[bool] = None,
         signed_json_body: Optional[bytes] = None,
-    ) -> CustomStreamWrapper:
+    ) -> "CustomStreamWrapper":
         """
-        Get a CustomStreamWrapper for synchronous streaming.
-
-        This is called when stream=True is passed to completion().
+        Simplified sync streaming - returns a generator that yields ModelResponse chunks.
         """
         from litellm.llms.custom_httpx.http_handler import (
             HTTPHandler,
             _get_httpx_client,
         )
-        from litellm.utils import CustomStreamWrapper
 
         if client is None or not isinstance(client, HTTPHandler):
             client = _get_httpx_client(params={})
@@ -488,7 +567,7 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM):
             api_base,
             headers=headers,
             data=signed_json_body if signed_json_body else json.dumps(data),
-            stream=True,  # THIS IS KEY - tells httpx to not buffer
+            stream=True,
             logging_obj=logging_obj,
         )
 
@@ -497,18 +576,6 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM):
                 status_code=response.status_code, message=str(response.read())
             )
 
-        # Create iterator for SSE stream
-        completion_stream = self.get_streaming_response(
-            model=model, raw_response=response
-        )
-
-        streaming_response = CustomStreamWrapper(
-            completion_stream=completion_stream,
-            model=model,
-            custom_llm_provider=custom_llm_provider,
-            logging_obj=logging_obj,
-        )
-
         # LOGGING
         logging_obj.post_call(
             input=messages,
@@ -517,7 +584,112 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM):
             additional_args={"complete_input_dict": data},
         )
 
-        return streaming_response
+        # Wrap the generator in CustomStreamWrapper
+        return CustomStreamWrapper(
+            completion_stream=self._stream_agentcore_response_sync(response, model),
+            model=model,
+            custom_llm_provider="bedrock",
+            logging_obj=logging_obj,
+        )
+
+    async def _stream_agentcore_response(
+        self,
+        response: httpx.Response,
+        model: str,
+    ) -> AsyncGenerator[ModelResponse, None]:
+        """
+        Internal async generator that parses SSE and yields ModelResponse chunks.
+        """
+        buffer = ""
+        async for text_chunk in response.aiter_text():
+            buffer += text_chunk
+
+            # Process complete lines
+            while '\n' in buffer:
+                line, buffer = buffer.split('\n', 1)
+                line = line.strip()
+
+                if not line or not line.startswith('data:'):
+                    continue
+
+                json_str = line[5:].strip()
+                if not json_str:
+                    continue
+
+                try:
+                    data_obj = json.loads(json_str)
+                    if not isinstance(data_obj, dict):
+                        continue
+
+                    # Process contentBlockDelta events
+                    if "event" in data_obj and isinstance(data_obj["event"], dict):
+                        event_payload = data_obj["event"]
+                        content_block_delta = event_payload.get("contentBlockDelta")
+
+                        if content_block_delta:
+                            delta = content_block_delta.get("delta", {})
+                            text = delta.get("text", "")
+
+                            if text:
+                                chunk = ModelResponse(
+                                    id=f"chatcmpl-{uuid.uuid4()}",
+                                    created=0,
+                                    model=model,
+                                    object="chat.completion.chunk",
+                                )
+                                chunk.choices = [
+                                    StreamingChoices(
+                                        finish_reason=None,
+                                        index=0,
+                                        delta=Delta(content=text, role="assistant"),
+                                    )
+                                ]
+                                yield chunk
+
+                        # Process metadata/usage
+                        metadata = event_payload.get("metadata")
+                        if metadata and "usage" in metadata:
+                            chunk = ModelResponse(
+                                id=f"chatcmpl-{uuid.uuid4()}",
+                                created=0,
+                                model=model,
+                                object="chat.completion.chunk",
+                            )
+                            chunk.choices = [
+                                StreamingChoices(
+                                    finish_reason="stop",
+                                    index=0,
+                                    delta=Delta(),
+                                )
+                            ]
+                            usage_data: AgentCoreUsage = metadata["usage"]  # type: ignore
+                            setattr(chunk, "usage", Usage(
+                                prompt_tokens=usage_data.get("inputTokens", 0),
+                                completion_tokens=usage_data.get("outputTokens", 0),
+                                total_tokens=usage_data.get("totalTokens", 0),
+                            ))
+                            yield chunk
+
+                    # Process final message
+                    if "message" in data_obj and isinstance(data_obj["message"], dict):
+                        chunk = ModelResponse(
+                            id=f"chatcmpl-{uuid.uuid4()}",
+                            created=0,
+                            model=model,
+                            object="chat.completion.chunk",
+                        )
+                        chunk.choices = [
+                            StreamingChoices(
+                                finish_reason="stop",
+                                index=0,
+                                delta=Delta(),
+                            )
+                        ]
+                        yield chunk
+
+                except json.JSONDecodeError:
+                    verbose_logger.debug(f"Skipping non-JSON SSE line: {line[:100]}")
+                    continue
 
     async def get_async_custom_stream_wrapper(
         self,
@@ -531,17 +703,14 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM):
         client: Optional["AsyncHTTPHandler"] = None,
         json_mode: Optional[bool] = None,
         signed_json_body: Optional[bytes] = None,
-    ) -> CustomStreamWrapper:
+    ) -> "CustomStreamWrapper":
         """
-        Get a CustomStreamWrapper for asynchronous streaming.
-
-        This is called when stream=True is passed to acompletion().
+        Simplified async streaming - returns an async generator that yields ModelResponse chunks.
         """
         from litellm.llms.custom_httpx.http_handler import (
             AsyncHTTPHandler,
             get_async_httpx_client,
         )
-        from litellm.utils import CustomStreamWrapper
 
         if client is None or not isinstance(client, AsyncHTTPHandler):
             client = get_async_httpx_client(
@@ -555,7 +724,7 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM):
             api_base,
             headers=headers,
             data=signed_json_body if signed_json_body else json.dumps(data),
-            stream=True,  # THIS IS KEY - tells httpx to not buffer
+            stream=True,
             logging_obj=logging_obj,
         )
 
@@ -564,18 +733,6 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM):
                 status_code=response.status_code, message=str(await response.aread())
             )
 
-        # Create iterator for SSE stream
-        completion_stream = self.get_streaming_response(
-            model=model, raw_response=response
-        )
-
-        streaming_response = CustomStreamWrapper(
-            completion_stream=completion_stream,
-            model=model,
-            custom_llm_provider=custom_llm_provider,
-            logging_obj=logging_obj,
-        )
-
         # LOGGING
         logging_obj.post_call(
             input=messages,
@@ -584,7 +741,13 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM):
             additional_args={"complete_input_dict": data},
         )
 
-        return streaming_response
+        # Wrap the async generator in CustomStreamWrapper
+        return CustomStreamWrapper(
+            completion_stream=self._stream_agentcore_response(response, model),
+            model=model,
+            custom_llm_provider="bedrock",
+            logging_obj=logging_obj,
+        )
 
     @property
     def has_custom_stream_wrapper(self) -> bool:
@@ -692,4 +855,5 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM):
         stream: Optional[bool],
         custom_llm_provider: Optional[str] = None,
     ) -> bool:
-        return True
+        # AgentCore supports true streaming - don't buffer
+        return False

litellm/llms/bedrock/chat/converse_transformation.py

@@ -53,7 +53,13 @@ from litellm.types.utils import (
     PromptTokensDetailsWrapper,
     Usage,
 )
-from litellm.utils import add_dummy_tool, has_tool_call_blocks, supports_reasoning
+from litellm.utils import (
+    add_dummy_tool,
+    any_assistant_message_has_thinking_blocks,
+    has_tool_call_blocks,
+    last_assistant_with_tool_calls_has_no_thinking_blocks,
+    supports_reasoning,
+)
 
 from ..common_utils import (
     BedrockError,
@@ -773,7 +779,7 @@ class AmazonConverseConfig(BaseConfig):
             return optional_params
 
         """
-        Follow similar approach to anthropic - translate to a single tool call. 
+        Follow similar approach to anthropic - translate to a single tool call.
 
         When using tools in this way: - https://docs.anthropic.com/en/docs/build-with-claude/tool-use#json-mode
         - You usually want to provide a single tool
@@ -1070,9 +1076,28 @@ class AmazonConverseConfig(BaseConfig):
                     llm_provider="bedrock",
                 )
 
+        # Drop thinking param if thinking is enabled but thinking_blocks are missing
+        # This prevents the error: "Expected thinking or redacted_thinking, but found tool_use"
+        #
+        # IMPORTANT: Only drop thinking if NO assistant messages have thinking_blocks.
+        # If any message has thinking_blocks, we must keep thinking enabled, otherwise
+        # Related issues: https://github.com/BerriAI/litellm/issues/14194
+        if (
+            optional_params.get("thinking") is not None
+            and messages is not None
+            and last_assistant_with_tool_calls_has_no_thinking_blocks(messages)
+            and not any_assistant_message_has_thinking_blocks(messages)
+        ):
+            if litellm.modify_params:
+                optional_params.pop("thinking", None)
+                litellm.verbose_logger.warning(
+                    "Dropping 'thinking' param because the last assistant message with tool_calls "
+                    "has no thinking_blocks. The model won't use extended thinking for this turn."
+                )
+
         # Prepare and separate parameters
-        inference_params, additional_request_params, request_metadata = (
-            self._prepare_request_params(optional_params, model)
+        inference_params, additional_request_params, request_metadata = self._prepare_request_params(
+            optional_params, model
         )
 
         original_tools = inference_params.pop("tools", [])
@@ -1459,11 +1484,11 @@ class AmazonConverseConfig(BaseConfig):
             )
 
         """
-        Bedrock Response Object has optional message block 
+        Bedrock Response Object has optional message block
 
         completion_response["output"].get("message", None)
 
-        A message block looks like this (Example 1): 
+        A message block looks like this (Example 1):
         "output": {
             "message": {
                 "role": "assistant",

litellm/llms/bedrock/chat/invoke_handler.py

@@ -374,6 +374,29 @@ class BedrockLLM(BaseAWSLLM):
     def __init__(self) -> None:
         super().__init__()
 
+    @staticmethod
+    def is_claude_messages_api_model(model: str) -> bool:
+        """
+        Check if the model uses the Claude Messages API (Claude 3+).
+
+        Handles:
+        - Regional prefixes: eu.anthropic.claude-*, us.anthropic.claude-*
+        - Claude 3 models: claude-3-haiku, claude-3-sonnet, claude-3-opus, claude-3-5-*, claude-3-7-*
+        - Claude 4 models: claude-opus-4, claude-sonnet-4, claude-haiku-4
+        """
+        # Normalize model string to lowercase for matching
+        model_lower = model.lower()
+
+        # Claude 3+ indicators (all use Messages API)
+        messages_api_indicators = [
+            "claude-3",      # Claude 3.x models
+            "claude-opus-4", # Claude Opus 4
+            "claude-sonnet-4", # Claude Sonnet 4
+            "claude-haiku-4",  # Claude Haiku 4
+        ]
+
+        return any(indicator in model_lower for indicator in messages_api_indicators)
+
     def convert_messages_to_prompt(
         self, model, messages, provider, custom_prompt_dict
     ) -> Tuple[str, Optional[list]]:
@@ -465,7 +488,7 @@ class BedrockLLM(BaseAWSLLM):
                         completion_response["generations"][0]["finish_reason"]
                     )
             elif provider == "anthropic":
-                if model.startswith("anthropic.claude-3"):
+                if self.is_claude_messages_api_model(model):
                     json_schemas: dict = {}
                     _is_function_call = False
                     ## Handle Tool Calling
@@ -595,13 +618,12 @@ class BedrockLLM(BaseAWSLLM):
                         outputText = choice["message"].get("content")
                     elif "text" in choice:  # fallback for completion format
                         outputText = choice["text"]
-                    
                     # Set finish reason
                     if "finish_reason" in choice:
                         model_response.choices[0].finish_reason = map_finish_reason(
                             choice["finish_reason"]
                         )
-                    
+
                     # Set usage if available
                     if "usage" in completion_response:
                         usage = completion_response["usage"]
@@ -838,7 +860,7 @@ class BedrockLLM(BaseAWSLLM):
                     ] = True  # cohere requires stream = True in inference params
                 data = json.dumps({"prompt": prompt, **inference_params})
         elif provider == "anthropic":
-            if model.startswith("anthropic.claude-3"):
+            if self.is_claude_messages_api_model(model):
                 # Separate system prompt from rest of message
                 system_prompt_idx: list[int] = []
                 system_messages: list[str] = []
@@ -936,13 +958,13 @@ class BedrockLLM(BaseAWSLLM):
             # Use AmazonBedrockOpenAIConfig for proper OpenAI transformation
             openai_config = AmazonBedrockOpenAIConfig()
             supported_params = openai_config.get_supported_openai_params(model=model)
-            
+
             # Filter to only supported OpenAI params
             filtered_params = {
-                k: v for k, v in inference_params.items() 
+                k: v for k, v in inference_params.items()
                 if k in supported_params
             }
-            
+
             # OpenAI uses messages format, not prompt
             data = json.dumps({"messages": messages, **filtered_params})
         else:

litellm/llms/chatgpt/authenticator.py

@@ -0,0 +1,388 @@
+import base64
+import json
+import os
+import time
+from typing import Any, Dict, Optional
+
+import httpx
+
+from litellm._logging import verbose_logger
+from litellm.llms.custom_httpx.http_handler import _get_httpx_client
+
+from .common_utils import (
+    CHATGPT_API_BASE,
+    CHATGPT_AUTH_BASE,
+    CHATGPT_CLIENT_ID,
+    CHATGPT_DEVICE_CODE_URL,
+    CHATGPT_DEVICE_TOKEN_URL,
+    CHATGPT_DEVICE_VERIFY_URL,
+    CHATGPT_OAUTH_TOKEN_URL,
+    GetAccessTokenError,
+    GetDeviceCodeError,
+    RefreshAccessTokenError,
+)
+
+TOKEN_EXPIRY_SKEW_SECONDS = 60
+DEVICE_CODE_TIMEOUT_SECONDS = 15 * 60
+DEVICE_CODE_COOLDOWN_SECONDS = 5 * 60
+DEVICE_CODE_POLL_SLEEP_SECONDS = 5
+
+
+class Authenticator:
+    def __init__(self) -> None:
+        self.token_dir = os.getenv(
+            "CHATGPT_TOKEN_DIR",
+            os.path.expanduser("~/.config/litellm/chatgpt"),
+        )
+        self.auth_file = os.path.join(
+            self.token_dir, os.getenv("CHATGPT_AUTH_FILE", "auth.json")
+        )
+        self._ensure_token_dir()
+
+    def get_api_base(self) -> str:
+        return (
+            os.getenv("CHATGPT_API_BASE")
+            or os.getenv("OPENAI_CHATGPT_API_BASE")
+            or CHATGPT_API_BASE
+        )
+
+    def get_access_token(self) -> str:
+        auth_data = self._read_auth_file()
+        if auth_data:
+            access_token = auth_data.get("access_token")
+            if access_token and not self._is_token_expired(auth_data, access_token):
+                return access_token
+            refresh_token = auth_data.get("refresh_token")
+            if refresh_token:
+                try:
+                    refreshed = self._refresh_tokens(refresh_token)
+                    return refreshed["access_token"]
+                except RefreshAccessTokenError as exc:
+                    verbose_logger.warning(
+                        "ChatGPT refresh token failed, re-login required: %s", exc
+                    )
+
+        cooldown_remaining = self._get_device_code_cooldown_remaining(auth_data)
+        if cooldown_remaining > 0:
+            token = self._wait_for_access_token(cooldown_remaining)
+            if token:
+                return token
+
+        tokens = self._login_device_code()
+        return tokens["access_token"]
+
+    def get_account_id(self) -> Optional[str]:
+        auth_data = self._read_auth_file()
+        if not auth_data:
+            return None
+        account_id = auth_data.get("account_id")
+        if account_id:
+            return account_id
+        id_token = auth_data.get("id_token")
+        access_token = auth_data.get("access_token")
+        derived = self._extract_account_id(id_token or access_token)
+        if derived:
+            auth_data["account_id"] = derived
+            self._write_auth_file(auth_data)
+        return derived
+
+    def _ensure_token_dir(self) -> None:
+        if not os.path.exists(self.token_dir):
+            os.makedirs(self.token_dir, exist_ok=True)
+
+    def _read_auth_file(self) -> Optional[Dict[str, Any]]:
+        try:
+            with open(self.auth_file, "r") as f:
+                return json.load(f)
+        except IOError:
+            return None
+        except json.JSONDecodeError as exc:
+            verbose_logger.warning("Invalid ChatGPT auth file: %s", exc)
+            return None
+
+    def _write_auth_file(self, data: Dict[str, Any]) -> None:
+        try:
+            with open(self.auth_file, "w") as f:
+                json.dump(data, f)
+        except IOError as exc:
+            verbose_logger.error("Failed to write ChatGPT auth file: %s", exc)
+
+    def _is_token_expired(self, auth_data: Dict[str, Any], access_token: str) -> bool:
+        expires_at = auth_data.get("expires_at")
+        if expires_at is None:
+            expires_at = self._get_expires_at(access_token)
+            if expires_at:
+                auth_data["expires_at"] = expires_at
+                self._write_auth_file(auth_data)
+        if expires_at is None:
+            return True
+        return time.time() >= float(expires_at) - TOKEN_EXPIRY_SKEW_SECONDS
+
+    def _get_expires_at(self, token: str) -> Optional[int]:
+        claims = self._decode_jwt_claims(token)
+        exp = claims.get("exp")
+        if isinstance(exp, (int, float)):
+            return int(exp)
+        return None
+
+    def _decode_jwt_claims(self, token: str) -> Dict[str, Any]:
+        try:
+            parts = token.split(".")
+            if len(parts) < 2:
+                return {}
+            payload_b64 = parts[1]
+            payload_b64 += "=" * (-len(payload_b64) % 4)
+            payload_bytes = base64.urlsafe_b64decode(payload_b64)
+            return json.loads(payload_bytes.decode("utf-8"))
+        except Exception:
+            return {}
+
+    def _extract_account_id(self, token: Optional[str]) -> Optional[str]:
+        if not token:
+            return None
+        claims = self._decode_jwt_claims(token)
+        auth_claims = claims.get("https://api.openai.com/auth")
+        if isinstance(auth_claims, dict):
+            account_id = auth_claims.get("chatgpt_account_id")
+            if isinstance(account_id, str) and account_id:
+                return account_id
+        return None
+
+    def _login_device_code(self) -> Dict[str, str]:
+        cooldown_remaining = self._get_device_code_cooldown_remaining(
+            self._read_auth_file()
+        )
+        if cooldown_remaining > 0:
+            token = self._wait_for_access_token(cooldown_remaining)
+            if token:
+                return {"access_token": token}
+
+        device_code = self._request_device_code()
+        self._record_device_code_request()
+        print(  # noqa: T201
+            "Sign in with ChatGPT using device code:\n"
+            f"1) Visit {CHATGPT_DEVICE_VERIFY_URL}\n"
+            f"2) Enter code: {device_code['user_code']}\n"
+            "Device codes are a common phishing target. Never share this code.",
+            flush=True,
+        )
+        auth_code = self._poll_for_authorization_code(device_code)
+        tokens = self._exchange_code_for_tokens(auth_code)
+        auth_data = self._build_auth_record(tokens)
+        self._write_auth_file(auth_data)
+        return tokens
+
+    def _request_device_code(self) -> Dict[str, str]:
+        try:
+            client = _get_httpx_client()
+            resp = client.post(
+                CHATGPT_DEVICE_CODE_URL,
+                json={"client_id": CHATGPT_CLIENT_ID},
+            )
+            resp.raise_for_status()
+            data = resp.json()
+        except httpx.HTTPStatusError as exc:
+            raise GetDeviceCodeError(
+                message=f"Failed to request device code: {exc}",
+                status_code=exc.response.status_code,
+            )
+        except Exception as exc:
+            raise GetDeviceCodeError(
+                message=f"Failed to request device code: {exc}",
+                status_code=400,
+            )
+
+        device_auth_id = data.get("device_auth_id")
+        user_code = data.get("user_code") or data.get("usercode")
+        interval = data.get("interval")
+        if not device_auth_id or not user_code:
+            raise GetDeviceCodeError(
+                message=f"Device code response missing fields: {data}",
+                status_code=400,
+            )
+        return {
+            "device_auth_id": device_auth_id,
+            "user_code": user_code,
+            "interval": str(interval or "5"),
+        }
+
+    def _poll_for_authorization_code(self, device_code: Dict[str, str]) -> Dict[str, str]:
+        client = _get_httpx_client()
+        interval = int(device_code.get("interval", "5"))
+        start_time = time.time()
+        while time.time() - start_time < DEVICE_CODE_TIMEOUT_SECONDS:
+            try:
+                resp = client.post(
+                    CHATGPT_DEVICE_TOKEN_URL,
+                    json={
+                        "device_auth_id": device_code["device_auth_id"],
+                        "user_code": device_code["user_code"],
+                    },
+                )
+                if resp.status_code == 200:
+                    data = resp.json()
+                    if all(
+                        key in data
+                        for key in (
+                            "authorization_code",
+                            "code_challenge",
+                            "code_verifier",
+                        )
+                    ):
+                        return data
+                if resp.status_code in (403, 404):
+                    time.sleep(max(interval, DEVICE_CODE_POLL_SLEEP_SECONDS))
+                    continue
+                resp.raise_for_status()
+            except httpx.HTTPStatusError as exc:
+                status_code = exc.response.status_code if exc.response else None
+                if status_code in (403, 404):
+                    time.sleep(max(interval, DEVICE_CODE_POLL_SLEEP_SECONDS))
+                    continue
+                raise GetAccessTokenError(
+                    message=f"Polling failed: {exc}",
+                    status_code=exc.response.status_code,
+                )
+            except Exception as exc:
+                raise GetAccessTokenError(
+                    message=f"Polling failed: {exc}",
+                    status_code=400,
+                )
+            time.sleep(max(interval, DEVICE_CODE_POLL_SLEEP_SECONDS))
+
+        raise GetAccessTokenError(
+            message="Timed out waiting for device authorization",
+            status_code=408,
+        )
+
+    def _exchange_code_for_tokens(self, code_data: Dict[str, str]) -> Dict[str, str]:
+        try:
+            client = _get_httpx_client()
+            redirect_uri = f"{CHATGPT_AUTH_BASE}/deviceauth/callback"
+            body = (
+                "grant_type=authorization_code"
+                f"&code={code_data['authorization_code']}"
+                f"&redirect_uri={redirect_uri}"
+                f"&client_id={CHATGPT_CLIENT_ID}"
+                f"&code_verifier={code_data['code_verifier']}"
+            )
+            resp = client.post(
+                CHATGPT_OAUTH_TOKEN_URL,
+                headers={"Content-Type": "application/x-www-form-urlencoded"},
+                content=body,
+            )
+            resp.raise_for_status()
+            data = resp.json()
+        except httpx.HTTPStatusError as exc:
+            raise GetAccessTokenError(
+                message=f"Token exchange failed: {exc}",
+                status_code=exc.response.status_code,
+            )
+        except Exception as exc:
+            raise GetAccessTokenError(
+                message=f"Token exchange failed: {exc}",
+                status_code=400,
+            )
+
+        if not all(key in data for key in ("access_token", "refresh_token", "id_token")):
+            raise GetAccessTokenError(
+                message=f"Token exchange response missing fields: {data}",
+                status_code=400,
+            )
+        return {
+            "access_token": data["access_token"],
+            "refresh_token": data["refresh_token"],
+            "id_token": data["id_token"],
+        }
+
+    def _refresh_tokens(self, refresh_token: str) -> Dict[str, str]:
+        try:
+            client = _get_httpx_client()
+            resp = client.post(
+                CHATGPT_OAUTH_TOKEN_URL,
+                json={
+                    "client_id": CHATGPT_CLIENT_ID,
+                    "grant_type": "refresh_token",
+                    "refresh_token": refresh_token,
+                    "scope": "openid profile email",
+                },
+            )
+            resp.raise_for_status()
+            data = resp.json()
+        except httpx.HTTPStatusError as exc:
+            raise RefreshAccessTokenError(
+                message=f"Refresh token failed: {exc}",
+                status_code=exc.response.status_code,
+            )
+        except Exception as exc:
+            raise RefreshAccessTokenError(
+                message=f"Refresh token failed: {exc}",
+                status_code=400,
+            )
+
+        access_token = data.get("access_token")
+        id_token = data.get("id_token")
+        if not access_token or not id_token:
+            raise RefreshAccessTokenError(
+                message=f"Refresh response missing fields: {data}",
+                status_code=400,
+            )
+
+        refreshed = {
+            "access_token": access_token,
+            "refresh_token": data.get("refresh_token", refresh_token),
+            "id_token": id_token,
+        }
+        auth_data = self._build_auth_record(refreshed)
+        self._write_auth_file(auth_data)
+        return refreshed
+
+    def _build_auth_record(self, tokens: Dict[str, str]) -> Dict[str, Any]:
+        access_token = tokens.get("access_token")
+        id_token = tokens.get("id_token")
+        expires_at = self._get_expires_at(access_token) if access_token else None
+        account_id = self._extract_account_id(id_token or access_token)
+        return {
+            "access_token": access_token,
+            "refresh_token": tokens.get("refresh_token"),
+            "id_token": id_token,
+            "expires_at": expires_at,
+            "account_id": account_id,
+        }
+
+    def _get_device_code_cooldown_remaining(
+        self, auth_data: Optional[Dict[str, Any]]
+    ) -> float:
+        if not auth_data:
+            return 0.0
+        requested_at = auth_data.get("device_code_requested_at")
+        if not isinstance(requested_at, (int, float, str)):
+            return 0.0
+        try:
+            requested_at = float(requested_at)
+        except (TypeError, ValueError):
+            return 0.0
+        elapsed = time.time() - requested_at
+        remaining = DEVICE_CODE_COOLDOWN_SECONDS - elapsed
+        return max(0.0, remaining)
+
+    def _record_device_code_request(self) -> None:
+        auth_data = self._read_auth_file() or {}
+        auth_data["device_code_requested_at"] = time.time()
+        self._write_auth_file(auth_data)
+
+    def _wait_for_access_token(self, timeout_seconds: float) -> Optional[str]:
+        deadline = time.time() + timeout_seconds
+        while time.time() < deadline:
+            auth_data = self._read_auth_file()
+            if auth_data:
+                access_token = auth_data.get("access_token")
+                if access_token and not self._is_token_expired(
+                    auth_data, access_token
+                ):
+                    return access_token
+            sleep_for = min(DEVICE_CODE_POLL_SLEEP_SECONDS, max(0.0, deadline - time.time()))
+            if sleep_for <= 0:
+                break
+            time.sleep(sleep_for)
+        return None

litellm/llms/chatgpt/chat/transformation.py

@@ -0,0 +1,75 @@
+from typing import List, Optional, Tuple
+
+from litellm.exceptions import AuthenticationError
+from litellm.llms.openai.openai import OpenAIConfig
+from litellm.types.llms.openai import AllMessageValues
+
+from ..authenticator import Authenticator
+from ..common_utils import (
+    GetAccessTokenError,
+    ensure_chatgpt_session_id,
+    get_chatgpt_default_headers,
+)
+
+
+class ChatGPTConfig(OpenAIConfig):
+    def __init__(
+        self,
+        api_key: Optional[str] = None,
+        api_base: Optional[str] = None,
+        custom_llm_provider: str = "openai",
+    ) -> None:
+        super().__init__()
+        self.authenticator = Authenticator()
+
+    def _get_openai_compatible_provider_info(
+        self,
+        model: str,
+        api_base: Optional[str],
+        api_key: Optional[str],
+        custom_llm_provider: str,
+    ) -> Tuple[Optional[str], Optional[str], str]:
+        dynamic_api_base = self.authenticator.get_api_base()
+        try:
+            dynamic_api_key = self.authenticator.get_access_token()
+        except GetAccessTokenError as e:
+            raise AuthenticationError(
+                model=model,
+                llm_provider=custom_llm_provider,
+                message=str(e),
+            )
+        return dynamic_api_base, dynamic_api_key, custom_llm_provider
+
+    def validate_environment(
+        self,
+        headers: dict,
+        model: str,
+        messages: List[AllMessageValues],
+        optional_params: dict,
+        litellm_params: dict,
+        api_key: Optional[str] = None,
+        api_base: Optional[str] = None,
+    ) -> dict:
+        validated_headers = super().validate_environment(
+            headers, model, messages, optional_params, litellm_params, api_key, api_base
+        )
+
+        account_id = self.authenticator.get_account_id()
+        session_id = ensure_chatgpt_session_id(litellm_params)
+        default_headers = get_chatgpt_default_headers(
+            api_key or "", account_id, session_id
+        )
+        return {**default_headers, **validated_headers}
+
+    def map_openai_params(
+        self,
+        non_default_params: dict,
+        optional_params: dict,
+        model: str,
+        drop_params: bool,
+    ) -> dict:
+        optional_params = super().map_openai_params(
+            non_default_params, optional_params, model, drop_params
+        )
+        optional_params.setdefault("stream", False)
+        return optional_params

litellm/llms/chatgpt/common_utils.py

@@ -0,0 +1,301 @@
+"""
+Constants and helpers for ChatGPT subscription OAuth.
+"""
+import os
+import platform
+from typing import Any, Optional, Union
+from uuid import uuid4
+
+import httpx
+
+from litellm.llms.base_llm.chat.transformation import BaseLLMException
+
+# OAuth + API constants (derived from openai/codex)
+CHATGPT_AUTH_BASE = "https://auth.openai.com"
+CHATGPT_DEVICE_CODE_URL = f"{CHATGPT_AUTH_BASE}/api/accounts/deviceauth/usercode"
+CHATGPT_DEVICE_TOKEN_URL = f"{CHATGPT_AUTH_BASE}/api/accounts/deviceauth/token"
+CHATGPT_OAUTH_TOKEN_URL = f"{CHATGPT_AUTH_BASE}/oauth/token"
+CHATGPT_DEVICE_VERIFY_URL = f"{CHATGPT_AUTH_BASE}/codex/device"
+CHATGPT_API_BASE = "https://chatgpt.com/backend-api/codex"
+CHATGPT_CLIENT_ID = "app_EMoamEEZ73f0CkXaXp7hrann"
+
+DEFAULT_ORIGINATOR = "codex_cli_rs"
+DEFAULT_USER_AGENT = "codex_cli_rs/0.0.0 (Unknown 0; unknown) unknown"
+CHATGPT_DEFAULT_INSTRUCTIONS = """You are Codex, based on GPT-5. You are running as a coding agent in the Codex CLI on a user's computer.
+
+## General
+
+- When searching for text or files, prefer using `rg` or `rg --files` respectively because `rg` is much faster than alternatives like `grep`. (If the `rg` command is not found, then use alternatives.)
+
+## Editing constraints
+
+- Default to ASCII when editing or creating files. Only introduce non-ASCII or other Unicode characters when there is a clear justification and the file already uses them.
+- Add succinct code comments that explain what is going on if code is not self-explanatory. You should not add comments like "Assigns the value to the variable", but a brief comment might be useful ahead of a complex code block that the user would otherwise have to spend time parsing out. Usage of these comments should be rare.
+- Try to use apply_patch for single file edits, but it is fine to explore other options to make the edit if it does not work well. Do not use apply_patch for changes that are auto-generated (i.e. generating package.json or running a lint or format command like gofmt) or when scripting is more efficient (such as search and replacing a string across a codebase).
+- You may be in a dirty git worktree.
+    * NEVER revert existing changes you did not make unless explicitly requested, since these changes were made by the user.
+    * If asked to make a commit or code edits and there are unrelated changes to your work or changes that you didn't make in those files, don't revert those changes.
+    * If the changes are in files you've touched recently, you should read carefully and understand how you can work with the changes rather than reverting them.
+    * If the changes are in unrelated files, just ignore them and don't revert them.
+- Do not amend a commit unless explicitly requested to do so.
+- While you are working, you might notice unexpected changes that you didn't make. If this happens, STOP IMMEDIATELY and ask the user how they would like to proceed.
+- **NEVER** use destructive commands like `git reset --hard` or `git checkout --` unless specifically requested or approved by the user.
+
+## Plan tool
+
+When using the planning tool:
+- Skip using the planning tool for straightforward tasks (roughly the easiest 25%).
+- Do not make single-step plans.
+- When you made a plan, update it after having performed one of the sub-tasks that you shared on the plan.
+
+## Special user requests
+
+- If the user makes a simple request (such as asking for the time) which you can fulfill by running a terminal command (such as `date`), you should do so.
+- If the user asks for a "review", default to a code review mindset: prioritise identifying bugs, risks, behavioural regressions, and missing tests. Findings must be the primary focus of the response - keep summaries or overviews brief and only after enumerating the issues. Present findings first (ordered by severity with file/line references), follow with open questions or assumptions, and offer a change-summary only as a secondary detail. If no findings are discovered, state that explicitly and mention any residual risks or testing gaps.
+
+## Frontend tasks
+When doing frontend design tasks, avoid collapsing into "AI slop" or safe, average-looking layouts.
+Aim for interfaces that feel intentional, bold, and a bit surprising.
+- Typography: Use expressive, purposeful fonts and avoid default stacks (Inter, Roboto, Arial, system).
+- Color & Look: Choose a clear visual direction; define CSS variables; avoid purple-on-white defaults. No purple bias or dark mode bias.
+- Motion: Use a few meaningful animations (page-load, staggered reveals) instead of generic micro-motions.
+- Background: Don't rely on flat, single-color backgrounds; use gradients, shapes, or subtle patterns to build atmosphere.
+- Overall: Avoid boilerplate layouts and interchangeable UI patterns. Vary themes, type families, and visual languages across outputs.
+- Ensure the page loads properly on both desktop and mobile
+
+Exception: If working within an existing website or design system, preserve the established patterns, structure, and visual language.
+
+## Presenting your work and final message
+
+You are producing plain text that will later be styled by the CLI. Follow these rules exactly. Formatting should make results easy to scan, but not feel mechanical. Use judgment to decide how much structure adds value.
+
+- Default: be very concise; friendly coding teammate tone.
+- Ask only when needed; suggest ideas; mirror the user's style.
+- For substantial work, summarize clearly; follow final-answer formatting.
+- Skip heavy formatting for simple confirmations.
+- Don't dump large files you've written; reference paths only.
+- No "save/copy this file" - User is on the same machine.
+- Offer logical next steps (tests, commits, build) briefly; add verify steps if you couldn't do something.
+- For code changes:
+  * Lead with a quick explanation of the change, and then give more details on the context covering where and why a change was made. Do not start this explanation with "summary", just jump right in.
+  * If there are natural next steps the user may want to take, suggest them at the end of your response. Do not make suggestions if there are no natural next steps.
+  * When suggesting multiple options, use numeric lists for the suggestions so the user can quickly respond with a single number.
+- The user does not command execution outputs. When asked to show the output of a command (e.g. `git show`), relay the important details in your answer or summarize the key lines so the user understands the result.
+
+### Final answer structure and style guidelines
+
+- Plain text; CLI handles styling. Use structure only when it helps scanability.
+- Headers: optional; short Title Case (1-3 words) wrapped in **...**; no blank line before the first bullet; add only if they truly help.
+- Bullets: use - ; merge related points; keep to one line when possible; 4-6 per list ordered by importance; keep phrasing consistent.
+- Monospace: backticks for commands/paths/env vars/code ids and inline examples; use for literal keyword bullets; never combine with **.
+- Code samples or multi-line snippets should be wrapped in fenced code blocks; include an info string as often as possible.
+- Structure: group related bullets; order sections general -> specific -> supporting; for subsections, start with a bolded keyword bullet, then items; match complexity to the task.
+- Tone: collaborative, concise, factual; present tense, active voice; self-contained; no "above/below"; parallel wording.
+- Don'ts: no nested bullets/hierarchies; no ANSI codes; don't cram unrelated keywords; keep keyword lists short--wrap/reformat if long; avoid naming formatting styles in answers.
+- Adaptation: code explanations -> precise, structured with code refs; simple tasks -> lead with outcome; big changes -> logical walkthrough + rationale + next actions; casual one-offs -> plain sentences, no headers/bullets.
+- File References: When referencing files in your response follow the below rules:
+  * Use inline code to make file paths clickable.
+  * Each reference should have a stand alone path. Even if it's the same file.
+  * Accepted: absolute, workspace-relative, a/ or b/ diff prefixes, or bare filename/suffix.
+  * Optionally include line/column (1-based): :line[:column] or #Lline[Ccolumn] (column defaults to 1).
+  * Do not use URIs like file://, vscode://, or https://.
+  * Do not provide range of lines
+  * Examples: src/app.ts, src/app.ts:42, b/server/index.js#L10, C:\\repo\\project\\main.rs:12:5
+"""
+
+
+class ChatGPTAuthError(BaseLLMException):
+    def __init__(
+        self,
+        status_code,
+        message,
+        request: Optional[httpx.Request] = None,
+        response: Optional[httpx.Response] = None,
+        headers: Optional[Union[httpx.Headers, dict]] = None,
+        body: Optional[dict] = None,
+    ):
+        super().__init__(
+            status_code=status_code,
+            message=message,
+            request=request,
+            response=response,
+            headers=headers,
+            body=body,
+        )
+
+
+class GetDeviceCodeError(ChatGPTAuthError):
+    pass
+
+
+class GetAccessTokenError(ChatGPTAuthError):
+    pass
+
+
+class RefreshAccessTokenError(ChatGPTAuthError):
+    pass
+
+
+def _safe_header_value(value: str) -> str:
+    if not value:
+        return ""
+    return "".join(ch if 32 <= ord(ch) <= 126 else "_" for ch in value)
+
+
+def _sanitize_user_agent_token(value: str) -> str:
+    if not value:
+        return ""
+    return "".join(
+        ch if (ch.isalnum() or ch in "-_./") else "_" for ch in value
+    )
+
+
+def _terminal_user_agent() -> str:
+    term_program = os.getenv("TERM_PROGRAM")
+    if term_program:
+        version = os.getenv("TERM_PROGRAM_VERSION")
+        token = f"{term_program}/{version}" if version else term_program
+        return _sanitize_user_agent_token(token) or "unknown"
+
+    wezterm_version = os.getenv("WEZTERM_VERSION")
+    if wezterm_version is not None:
+        token = (
+            f"WezTerm/{wezterm_version}" if wezterm_version else "WezTerm"
+        )
+        return _sanitize_user_agent_token(token) or "WezTerm"
+
+    if (
+        os.getenv("ITERM_SESSION_ID")
+        or os.getenv("ITERM_PROFILE")
+        or os.getenv("ITERM_PROFILE_NAME")
+    ):
+        return "iTerm.app"
+
+    if os.getenv("TERM_SESSION_ID"):
+        return "Apple_Terminal"
+
+    if os.getenv("KITTY_WINDOW_ID") or "kitty" in (os.getenv("TERM") or ""):
+        return "kitty"
+
+    if os.getenv("ALACRITTY_SOCKET") or os.getenv("TERM") == "alacritty":
+        return "Alacritty"
+
+    konsole_version = os.getenv("KONSOLE_VERSION")
+    if konsole_version is not None:
+        token = (
+            f"Konsole/{konsole_version}" if konsole_version else "Konsole"
+        )
+        return _sanitize_user_agent_token(token) or "Konsole"
+
+    if os.getenv("GNOME_TERMINAL_SCREEN"):
+        return "gnome-terminal"
+
+    vte_version = os.getenv("VTE_VERSION")
+    if vte_version is not None:
+        token = f"VTE/{vte_version}" if vte_version else "VTE"
+        return _sanitize_user_agent_token(token) or "VTE"
+
+    if os.getenv("WT_SESSION"):
+        return "WindowsTerminal"
+
+    term = os.getenv("TERM")
+    if term:
+        return _sanitize_user_agent_token(term) or "unknown"
+
+    return "unknown"
+
+
+def _get_litellm_version() -> str:
+    try:
+        from importlib.metadata import version
+
+        return version("litellm")
+    except Exception:
+        return "0.0.0"
+
+
+def get_chatgpt_originator() -> str:
+    originator = os.getenv("CHATGPT_ORIGINATOR") or DEFAULT_ORIGINATOR
+    return _safe_header_value(originator) or DEFAULT_ORIGINATOR
+
+
+def get_chatgpt_user_agent(originator: str) -> str:
+    override = os.getenv("CHATGPT_USER_AGENT")
+    if override:
+        return _safe_header_value(override) or DEFAULT_USER_AGENT
+    version = _get_litellm_version()
+    os_type = platform.system() or "Unknown"
+    os_version = platform.release() or "0"
+    arch = platform.machine() or "unknown"
+    terminal_ua = _terminal_user_agent()
+    suffix = os.getenv("CHATGPT_USER_AGENT_SUFFIX", "").strip()
+    suffix = f" ({suffix})" if suffix else ""
+    candidate = (
+        f"{originator}/{version} ({os_type} {os_version}; {arch}) {terminal_ua}{suffix}"
+    )
+    return _safe_header_value(candidate) or DEFAULT_USER_AGENT
+
+
+def get_chatgpt_default_headers(
+    access_token: str,
+    account_id: Optional[str],
+    session_id: Optional[str] = None,
+) -> dict:
+    originator = get_chatgpt_originator()
+    user_agent = get_chatgpt_user_agent(originator)
+    headers = {
+        "Authorization": f"Bearer {access_token}",
+        "content-type": "application/json",
+        "accept": "text/event-stream",
+        "originator": originator,
+        "user-agent": user_agent,
+    }
+    if session_id:
+        headers["session_id"] = session_id
+    if account_id:
+        headers["ChatGPT-Account-Id"] = account_id
+    return headers
+
+
+def get_chatgpt_default_instructions() -> str:
+    return os.getenv("CHATGPT_DEFAULT_INSTRUCTIONS") or CHATGPT_DEFAULT_INSTRUCTIONS
+
+
+def _normalize_litellm_params(litellm_params: Optional[Any]) -> dict:
+    if litellm_params is None:
+        return {}
+    if isinstance(litellm_params, dict):
+        return litellm_params
+    if hasattr(litellm_params, "model_dump"):
+        try:
+            return litellm_params.model_dump()
+        except Exception:
+            return {}
+    if hasattr(litellm_params, "dict"):
+        try:
+            return litellm_params.dict()
+        except Exception:
+            return {}
+    return {}
+
+
+def get_chatgpt_session_id(litellm_params: Optional[Any]) -> Optional[str]:
+    params = _normalize_litellm_params(litellm_params)
+    for key in ("litellm_session_id", "session_id"):
+        value = params.get(key)
+        if value:
+            return str(value)
+    metadata = params.get("metadata")
+    if isinstance(metadata, dict):
+        value = metadata.get("session_id")
+        if value:
+            return str(value)
+    for key in ("litellm_trace_id", "litellm_call_id"):
+        value = params.get(key)
+        if value:
+            return str(value)
+    return None
+
+
+def ensure_chatgpt_session_id(litellm_params: Optional[Any]) -> str:
+    return get_chatgpt_session_id(litellm_params) or str(uuid4())

litellm/llms/chatgpt/responses/transformation.py

@@ -0,0 +1,191 @@
+import json
+from typing import Any, Optional
+
+from litellm.exceptions import AuthenticationError
+from litellm.constants import STREAM_SSE_DONE_STRING
+from litellm.litellm_core_utils.core_helpers import process_response_headers
+from litellm.llms.openai.common_utils import OpenAIError
+from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig
+from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import (
+    _safe_convert_created_field,
+)
+from litellm.types.llms.openai import (
+    ResponsesAPIResponse,
+    ResponsesAPIStreamEvents,
+)
+from litellm.types.router import GenericLiteLLMParams
+from litellm.types.utils import LlmProviders
+from litellm.utils import CustomStreamWrapper
+
+from ..authenticator import Authenticator
+from ..common_utils import (
+    CHATGPT_API_BASE,
+    GetAccessTokenError,
+    ensure_chatgpt_session_id,
+    get_chatgpt_default_headers,
+    get_chatgpt_default_instructions,
+)
+
+
+class ChatGPTResponsesAPIConfig(OpenAIResponsesAPIConfig):
+    def __init__(self) -> None:
+        super().__init__()
+        self.authenticator = Authenticator()
+
+    @property
+    def custom_llm_provider(self) -> LlmProviders:
+        return LlmProviders.CHATGPT
+
+    def validate_environment(
+        self,
+        headers: dict,
+        model: str,
+        litellm_params: Optional[GenericLiteLLMParams],
+    ) -> dict:
+        try:
+            access_token = self.authenticator.get_access_token()
+        except GetAccessTokenError as e:
+            raise AuthenticationError(
+                model=model,
+                llm_provider="chatgpt",
+                message=str(e),
+            )
+
+        account_id = self.authenticator.get_account_id()
+        session_id = ensure_chatgpt_session_id(litellm_params)
+        default_headers = get_chatgpt_default_headers(
+            access_token, account_id, session_id
+        )
+        return {**default_headers, **headers}
+
+    def transform_responses_api_request(
+        self,
+        model: str,
+        input: Any,
+        response_api_optional_request_params: dict,
+        litellm_params: GenericLiteLLMParams,
+        headers: dict,
+    ) -> dict:
+        request = super().transform_responses_api_request(
+            model,
+            input,
+            response_api_optional_request_params,
+            litellm_params,
+            headers,
+        )
+        request.pop("max_output_tokens", None)
+        request.pop("max_tokens", None)
+        request.pop("max_completion_tokens", None)
+        request.pop("metadata", None)
+        base_instructions = get_chatgpt_default_instructions()
+        existing_instructions = request.get("instructions")
+        if existing_instructions:
+            if base_instructions not in existing_instructions:
+                request["instructions"] = (
+                    f"{base_instructions}\n\n{existing_instructions}"
+                )
+        else:
+            request["instructions"] = base_instructions
+        request["store"] = False
+        request["stream"] = True
+        include = list(request.get("include") or [])
+        if "reasoning.encrypted_content" not in include:
+            include.append("reasoning.encrypted_content")
+        request["include"] = include
+        return request
+
+    def transform_response_api_response(
+        self,
+        model: str,
+        raw_response: Any,
+        logging_obj: Any,
+    ):
+        content_type = (raw_response.headers or {}).get("content-type", "")
+        body_text = raw_response.text or ""
+        if "text/event-stream" not in content_type.lower():
+            trimmed_body = body_text.lstrip()
+            if not (
+                trimmed_body.startswith("event:")
+                or trimmed_body.startswith("data:")
+                or "\nevent:" in body_text
+                or "\ndata:" in body_text
+            ):
+                return super().transform_response_api_response(
+                    model=model,
+                    raw_response=raw_response,
+                    logging_obj=logging_obj,
+                )
+
+        logging_obj.post_call(
+            original_response=raw_response.text,
+            additional_args={"complete_input_dict": {}},
+        )
+
+        completed_response = None
+        error_message = None
+        for chunk in body_text.splitlines():
+            stripped_chunk = CustomStreamWrapper._strip_sse_data_from_chunk(chunk)
+            if not stripped_chunk:
+                continue
+            stripped_chunk = stripped_chunk.strip()
+            if not stripped_chunk:
+                continue
+            if stripped_chunk == STREAM_SSE_DONE_STRING:
+                break
+            try:
+                parsed_chunk = json.loads(stripped_chunk)
+            except json.JSONDecodeError:
+                continue
+            if not isinstance(parsed_chunk, dict):
+                continue
+            event_type = parsed_chunk.get("type")
+            if event_type == ResponsesAPIStreamEvents.RESPONSE_COMPLETED:
+                response_payload = parsed_chunk.get("response")
+                if isinstance(response_payload, dict):
+                    response_payload = dict(response_payload)
+                    if "created_at" in response_payload:
+                        response_payload["created_at"] = _safe_convert_created_field(
+                            response_payload["created_at"]
+                        )
+                    try:
+                        completed_response = ResponsesAPIResponse(**response_payload)
+                    except Exception:
+                        completed_response = ResponsesAPIResponse.model_construct(
+                            **response_payload
+                        )
+                break
+            if event_type in (
+                ResponsesAPIStreamEvents.RESPONSE_FAILED,
+                ResponsesAPIStreamEvents.ERROR,
+            ):
+                error_obj = parsed_chunk.get("error") or (
+                    parsed_chunk.get("response") or {}
+                ).get("error")
+                if error_obj is not None:
+                    if isinstance(error_obj, dict):
+                        error_message = error_obj.get("message") or str(error_obj)
+                    else:
+                        error_message = str(error_obj)
+
+        if completed_response is None:
+            raise OpenAIError(
+                message=error_message or raw_response.text,
+                status_code=raw_response.status_code,
+            )
+
+        raw_headers = dict(raw_response.headers)
+        processed_headers = process_response_headers(raw_headers)
+        if not hasattr(completed_response, "_hidden_params"):
+            setattr(completed_response, "_hidden_params", {})
+        completed_response._hidden_params["additional_headers"] = processed_headers
+        completed_response._hidden_params["headers"] = raw_headers
+        return completed_response
+
+    def get_complete_url(
+        self,
+        api_base: Optional[str],
+        litellm_params: dict,
+    ) -> str:
+        api_base = api_base or self.authenticator.get_api_base() or CHATGPT_API_BASE
+        api_base = api_base.rstrip("/")
+        return f"{api_base}/responses"

litellm/llms/openai/realtime/handler.py

@@ -56,7 +56,9 @@ class OpenAIRealtime(OpenAIChatCompletion):
         url = self._construct_url(api_base, query_params)
 
         try:
-            ssl_context = get_shared_realtime_ssl_context()
+            # Only use SSL context for secure websocket connections (wss://)
+            # websockets library doesn't accept ssl argument for ws:// URIs
+            ssl_context = None if url.startswith("ws://") else get_shared_realtime_ssl_context()
             # Log a masked request preview consistent with other endpoints.
             logging_obj.pre_call(
                 input=None,

litellm/llms/vertex_ai/common_utils.py

@@ -150,6 +150,34 @@ def get_supports_response_schema(
     return _supports_response_schema
 
 
+def supports_response_json_schema(model: str) -> bool:
+    """
+    Check if the model supports responseJsonSchema (JSON Schema format).
+
+    responseJsonSchema is supported by Gemini 2.0+ models and uses standard
+    JSON Schema format with lowercase types (string, object, etc.) instead of
+    the OpenAPI-style responseSchema with uppercase types (STRING, OBJECT, etc.).
+
+    Benefits of responseJsonSchema:
+    - Supports additionalProperties for stricter schema validation
+    - Uses standard JSON Schema format (no type conversion needed)
+    - Better compatibility with Pydantic's model_json_schema()
+
+    Args:
+        model: The model name (e.g., "gemini-2.0-flash", "gemini-2.5-pro")
+
+    Returns:
+        True if the model supports responseJsonSchema, False otherwise
+    """
+    model_lower = model.lower()
+
+    # Gemini 2.0+ and 2.5+ models support responseJsonSchema
+    # Pattern matches: gemini-2.0-*, gemini-2.5-*, gemini-3-*, etc.
+    gemini_2_plus_pattern = re.compile(r"gemini-([2-9]|[1-9]\d+)\.")
+
+    return bool(gemini_2_plus_pattern.search(model_lower))
+
+
 from typing import Literal, Optional
 
 all_gemini_url_modes = Literal[
@@ -487,6 +515,44 @@ def _build_vertex_schema(parameters: dict, add_property_ordering: bool = False):
     return parameters
 
 
+def _build_json_schema(parameters: dict) -> dict:
+    """
+    Build a JSON Schema for use with Gemini's responseJsonSchema parameter.
+
+    Unlike _build_vertex_schema (used for responseSchema), this function:
+    - Does NOT convert types to uppercase (keeps standard JSON Schema format)
+    - Does NOT add propertyOrdering
+    - Does NOT filter fields (allows additionalProperties)
+    - Still unpacks $defs/$ref (Gemini doesn't support JSON Schema references)
+
+    Parameters:
+        parameters: dict - the JSON schema to process
+
+    Returns:
+        dict - the processed schema in standard JSON Schema format
+    """
+    # Unpack $defs references (Gemini doesn't support $ref)
+    defs = parameters.pop("$defs", {})
+    for name, value in defs.items():
+        unpack_defs(value, defs)
+    unpack_defs(parameters, defs)
+
+    # Convert anyOf with null to nullable
+    convert_anyof_null_to_nullable(parameters)
+
+    # Handle empty strings in enum values - Gemini doesn't accept empty strings in enums
+    _fix_enum_empty_strings(parameters)
+
+    # Remove enums for non-string typed fields (Gemini requires enum only on strings)
+    _fix_enum_types(parameters)
+
+    # Handle empty items objects
+    process_items(parameters)
+    add_object_type(parameters)
+
+    return parameters
+
+
 def _filter_anyof_fields(schema_dict: Dict[str, Any]) -> Dict[str, Any]:
     """
     When anyof is present, only keep the anyof field and its contents - otherwise VertexAI will throw an error - https://github.com/BerriAI/litellm/issues/11164

litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py

@@ -92,7 +92,12 @@ from litellm.utils import (
 )
 
 from ....utils import _remove_additional_properties, _remove_strict_from_schema
-from ..common_utils import VertexAIError, _build_vertex_schema
+from ..common_utils import (
+    VertexAIError,
+    _build_json_schema,
+    _build_vertex_schema,
+    supports_response_json_schema,
+)
 from ..vertex_llm_base import VertexBase
 from .transformation import (
     _gemini_convert_messages_with_history,
@@ -624,30 +629,55 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
             )
         return old_schema
 
-    def apply_response_schema_transformation(self, value: dict, optional_params: dict):
+    def apply_response_schema_transformation(
+        self, value: dict, optional_params: dict, model: str
+    ):
         new_value = deepcopy(value)
-        # remove 'additionalProperties' from json schema
-        new_value = _remove_additional_properties(new_value)
-        # remove 'strict' from json schema
+        # remove 'strict' from json schema (not supported by Gemini)
         new_value = _remove_strict_from_schema(new_value)
-        if new_value["type"] == "json_object":
+
+        # Automatically use responseJsonSchema for Gemini 2.0+ models
+        # responseJsonSchema uses standard JSON Schema format and supports additionalProperties
+        # For older models (Gemini 1.5), fall back to responseSchema (OpenAPI format)
+        use_json_schema = supports_response_json_schema(model)
+
+        if not use_json_schema:
+            # For responseSchema, remove 'additionalProperties' (not supported)
+            new_value = _remove_additional_properties(new_value)
+
+        # Handle response type
+        if new_value.get("type") == "json_object":
             optional_params["response_mime_type"] = "application/json"
-        elif new_value["type"] == "text":
+        elif new_value.get("type") == "text":
             optional_params["response_mime_type"] = "text/plain"
+
+        # Extract schema from response_format
+        schema = None
         if "response_schema" in new_value:
             optional_params["response_mime_type"] = "application/json"
-            optional_params["response_schema"] = new_value["response_schema"]
-        elif new_value["type"] == "json_schema":  # type: ignore
-            if "json_schema" in new_value and "schema" in new_value["json_schema"]:  # type: ignore
+            schema = new_value["response_schema"]
+        elif new_value.get("type") == "json_schema":
+            if "json_schema" in new_value and "schema" in new_value["json_schema"]:
                 optional_params["response_mime_type"] = "application/json"
-                optional_params["response_schema"] = new_value["json_schema"]["schema"]  # type: ignore
+                schema = new_value["json_schema"]["schema"]
 
-        if "response_schema" in optional_params and isinstance(
-            optional_params["response_schema"], dict
-        ):
-            optional_params["response_schema"] = self._map_response_schema(
-                value=optional_params["response_schema"]
-            )
+        if schema and isinstance(schema, dict):
+            if use_json_schema:
+                # Use responseJsonSchema (Gemini 2.0+ only, opt-in)
+                # - Standard JSON Schema format (lowercase types)
+                # - Supports additionalProperties
+                # - No propertyOrdering needed
+                optional_params["response_json_schema"] = _build_json_schema(
+                    deepcopy(schema)
+                )
+            else:
+                # Use responseSchema (default, backwards compatible)
+                # - OpenAPI-style format (uppercase types)
+                # - No additionalProperties support
+                # - Requires propertyOrdering
+                optional_params["response_schema"] = self._map_response_schema(
+                    value=schema
+                )
 
     @staticmethod
     def _map_reasoning_effort_to_thinking_budget(
@@ -947,7 +977,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
                 optional_params["max_output_tokens"] = value
             elif param == "response_format" and isinstance(value, dict):  # type: ignore
                 self.apply_response_schema_transformation(
-                    value=value, optional_params=optional_params
+                    value=value, optional_params=optional_params, model=model
                 )
             elif param == "frequency_penalty":
                 if self._supports_penalty_parameters(model):

litellm/llms/vertex_ai/multimodal_embeddings/transformation.py

@@ -265,7 +265,7 @@ class VertexAIMultimodalEmbeddingConfig(BaseEmbeddingConfig):
                 image_count += 1
 
         ## Calculate video embeddings usage
-        video_length_seconds = 0
+        video_length_seconds = 0.0
         for prediction in vertex_predictions["predictions"]:
             video_embeddings = prediction.get("videoEmbeddings")
             if video_embeddings:

litellm/llms/vertex_ai/vertex_llm_base.py

@@ -23,6 +23,11 @@ from .common_utils import (
     is_global_only_vertex_model,
 )
 
+GOOGLE_IMPORT_ERROR_MESSAGE = (
+    "Google Cloud SDK not found. Install it with: pip install 'litellm[google]' "
+    "or pip install google-cloud-aiplatform"
+)
+
 if TYPE_CHECKING:
     from google.auth.credentials import Credentials as GoogleCredentialsObject
 else:
@@ -138,7 +143,10 @@ class VertexBase:
 
     # Google Auth Helpers -- extracted for mocking purposes in tests
     def _credentials_from_identity_pool(self, json_obj, scopes):
-        from google.auth import identity_pool
+        try:
+            from google.auth import identity_pool
+        except ImportError:
+            raise ImportError(GOOGLE_IMPORT_ERROR_MESSAGE)
 
         creds = identity_pool.Credentials.from_info(json_obj)
         if scopes and hasattr(creds, "requires_scopes") and creds.requires_scopes:
@@ -146,7 +154,10 @@ class VertexBase:
         return creds
 
     def _credentials_from_identity_pool_with_aws(self, json_obj, scopes):
-        from google.auth import aws
+        try:
+            from google.auth import aws
+        except ImportError:
+            raise ImportError(GOOGLE_IMPORT_ERROR_MESSAGE)
 
         creds = aws.Credentials.from_info(json_obj)
         if scopes and hasattr(creds, "requires_scopes") and creds.requires_scopes:
@@ -154,22 +165,30 @@ class VertexBase:
         return creds
 
     def _credentials_from_authorized_user(self, json_obj, scopes):
-        import google.oauth2.credentials
+        try:
+            import google.oauth2.credentials
+        except ImportError:
+            raise ImportError(GOOGLE_IMPORT_ERROR_MESSAGE)
 
         return google.oauth2.credentials.Credentials.from_authorized_user_info(
             json_obj, scopes=scopes
         )
 
     def _credentials_from_service_account(self, json_obj, scopes):
-        import google.oauth2.service_account
+        try:
+            import google.oauth2.service_account
+        except ImportError:
+            raise ImportError(GOOGLE_IMPORT_ERROR_MESSAGE)
 
         return google.oauth2.service_account.Credentials.from_service_account_info(
             json_obj, scopes=scopes
         )
 
     def _credentials_from_default_auth(self, scopes):
-
-        import google.auth as google_auth
+        try:
+            import google.auth as google_auth
+        except ImportError:
+            raise ImportError(GOOGLE_IMPORT_ERROR_MESSAGE)
 
         return google_auth.default(scopes=scopes)
 
@@ -261,9 +280,12 @@ class VertexBase:
         return api_base
 
     def refresh_auth(self, credentials: Any) -> None:
-        from google.auth.transport.requests import (
-            Request,  # type: ignore[import-untyped]
-        )
+        try:
+            from google.auth.transport.requests import (
+                Request,  # type: ignore[import-untyped]
+            )
+        except ImportError:
+            raise ImportError(GOOGLE_IMPORT_ERROR_MESSAGE)
 
         credentials.refresh(Request())
 

litellm/model_prices_and_context_window_backup.json

@@ -13465,6 +13465,31 @@
         "supports_vision": true,
         "supports_web_search": true
     },
+    "gemini-2.5-computer-use-preview-10-2025": {
+        "input_cost_per_token": 1.25e-06,
+        "input_cost_per_token_above_200k_tokens": 2.5e-06,
+        "litellm_provider": "vertex_ai-language-models",
+        "max_images_per_prompt": 3000,
+        "max_input_tokens": 128000,
+        "max_output_tokens": 64000,
+        "max_tokens": 64000,
+        "mode": "chat",
+        "output_cost_per_token": 1e-05,
+        "output_cost_per_token_above_200k_tokens": 1.5e-05,
+        "source": "https://cloud.google.com/vertex-ai/generative-ai/docs/computer-use",
+        "supported_modalities": [
+            "text",
+            "image"
+        ],
+        "supported_output_modalities": [
+            "text"
+        ],
+        "supports_computer_use": true,
+        "supports_function_calling": true,
+        "supports_system_messages": true,
+        "supports_tool_choice": true,
+        "supports_vision": true
+    },
     "gemini-embedding-001": {
         "input_cost_per_token": 1.5e-07,
         "litellm_provider": "vertex_ai-embedding-models",
@@ -15950,6 +15975,63 @@
         "max_tokens": 8191,
         "mode": "embedding"
     },
+    "chatgpt/gpt-5.2-codex": {
+        "litellm_provider": "chatgpt",
+        "max_input_tokens": 128000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "mode": "responses",
+        "supported_endpoints": [
+            "/v1/responses"
+        ],
+        "supports_function_calling": true,
+        "supports_parallel_function_calling": true,
+        "supports_response_schema": true,
+        "supports_vision": true
+    },
+    "chatgpt/gpt-5.2": {
+        "litellm_provider": "chatgpt",
+        "max_input_tokens": 128000,
+        "max_output_tokens": 64000,
+        "max_tokens": 64000,
+        "mode": "responses",
+        "supported_endpoints": [
+            "/v1/chat/completions",
+            "/v1/responses"
+        ],
+        "supports_function_calling": true,
+        "supports_parallel_function_calling": true,
+        "supports_response_schema": true,
+        "supports_vision": true
+    },
+    "chatgpt/gpt-5.1-codex-max": {
+        "litellm_provider": "chatgpt",
+        "max_input_tokens": 128000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "mode": "responses",
+        "supported_endpoints": [
+            "/v1/responses"
+        ],
+        "supports_function_calling": true,
+        "supports_parallel_function_calling": true,
+        "supports_response_schema": true,
+        "supports_vision": true
+    },
+    "chatgpt/gpt-5.1-codex-mini": {
+        "litellm_provider": "chatgpt",
+        "max_input_tokens": 128000,
+        "max_output_tokens": 64000,
+        "max_tokens": 64000,
+        "mode": "responses",
+        "supported_endpoints": [
+            "/v1/responses"
+        ],
+        "supports_function_calling": true,
+        "supports_parallel_function_calling": true,
+        "supports_response_schema": true,
+        "supports_vision": true
+    },
     "gigachat/GigaChat-2-Lite": {
         "input_cost_per_token": 0.0,
         "litellm_provider": "gigachat",

litellm/proxy/_types.py

@@ -360,7 +360,6 @@ class LiteLLMRoutes(enum.Enum):
         # OCR
         "/ocr",
         "/v1/ocr",
-
         # containers API
         "/containers",
         "/v1/containers",
@@ -421,6 +420,7 @@ class LiteLLMRoutes(enum.Enum):
         "/a2a/{agent_id}",
         "/a2a/{agent_id}/message/send",
         "/a2a/{agent_id}/message/stream",
+        "/a2a/{agent_id}/.well-known/agent-card.json",
     ]
 
     google_routes = [
@@ -836,9 +836,9 @@ class GenerateRequestBase(LiteLLMPydanticObjectBase):
     allowed_cache_controls: Optional[list] = []
     config: Optional[dict] = {}
     permissions: Optional[dict] = {}
-    model_max_budget: Optional[dict] = (
-        {}
-    )  # {"gpt-4": 5.0, "gpt-3.5-turbo": 5.0}, defaults to {}
+    model_max_budget: Optional[
+        dict
+    ] = {}  # {"gpt-4": 5.0, "gpt-3.5-turbo": 5.0}, defaults to {}
 
     model_config = ConfigDict(protected_namespaces=())
     model_rpm_limit: Optional[dict] = None
@@ -1372,12 +1372,12 @@ class NewCustomerRequest(BudgetNewRequest):
     blocked: bool = False  # allow/disallow requests for this end-user
     budget_id: Optional[str] = None  # give either a budget_id or max_budget
     spend: Optional[float] = None
-    allowed_model_region: Optional[AllowedModelRegion] = (
-        None  # require all user requests to use models in this specific region
-    )
-    default_model: Optional[str] = (
-        None  # if no equivalent model in allowed region - default all requests to this model
-    )
+    allowed_model_region: Optional[
+        AllowedModelRegion
+    ] = None  # require all user requests to use models in this specific region
+    default_model: Optional[
+        str
+    ] = None  # if no equivalent model in allowed region - default all requests to this model
 
     @model_validator(mode="before")
     @classmethod
@@ -1399,12 +1399,12 @@ class UpdateCustomerRequest(LiteLLMPydanticObjectBase):
     blocked: bool = False  # allow/disallow requests for this end-user
     max_budget: Optional[float] = None
     budget_id: Optional[str] = None  # give either a budget_id or max_budget
-    allowed_model_region: Optional[AllowedModelRegion] = (
-        None  # require all user requests to use models in this specific region
-    )
-    default_model: Optional[str] = (
-        None  # if no equivalent model in allowed region - default all requests to this model
-    )
+    allowed_model_region: Optional[
+        AllowedModelRegion
+    ] = None  # require all user requests to use models in this specific region
+    default_model: Optional[
+        str
+    ] = None  # if no equivalent model in allowed region - default all requests to this model
 
 
 class DeleteCustomerRequest(LiteLLMPydanticObjectBase):
@@ -1490,15 +1490,15 @@ class NewTeamRequest(TeamBase):
     ] = None  # raise an error if 'guaranteed_throughput' is set and we're overallocating tpm
 
     model_tpm_limit: Optional[Dict[str, int]] = None
-    team_member_budget: Optional[float] = (
-        None  # allow user to set a budget for all team members
-    )
-    team_member_rpm_limit: Optional[int] = (
-        None  # allow user to set RPM limit for all team members
-    )
-    team_member_tpm_limit: Optional[int] = (
-        None  # allow user to set TPM limit for all team members
-    )
+    team_member_budget: Optional[
+        float
+    ] = None  # allow user to set a budget for all team members
+    team_member_rpm_limit: Optional[
+        int
+    ] = None  # allow user to set RPM limit for all team members
+    team_member_tpm_limit: Optional[
+        int
+    ] = None  # allow user to set TPM limit for all team members
     team_member_key_duration: Optional[str] = None  # e.g. "1d", "1w", "1m"
     allowed_vector_store_indexes: Optional[List[AllowedVectorStoreIndexItem]] = None
 
@@ -1586,9 +1586,9 @@ class BlockKeyRequest(LiteLLMPydanticObjectBase):
 
 class AddTeamCallback(LiteLLMPydanticObjectBase):
     callback_name: str
-    callback_type: Optional[Literal["success", "failure", "success_and_failure"]] = (
-        "success_and_failure"
-    )
+    callback_type: Optional[
+        Literal["success", "failure", "success_and_failure"]
+    ] = "success_and_failure"
     callback_vars: Dict[str, str]
 
     @model_validator(mode="before")
@@ -1916,9 +1916,9 @@ class ConfigList(LiteLLMPydanticObjectBase):
     stored_in_db: Optional[bool]
     field_default_value: Any
     premium_field: bool = False
-    nested_fields: Optional[List[FieldDetail]] = (
-        None  # For nested dictionary or Pydantic fields
-    )
+    nested_fields: Optional[
+        List[FieldDetail]
+    ] = None  # For nested dictionary or Pydantic fields
 
 
 class UserHeaderMapping(LiteLLMPydanticObjectBase):
@@ -2127,7 +2127,9 @@ class LiteLLM_VerificationToken(LiteLLMPydanticObjectBase):
     rotation_interval: Optional[str] = None  # How often to rotate (e.g., "30d", "90d")
     last_rotation_at: Optional[datetime] = None  # When this key was last rotated
     key_rotation_at: Optional[datetime] = None  # When this key should next be rotated
-    router_settings: Optional[Dict] = None  # Router settings for this key (Key > Team > Global precedence)
+    router_settings: Optional[
+        Dict
+    ] = None  # Router settings for this key (Key > Team > Global precedence)
 
     model_config = ConfigDict(protected_namespaces=())
 
@@ -2332,9 +2334,9 @@ class LiteLLM_OrganizationMembershipTable(LiteLLMPydanticObjectBase):
     budget_id: Optional[str] = None
     created_at: datetime
     updated_at: datetime
-    user: Optional[Any] = (
-        None  # You might want to replace 'Any' with a more specific type if available
-    )
+    user: Optional[
+        Any
+    ] = None  # You might want to replace 'Any' with a more specific type if available
     litellm_budget_table: Optional[LiteLLM_BudgetTable] = None
 
     model_config = ConfigDict(protected_namespaces=())
@@ -3306,9 +3308,9 @@ class TeamModelDeleteRequest(BaseModel):
 # Organization Member Requests
 class OrganizationMemberAddRequest(OrgMemberAddRequest):
     organization_id: str
-    max_budget_in_organization: Optional[float] = (
-        None  # Users max budget within the organization
-    )
+    max_budget_in_organization: Optional[
+        float
+    ] = None  # Users max budget within the organization
 
 
 class OrganizationMemberDeleteRequest(MemberDeleteRequest):
@@ -3523,9 +3525,9 @@ class ProviderBudgetResponse(LiteLLMPydanticObjectBase):
     Maps provider names to their budget configs.
     """
 
-    providers: Dict[str, ProviderBudgetResponseObject] = (
-        {}
-    )  # Dictionary mapping provider names to their budget configurations
+    providers: Dict[
+        str, ProviderBudgetResponseObject
+    ] = {}  # Dictionary mapping provider names to their budget configurations
 
 
 class ProxyStateVariables(TypedDict):
@@ -3668,9 +3670,9 @@ class LiteLLM_JWTAuth(LiteLLMPydanticObjectBase):
     enforce_rbac: bool = False
     roles_jwt_field: Optional[str] = None  # v2 on role mappings
     role_mappings: Optional[List[RoleMapping]] = None
-    object_id_jwt_field: Optional[str] = (
-        None  # can be either user / team, inferred from the role mapping
-    )
+    object_id_jwt_field: Optional[
+        str
+    ] = None  # can be either user / team, inferred from the role mapping
     scope_mappings: Optional[List[ScopeMapping]] = None
     enforce_scope_based_access: bool = False
     enforce_team_based_model_access: bool = False

litellm/proxy/auth/auth_utils.py

@@ -311,6 +311,88 @@ def get_request_route(request: Request) -> str:
         return request.url.path
 
 
+def normalize_request_route(route: str) -> str:
+    """
+    Normalize request routes by replacing dynamic path parameters with placeholders.
+    
+    This prevents high cardinality in Prometheus metrics by collapsing routes like:
+    - /v1/responses/1234567890 -> /v1/responses/{response_id}
+    - /v1/threads/thread_123 -> /v1/threads/{thread_id}
+    
+    Args:
+        route: The request route path
+        
+    Returns:
+        Normalized route with dynamic parameters replaced by placeholders
+        
+    Examples:
+        >>> normalize_request_route("/v1/responses/abc123")
+        '/v1/responses/{response_id}'
+        >>> normalize_request_route("/v1/responses/abc123/cancel")
+        '/v1/responses/{response_id}/cancel'
+        >>> normalize_request_route("/chat/completions")
+        '/chat/completions'
+    """
+    # Define patterns for routes with dynamic IDs
+    # Format: (regex_pattern, replacement_template)
+    patterns = [
+        # Responses API - must come before generic patterns
+        (r'^(/(?:openai/)?v1/responses)/([^/]+)(/input_items)$', r'\1/{response_id}\3'),
+        (r'^(/(?:openai/)?v1/responses)/([^/]+)(/cancel)$', r'\1/{response_id}\3'),
+        (r'^(/(?:openai/)?v1/responses)/([^/]+)$', r'\1/{response_id}'),
+        (r'^(/responses)/([^/]+)(/input_items)$', r'\1/{response_id}\3'),
+        (r'^(/responses)/([^/]+)(/cancel)$', r'\1/{response_id}\3'),
+        (r'^(/responses)/([^/]+)$', r'\1/{response_id}'),
+        
+        # Threads API
+        (r'^(/(?:openai/)?v1/threads)/([^/]+)(/runs)/([^/]+)(/steps)/([^/]+)$', r'\1/{thread_id}\3/{run_id}\5/{step_id}'),
+        (r'^(/(?:openai/)?v1/threads)/([^/]+)(/runs)/([^/]+)(/steps)$', r'\1/{thread_id}\3/{run_id}\5'),
+        (r'^(/(?:openai/)?v1/threads)/([^/]+)(/runs)/([^/]+)(/cancel)$', r'\1/{thread_id}\3/{run_id}\5'),
+        (r'^(/(?:openai/)?v1/threads)/([^/]+)(/runs)/([^/]+)(/submit_tool_outputs)$', r'\1/{thread_id}\3/{run_id}\5'),
+        (r'^(/(?:openai/)?v1/threads)/([^/]+)(/runs)/([^/]+)$', r'\1/{thread_id}\3/{run_id}'),
+        (r'^(/(?:openai/)?v1/threads)/([^/]+)(/runs)$', r'\1/{thread_id}\3'),
+        (r'^(/(?:openai/)?v1/threads)/([^/]+)(/messages)/([^/]+)$', r'\1/{thread_id}\3/{message_id}'),
+        (r'^(/(?:openai/)?v1/threads)/([^/]+)(/messages)$', r'\1/{thread_id}\3'),
+        (r'^(/(?:openai/)?v1/threads)/([^/]+)$', r'\1/{thread_id}'),
+        
+        # Vector Stores API
+        (r'^(/(?:openai/)?v1/vector_stores)/([^/]+)(/files)/([^/]+)$', r'\1/{vector_store_id}\3/{file_id}'),
+        (r'^(/(?:openai/)?v1/vector_stores)/([^/]+)(/files)$', r'\1/{vector_store_id}\3'),
+        (r'^(/(?:openai/)?v1/vector_stores)/([^/]+)(/file_batches)/([^/]+)$', r'\1/{vector_store_id}\3/{batch_id}'),
+        (r'^(/(?:openai/)?v1/vector_stores)/([^/]+)(/file_batches)$', r'\1/{vector_store_id}\3'),
+        (r'^(/(?:openai/)?v1/vector_stores)/([^/]+)$', r'\1/{vector_store_id}'),
+        
+        # Assistants API
+        (r'^(/(?:openai/)?v1/assistants)/([^/]+)$', r'\1/{assistant_id}'),
+        
+        # Files API
+        (r'^(/(?:openai/)?v1/files)/([^/]+)(/content)$', r'\1/{file_id}\3'),
+        (r'^(/(?:openai/)?v1/files)/([^/]+)$', r'\1/{file_id}'),
+        
+        # Batches API
+        (r'^(/(?:openai/)?v1/batches)/([^/]+)(/cancel)$', r'\1/{batch_id}\3'),
+        (r'^(/(?:openai/)?v1/batches)/([^/]+)$', r'\1/{batch_id}'),
+        
+        # Fine-tuning API
+        (r'^(/(?:openai/)?v1/fine_tuning/jobs)/([^/]+)(/events)$', r'\1/{fine_tuning_job_id}\3'),
+        (r'^(/(?:openai/)?v1/fine_tuning/jobs)/([^/]+)(/cancel)$', r'\1/{fine_tuning_job_id}\3'),
+        (r'^(/(?:openai/)?v1/fine_tuning/jobs)/([^/]+)(/checkpoints)$', r'\1/{fine_tuning_job_id}\3'),
+        (r'^(/(?:openai/)?v1/fine_tuning/jobs)/([^/]+)$', r'\1/{fine_tuning_job_id}'),
+        
+        # Models API
+        (r'^(/(?:openai/)?v1/models)/([^/]+)$', r'\1/{model}'),
+    ]
+    
+    # Apply patterns in order
+    for pattern, replacement in patterns:
+        normalized = re.sub(pattern, replacement, route)
+        if normalized != route:
+            return normalized
+    
+    # Return original route if no pattern matched
+    return route
+
+
 async def check_if_request_size_is_safe(request: Request) -> bool:
     """
     Enterprise Only:

litellm/proxy/auth/user_api_key_auth.py

@@ -28,8 +28,8 @@ from litellm.proxy.auth.auth_checks import (
     _delete_cache_key_object,
     _get_user_role,
     _is_user_proxy_admin,
-    _virtual_key_max_budget_check,
     _virtual_key_max_budget_alert_check,
+    _virtual_key_max_budget_check,
     _virtual_key_soft_budget_check,
     can_key_call_model,
     common_checks,
@@ -45,6 +45,7 @@ from litellm.proxy.auth.auth_utils import (
     get_end_user_id_from_request_body,
     get_model_from_request,
     get_request_route,
+    normalize_request_route,
     pre_db_read_auth_checks,
     route_in_additonal_public_routes,
 )
@@ -1261,7 +1262,7 @@ async def user_api_key_auth(
     if end_user_id is not None:
         user_api_key_auth_obj.end_user_id = end_user_id
 
-    user_api_key_auth_obj.request_route = route
+    user_api_key_auth_obj.request_route = normalize_request_route(route)
 
     return user_api_key_auth_obj
 

litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py

@@ -893,15 +893,53 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM):
                 return
 
         #########################################################
-        ########## 1. Make parallel Bedrock API requests ##########
+        ########## 1. Make Bedrock API requests ##########
         #########################################################
+        # Import asyncio for parallel execution
+        import asyncio
+
+        # Determine if INPUT validation is needed in post_call
+        # Skip INPUT validation if pre_call or during_call is already enabled
+        # (to avoid redundant validation - those hooks would have already validated INPUT)
+        should_validate_input = not (
+            self._event_hook_is_event_type(GuardrailEventHooks.pre_call)
+            or self._event_hook_is_event_type(GuardrailEventHooks.during_call)
+        )
+
         output_content_bedrock: Optional[Union[BedrockGuardrailResponse, str]] = None
-        try:
-            output_content_bedrock = await self.make_bedrock_api_request(
+
+        if should_validate_input:
+            # Prepare input messages (with optional filtering for latest role message)
+            input_filter = self._prepare_guardrail_messages_for_role(
+                messages=new_messages
+            )
+            input_messages = input_filter.payload_messages or new_messages
+
+            # Create tasks for parallel execution of both INPUT and OUTPUT validation
+            input_task = self.make_bedrock_api_request(
+                source="INPUT",
+                messages=input_messages,
+                request_data=data,
+            )
+            output_task = self.make_bedrock_api_request(
                 source="OUTPUT", response=response, request_data=data
             )
-        except GuardrailInterventionNormalStringError as e:
-            output_content_bedrock = e.message
+
+            # Execute both requests in parallel
+            try:
+                _, output_content_bedrock = await asyncio.gather(
+                    input_task, output_task
+                )
+            except GuardrailInterventionNormalStringError as e:
+                output_content_bedrock = e.message
+        else:
+            # Only run OUTPUT validation (INPUT was already validated in pre_call or during_call)
+            try:
+                output_content_bedrock = await self.make_bedrock_api_request(
+                    source="OUTPUT", response=response, request_data=data
+                )
+            except GuardrailInterventionNormalStringError as e:
+                output_content_bedrock = e.message
 
         #########################################################
         ########## 2. Apply masking to response with output guardrail response ##########
@@ -910,7 +948,7 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM):
             response = self.create_guardrail_blocked_response(
                 response=output_content_bedrock
             )
-        else:
+        elif output_content_bedrock is not None:
             self._apply_masking_to_response(
                 response=response,
                 bedrock_guardrail_response=output_content_bedrock,
@@ -997,36 +1035,54 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM):
         )
         if isinstance(assembled_model_response, ModelResponse):
             ####################################################################
-            ########## 1. Make parallel Bedrock Apply Guardrail API requests ##########
+            ########## 1. Make Bedrock Apply Guardrail API requests ##########
 
             # Bedrock will raise an exception if this violates the guardrail policy
             ###################################################################
-            # Create tasks for parallel execution
-            input_filter = self._prepare_guardrail_messages_for_role(
-                messages=request_data.get("messages")
+            # Determine if INPUT validation is needed in post_call
+            # Skip INPUT validation if pre_call or during_call is already enabled
+            # (to avoid redundant validation - those hooks would have already validated INPUT)
+            should_validate_input = not (
+                self._event_hook_is_event_type(GuardrailEventHooks.pre_call)
+                or self._event_hook_is_event_type(GuardrailEventHooks.during_call)
             )
-            input_messages = input_filter.payload_messages or request_data.get(
-                "messages"
-            )
-            input_task = self.make_bedrock_api_request(
-                source="INPUT",
-                messages=input_messages,
-                request_data=request_data,
-            )  # Only input messages
+
             output_guardrail_response: Optional[
                 Union[BedrockGuardrailResponse, str]
             ] = None
-            output_task = self.make_bedrock_api_request(
-                source="OUTPUT", response=assembled_model_response
-            )  # Only response
 
-            # Execute both requests in parallel
-            try:
-                _, output_guardrail_response = await asyncio.gather(
-                    input_task, output_task
+            if should_validate_input:
+                # Create tasks for parallel execution
+                input_filter = self._prepare_guardrail_messages_for_role(
+                    messages=request_data.get("messages")
                 )
-            except GuardrailInterventionNormalStringError as e:
-                output_guardrail_response = e.message
+                input_messages = input_filter.payload_messages or request_data.get(
+                    "messages"
+                )
+                input_task = self.make_bedrock_api_request(
+                    source="INPUT",
+                    messages=input_messages,
+                    request_data=request_data,
+                )  # Only input messages
+                output_task = self.make_bedrock_api_request(
+                    source="OUTPUT", response=assembled_model_response
+                )  # Only response
+
+                # Execute both requests in parallel
+                try:
+                    _, output_guardrail_response = await asyncio.gather(
+                        input_task, output_task
+                    )
+                except GuardrailInterventionNormalStringError as e:
+                    output_guardrail_response = e.message
+            else:
+                # Only run OUTPUT validation (INPUT was already validated in pre_call or during_call)
+                try:
+                    output_guardrail_response = await self.make_bedrock_api_request(
+                        source="OUTPUT", response=assembled_model_response
+                    )
+                except GuardrailInterventionNormalStringError as e:
+                    output_guardrail_response = e.message
 
             #########################################################################
             ########## 2. Apply masking to response with output guardrail response ##########

litellm/proxy/proxy_config.yaml

@@ -1,51 +1,27 @@
 model_list:
-  - model_name: gemini/*
+  # Anthropic direct
+  - model_name: anthropic-claude
     litellm_params:
-      model: gemini/*
-  - model_name: -claude-sonnet-4-5-20250929
-    litellm_params:
-      model: bedrock/invoke/us.anthropic.claude-sonnet-4-5-20250929-v1:0
-    model_info:
-      cache_creation_input_token_cost: 3.75e-06
-      cache_read_input_token_cost: 3e-07
-      input_cost_per_token: 3e-06
-      input_cost_per_token_above_200k_tokens: 6e-06
-      output_cost_per_token_above_200k_tokens: 2.25e-05
-      cache_creation_input_token_cost_above_200k_tokens: 7.5e-06
-      cache_read_input_token_cost_above_200k_tokens: 6e-07
-      litellm_provider: bedrock_converse
-      max_input_tokens: 200000
-      max_output_tokens: 64000
-      max_tokens: 200000
-      mode: chat
-      output_cost_per_token: 1.5e-05
-      search_context_cost_per_query:
-        search_context_size_high: 0.01
-        search_context_size_low: 0.01
-        search_context_size_medium: 0.01
-      supports_assistant_prefill: true
-      supports_computer_use: true
-      supports_function_calling: true
-      supports_pdf_input: true
-      supports_prompt_caching: true
-      supports_reasoning: true
-      supports_response_schema: true
-      supports_tool_choice: true
-      supports_vision: true
-      tool_use_system_prompt_tokens: 346
+      model: anthropic/claude-sonnet-4-20250514
+      api_key: os.environ/ANTHROPIC_API_KEY
 
-  - model_name: us.anthropic.claude-sonnet-4-20250514-v1:0
+  # Azure AI Anthropic
+  - model_name: azure-ai-claude
     litellm_params:
-      model: bedrock/converse/us.anthropic.claude-sonnet-4-20250514-v1:0
-    model_info:
-      litellm_provider: bedrock_converse
-      mode: chat
-  - model_name: claude-sonnet-4-5-20250929
-    litellm_params:
-      model: azure_ai/claude-opus-4-5
-      api_base: https://krish-mh44t553-eastus2.services.ai.azure.com
+      model: azure_ai/claude-3-5-sonnet
+      api_base: https://krish-mh44t553-eastus2.services.ai.azure.com/
       api_key: os.environ/AZURE_ANTHROPIC_API_KEY
 
+  # Azure AI Anthropic (alternate endpoint format)
+  - model_name: claude-4.5-haiku
+    litellm_params:
+      model: anthropic/claude-haiku-4-5
+      api_base: https://krish-mh44t553-eastus2.services.ai.azure.com/anthropic/v1/messages
+      api_version: "2023-06-01"
+      api_key: os.environ/AZURE_ANTHROPIC_API_KEY
+
+
+
 # Search Tools Configuration - Define search providers for WebSearch interception
 # search_tools:
 #   - search_tool_name: "my-perplexity-search"

litellm/proxy/utils.py

@@ -131,7 +131,6 @@ else:
 
 unified_guardrail = UnifiedLLMGuardrails()
 
-_anthropic_async_clients = {}
 
 def print_verbose(print_statement):
     """
@@ -961,8 +960,8 @@ class ProxyLogging:
         Returns:
             Updated data dictionary if guardrail passes, None if guardrail should be skipped
         """
-        from litellm.types.guardrails import GuardrailEventHooks
         from litellm.integrations.prometheus import PrometheusLogger
+        from litellm.types.guardrails import GuardrailEventHooks
 
         # Determine the event type based on call type
         event_type = GuardrailEventHooks.pre_call
@@ -2214,6 +2213,45 @@ class PrismaClient:
 
             raise e
 
+    async def _query_first_with_cached_plan_fallback(
+        self, sql_query: str
+    ) -> Optional[dict]:
+        """
+        Execute a query with automatic fallback for PostgreSQL cached plan errors.
+        
+        This handles the "cached plan must not change result type" error that occurs
+        during rolling deployments when schema changes are applied while old pods
+        still have cached query plans expecting the old schema.
+        
+        Args:
+            sql_query: SQL query string to execute
+            
+        Returns:
+            Query result or None
+            
+        Raises:
+            Original exception if not a cached plan error
+        """
+        try:
+            return await self.db.query_first(query=sql_query)
+        except Exception as e:
+            error_str = str(e)
+            if "cached plan must not change result type" in error_str:
+                # Force PostgreSQL to re-plan by invalidating the cache
+                # Add a unique comment to make the query different
+                sql_query_retry = sql_query.replace(
+                    "SELECT",
+                    f"SELECT /* cache_invalidated_{int(time.time() * 1000)} */"
+                )
+                verbose_proxy_logger.warning(
+                    "PostgreSQL cached plan error detected for token lookup, "
+                    "retrying with fresh plan. This may occur during rolling deployments "
+                    "when schema changes are applied."
+                )
+                return await self.db.query_first(query=sql_query_retry)
+            else:
+                raise
+
     @backoff.on_exception(
         backoff.expo,
         Exception,  # base exception to catch for the backoff
@@ -2545,7 +2583,7 @@ class PrismaClient:
                         WHERE v.token = '{token}'
                     """
 
-                    response = await self.db.query_first(query=sql_query)
+                    response = await self._query_first_with_cached_plan_fallback(sql_query)
 
                     if response is not None:
                         if response["team_models"] is None:
@@ -4253,74 +4291,6 @@ def construct_database_url_from_env_vars() -> Optional[str]:
     return None
 
 
-async def count_tokens_with_anthropic_api(
-    model_to_use: str,
-    messages: Optional[List[Dict[str, Any]]],
-    deployment: Optional[Dict[str, Any]] = None,
-) -> Optional[Dict[str, Any]]:
-    """
-    Helper function to count tokens using Anthropic API directly.
-
-    Args:
-        model_to_use: The model name to use for token counting
-        messages: The messages to count tokens for
-        deployment: Optional deployment configuration containing API key
-
-    Returns:
-        Optional dict with token count and tokenizer info, or None if failed
-    """
-    if not messages:
-        return None
-
-    try:
-        import os
-
-        import anthropic
-
-        # Get Anthropic API key from deployment config
-        anthropic_api_key = None
-        if deployment is not None:
-            anthropic_api_key = deployment.get("litellm_params", {}).get("api_key")
-
-        # Fallback to environment variable
-        if not anthropic_api_key:
-            anthropic_api_key = os.getenv("ANTHROPIC_API_KEY")
-
-        if anthropic_api_key and messages:
-            # Call Anthropic API directly for more accurate token counting
-            
-            # Use cached client if available to avoid socket exhaustion
-            if anthropic_api_key not in _anthropic_async_clients:
-                _anthropic_async_clients[anthropic_api_key] = anthropic.AsyncAnthropic(api_key=anthropic_api_key)
-            
-            client = _anthropic_async_clients[anthropic_api_key]
-
-            # Call with explicit parameters to satisfy type checking
-            # Type ignore for now since messages come from generic dict input
-            response = await client.beta.messages.count_tokens(
-                model=model_to_use,
-                messages=messages,  # type: ignore
-                betas=["token-counting-2024-11-01"],
-            )
-            total_tokens = response.input_tokens
-            tokenizer_used = "anthropic_api"
-
-            return {
-                "total_tokens": total_tokens,
-                "tokenizer_used": tokenizer_used,
-            }
-
-    except ImportError:
-        verbose_proxy_logger.warning(
-            "Anthropic library not available, falling back to LiteLLM tokenizer"
-        )
-    except Exception as e:
-        verbose_proxy_logger.warning(
-            f"Error calling Anthropic API: {e}, falling back to LiteLLM tokenizer"
-        )
-    return None
-
-
 async def get_available_models_for_user(
     user_api_key_dict: "UserAPIKeyAuth",
     llm_router: Optional["Router"],

litellm/responses/litellm_completion_transformation/streaming_iterator.py

@@ -81,6 +81,8 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
             Union[ModelResponse, TextCompletionResponse]
         ] = None
         self.final_text: str = ""
+        self._cached_item_id: Optional[str] = None
+        self._cached_response_id: Optional[str] = None
         self._pending_tool_events: List[BaseLiteLLMOpenAIResponseObject] = []
         self._tool_output_index_by_call_id: dict[str, int] = {}
         self._tool_args_by_call_id: dict[str, str] = {}
@@ -307,12 +309,15 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
         )
 
     def create_output_item_added_event(self) -> OutputItemAddedEvent:
+        if self._cached_item_id is None:
+            self._cached_item_id = f"msg_{str(uuid.uuid4())}"
+        
         return OutputItemAddedEvent(
             type=ResponsesAPIStreamEvents.OUTPUT_ITEM_ADDED,
             output_index=0,
             item=BaseLiteLLMOpenAIResponseObject(
                 **{
-                    "id": f"msg_{str(uuid.uuid4())}",
+                    "id": self._cached_item_id,
                     "type": "message",
                     "status": "in_progress",
                     "role": "assistant",
@@ -322,9 +327,12 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
         )
 
     def create_content_part_added_event(self) -> ContentPartAddedEvent:
+        if self._cached_item_id is None:
+            self._cached_item_id = f"msg_{str(uuid.uuid4())}"
+        
         return ContentPartAddedEvent(
             type=ResponsesAPIStreamEvents.CONTENT_PART_ADDED,
-            item_id=f"msg_{str(uuid.uuid4())}",
+            item_id=self._cached_item_id,
             output_index=0,
             content_index=0,
             part=BaseLiteLLMOpenAIResponseObject(
@@ -346,9 +354,12 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
     def create_output_text_done_event(
         self, litellm_complete_object: ModelResponse
     ) -> OutputTextDoneEvent:
+        if self._cached_item_id is None:
+            self._cached_item_id = f"msg_{str(uuid.uuid4())}"
+        
         return OutputTextDoneEvent(
             type=ResponsesAPIStreamEvents.OUTPUT_TEXT_DONE,
-            item_id=f"msg_{str(uuid.uuid4())}",
+            item_id=self._cached_item_id,
             output_index=0,
             content_index=0,
             text=getattr(litellm_complete_object.choices[0].message, "content", "")  # type: ignore
@@ -358,6 +369,8 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
     def create_output_content_part_done_event(
         self, litellm_complete_object: ModelResponse
     ) -> ContentPartDoneEvent:
+        if self._cached_item_id is None:
+            self._cached_item_id = f"msg_{str(uuid.uuid4())}"
 
         text = getattr(litellm_complete_object.choices[0].message, "content", "") or ""  # type: ignore
         reasoning_content = getattr(litellm_complete_object.choices[0].message, "reasoning_content", "") or ""  # type: ignore
@@ -383,7 +396,7 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
 
         return ContentPartDoneEvent(
             type=ResponsesAPIStreamEvents.CONTENT_PART_DONE,
-            item_id=f"msg_{str(uuid.uuid4())}",
+            item_id=self._cached_item_id,
             output_index=0,
             content_index=0,
             part=part,
@@ -392,6 +405,9 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
     def create_output_item_done_event(
         self, litellm_complete_object: ModelResponse
     ) -> OutputItemDoneEvent:
+        if self._cached_item_id is None:
+            self._cached_item_id = f"msg_{str(uuid.uuid4())}"
+        
         text = self.litellm_model_response.choices[0].message.content or ""  # type: ignore
         annotations = getattr(self.litellm_model_response.choices[0].message, "annotations", None)  # type: ignore
 
@@ -404,7 +420,7 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
             sequence_number=1,
             item=BaseLiteLLMOpenAIResponseObject(
                 **{
-                    "id": f"msg_{str(uuid.uuid4())}",
+                    "id": self._cached_item_id,
                     "status": "completed",
                     "type": "message",
                     "role": "assistant",
@@ -576,6 +592,10 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
         and the ReasoningSummaryTextDeltaEvent, which is used by the responses API to emit reasoning content.
         It also handles emitting annotation.added events when annotations are detected in the chunk.
         """
+        if self._cached_item_id is None and chunk.id:
+            self._cached_item_id = chunk.id
+        item_id = self._cached_item_id or chunk.id
+        
         # Check if this chunk has annotations first (before processing text/reasoning)
         # This ensures we detect and queue annotation events from the annotation chunk
         if chunk.choices and hasattr(chunk.choices[0].delta, "annotations"):
@@ -593,7 +613,7 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
                         annotation_dict = annotation.model_dump() if hasattr(annotation, 'model_dump') else dict(annotation)
                         event = OutputTextAnnotationAddedEvent(
                             type=ResponsesAPIStreamEvents.OUTPUT_TEXT_ANNOTATION_ADDED,
-                            item_id=chunk.id,
+                            item_id=item_id,
                             output_index=0,
                             content_index=0,
                             annotation_index=idx,
@@ -620,7 +640,7 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
         if delta_content:
             return OutputTextDeltaEvent(
                 type=ResponsesAPIStreamEvents.OUTPUT_TEXT_DELTA,
-                item_id=chunk.id,
+                item_id=item_id,
                 output_index=0,
                 content_index=0,
                 delta=delta_content,

litellm/router.py

@@ -4681,26 +4681,35 @@ class Router:
                 parent_otel_span=parent_otel_span,
             )
 
-            # raises an exception if this error should not be retries
-            self.should_retry_this_error(
-                error=e,
-                healthy_deployments=_healthy_deployments,
-                all_deployments=_all_deployments,
-                context_window_fallbacks=context_window_fallbacks,
-                regular_fallbacks=fallbacks,
-                content_policy_fallbacks=content_policy_fallbacks,
-            )
-
+            # Check retry policy FIRST, before should_retry_this_error
+            # This allows retry policies to override the healthy deployments check
+            _retry_policy_applies = False
             if (
                 self.retry_policy is not None
                 or self.model_group_retry_policy is not None
             ):
                 # get num_retries from retry policy
+                # Use the model_group captured at the start of the function, or get it from metadata
+                # kwargs.get("model") at this point is the deployment model, not the model_group
+                _model_group_for_retry_policy = model_group or _metadata.get("model_group") or kwargs.get("model")
                 _retry_policy_retries = self.get_num_retries_from_retry_policy(
-                    exception=original_exception, model_group=kwargs.get("model")
+                    exception=original_exception, model_group=_model_group_for_retry_policy
                 )
                 if _retry_policy_retries is not None:
                     num_retries = _retry_policy_retries
+                    _retry_policy_applies = True
+
+            # raises an exception if this error should not be retries
+            # Skip this check if retry policy applies (retry policy takes precedence)
+            if not _retry_policy_applies:
+                self.should_retry_this_error(
+                    error=e,
+                    healthy_deployments=_healthy_deployments,
+                    all_deployments=_all_deployments,
+                    context_window_fallbacks=context_window_fallbacks,
+                    regular_fallbacks=fallbacks,
+                    content_policy_fallbacks=content_policy_fallbacks,
+                )
             ## LOGGING
             if num_retries > 0:
                 kwargs = self.log_retry(kwargs=kwargs, e=original_exception)
@@ -4865,6 +4874,12 @@ class Router:
         ):
             raise error
 
+        status_code = getattr(error, "status_code", None)
+        if status_code is not None and not litellm._should_retry(status_code):
+            # 401/403 are special cases - allow retry if multiple deployments exist (handled below)
+            if status_code not in (401, 403):
+                raise error
+
         if isinstance(error, litellm.NotFoundError):
             raise error
         # Error we should only retry if there are other deployments

litellm/types/llms/anthropic.py

@@ -642,4 +642,8 @@ ANTHROPIC_TOOL_SEARCH_BETA_HEADER = "advanced-tool-use-2025-11-20"
 # Effort beta header constant
 ANTHROPIC_EFFORT_BETA_HEADER = "effort-2025-11-24"
 
+# OAuth constants
+ANTHROPIC_OAUTH_TOKEN_PREFIX = "sk-ant-oat"
+ANTHROPIC_OAUTH_BETA_HEADER = "oauth-2025-04-20"
+
 

litellm/types/llms/vertex_ai.py

@@ -207,6 +207,7 @@ class GenerationConfig(TypedDict, total=False):
     frequency_penalty: float
     response_mime_type: Literal["text/plain", "application/json"]
     response_schema: dict
+    response_json_schema: dict
     seed: int
     responseLogprobs: bool
     logprobs: int

litellm/types/utils.py

@@ -63,6 +63,19 @@ def _generate_id():  # private helper function
     return "chatcmpl-" + str(uuid.uuid4())
 
 
+
+class SafeAttributeModel:
+    """
+    A base model that provides safe attribute access.
+    """
+    def __delattr__(self, name):
+        try:
+            super().__delattr__(name)
+        except AttributeError:
+            # noop if attribute does not exist
+            pass
+
+
 class LiteLLMCommonStrings(Enum):
     redacted_by_litellm = "redacted by litellm. 'litellm.turn_off_message_logging=True'"
     llm_provider_not_provided = "Unmapped LLM provider for this endpoint. You passed model={model}, custom_llm_provider={custom_llm_provider}. Check supported provider and route: https://docs.litellm.ai/docs/providers"
@@ -1020,7 +1033,7 @@ def add_provider_specific_fields(
     setattr(object, "provider_specific_fields", provider_specific_fields)
 
 
-class Message(OpenAIObject):
+class Message(SafeAttributeModel, OpenAIObject):
     content: Optional[str]
     role: Literal["assistant", "user", "system", "tool", "function"]
     tool_calls: Optional[List[ChatCompletionMessageToolCall]]
@@ -1140,7 +1153,7 @@ class Message(OpenAIObject):
             return self.dict()
 
 
-class Delta(OpenAIObject):
+class Delta(SafeAttributeModel, OpenAIObject):
     reasoning_content: Optional[str] = None
     thinking_blocks: Optional[
         List[Union[ChatCompletionThinkingBlock, ChatCompletionRedactedThinkingBlock]]
@@ -1237,7 +1250,7 @@ class Delta(OpenAIObject):
         setattr(self, key, value)
 
 
-class Choices(OpenAIObject):
+class Choices(SafeAttributeModel, OpenAIObject):
     finish_reason: str
     index: int
     message: Message
@@ -1330,6 +1343,7 @@ class CacheCreationTokenDetails(BaseModel):
 
 
 class PromptTokensDetailsWrapper(
+    SafeAttributeModel,
     PromptTokensDetails
 ):  # extends with image generation fields (text_tokens, image_tokens)
     text_tokens: Optional[int] = None
@@ -1377,7 +1391,7 @@ class ServerToolUse(BaseModel):
     tool_search_requests: Optional[int] = None
 
 
-class Usage(CompletionUsage):
+class Usage(SafeAttributeModel, CompletionUsage):
     _cache_creation_input_tokens: int = PrivateAttr(
         0
     )  # hidden param for prompt caching. Might change, once openai introduces their equivalent.
@@ -2958,6 +2972,7 @@ GenericBudgetConfigType = Dict[str, BudgetConfig]
 
 class LlmProviders(str, Enum):
     OPENAI = "openai"
+    CHATGPT = "chatgpt"
     OPENAI_LIKE = "openai_like"  # embedding only
     JINA_AI = "jina_ai"
     XAI = "xai"

litellm/utils.py

@@ -87,6 +87,7 @@ def _get_cached_custom_logger():
     global _CustomLogger
     if _CustomLogger is None:
         from litellm.integrations.custom_logger import CustomLogger
+
         _CustomLogger = CustomLogger
     return _CustomLogger
 
@@ -100,6 +101,7 @@ def _get_cached_custom_guardrail():
     global _CustomGuardrail
     if _CustomGuardrail is None:
         from litellm.integrations.custom_guardrail import CustomGuardrail
+
         _CustomGuardrail = CustomGuardrail
     return _CustomGuardrail
 
@@ -113,6 +115,7 @@ def _get_cached_caching_handler_response():
     global _CachingHandlerResponse
     if _CachingHandlerResponse is None:
         from litellm.caching.caching_handler import CachingHandlerResponse
+
         _CachingHandlerResponse = CachingHandlerResponse
     return _CachingHandlerResponse
 
@@ -126,6 +129,7 @@ def _get_cached_llm_caching_handler():
     global _LLMCachingHandler
     if _LLMCachingHandler is None:
         from litellm.caching.caching_handler import LLMCachingHandler
+
         _LLMCachingHandler = LLMCachingHandler
     return _LLMCachingHandler
 
@@ -144,9 +148,11 @@ def _get_cached_audio_utils():
     global _audio_utils_module
     if _audio_utils_module is None:
         import litellm.litellm_core_utils.audio_utils.utils
+
         _audio_utils_module = litellm.litellm_core_utils.audio_utils.utils
     return _audio_utils_module
 
+
 from litellm.types.llms.openai import (
     AllMessageValues,
     AllPromptValues,
@@ -203,10 +209,6 @@ from litellm.types.utils import (
 #  Thank you users! We ❤️ you! - Krrish & Ishaan
 
 
-
-
-
-
 try:
     # Python 3.9+
     with resources.files("litellm.litellm_core_utils.tokenizers").joinpath(
@@ -250,10 +252,14 @@ from litellm.llms.base_llm.base_utils import (
 if TYPE_CHECKING:
     # Heavy types that are only needed for type checking; avoid importing
     # their modules at runtime during `litellm` import.
-    from litellm.caching.caching_handler import CachingHandlerResponse, LLMCachingHandler
+    from litellm.caching.caching_handler import (
+        CachingHandlerResponse,
+        LLMCachingHandler,
+    )
     from litellm.integrations.custom_logger import CustomLogger
     from litellm.llms.base_llm.files.transformation import BaseFilesConfig
     from litellm.proxy._types import AllowedModelRegion
+
     # Type stubs for lazy-loaded functions to help mypy understand their types
     # These imports allow mypy to understand the types when these are accessed via __getattr__
     from litellm.litellm_core_utils.exception_mapping_utils import exception_type
@@ -288,10 +294,13 @@ if TYPE_CHECKING:
     )
     from litellm.llms.base_llm.ocr.transformation import BaseOCRConfig
     from litellm.llms.base_llm.search.transformation import BaseSearchConfig
-    from litellm.llms.base_llm.text_to_speech.transformation import BaseTextToSpeechConfig
+    from litellm.llms.base_llm.text_to_speech.transformation import (
+        BaseTextToSpeechConfig,
+    )
     from litellm.llms.bedrock.common_utils import BedrockModelInfo
     from litellm.llms.cohere.common_utils import CohereModelInfo
     from litellm.llms.mistral.ocr.transformation import MistralOCRConfig
+
     # Type stubs for lazy-loaded functions and classes
     from litellm.litellm_core_utils.cached_imports import (
         get_coroutine_checker,
@@ -335,6 +344,7 @@ if TYPE_CHECKING:
         reset_retry_policy,
     )
     from litellm.secret_managers.main import get_secret
+
     # Type stubs for lazy-loaded config classes and types
     from litellm.llms.base_llm.batches.transformation import BaseBatchesConfig
     from litellm.llms.base_llm.containers.transformation import BaseContainerConfig
@@ -618,7 +628,7 @@ def load_credentials_from_list(kwargs: dict):
     Updates kwargs with the credentials if credential_name in kwarg
     """
     # Access CredentialAccessor via module to trigger lazy loading if needed
-    CredentialAccessor = getattr(sys.modules[__name__], 'CredentialAccessor')
+    CredentialAccessor = getattr(sys.modules[__name__], "CredentialAccessor")
 
     credential_name = kwargs.get("litellm_credential_name")
     if credential_name and litellm.credential_list:
@@ -699,9 +709,7 @@ def _process_tool_message_id(msg_copy: dict, thought_signature_separator: str) -
     """
     Process tool message to remove thought signature from tool_call_id.
     """
-    if msg_copy.get("role") == "tool" and isinstance(
-        msg_copy.get("tool_call_id"), str
-    ):
+    if msg_copy.get("role") == "tool" and isinstance(msg_copy.get("tool_call_id"), str):
         if thought_signature_separator in msg_copy["tool_call_id"]:
             msg_copy["tool_call_id"] = _remove_thought_signature_from_id(
                 msg_copy["tool_call_id"], thought_signature_separator
@@ -763,12 +771,12 @@ def function_setup(  # noqa: PLR0915
         function_id: Optional[str] = kwargs["id"] if "id" in kwargs else None
 
         ## LAZY LOAD COROUTINE CHECKER ##
-        get_coroutine_checker = getattr(sys.modules[__name__], 'get_coroutine_checker')
+        get_coroutine_checker = getattr(sys.modules[__name__], "get_coroutine_checker")
 
         ## DYNAMIC CALLBACKS ##
-        dynamic_callbacks: Optional[List[Union[str, Callable, "CustomLogger"]]] = (
-            kwargs.pop("callbacks", None)
-        )
+        dynamic_callbacks: Optional[
+            List[Union[str, Callable, "CustomLogger"]]
+        ] = kwargs.pop("callbacks", None)
         all_callbacks = get_dynamic_callbacks(dynamic_callbacks=dynamic_callbacks)
 
         if len(all_callbacks) > 0:
@@ -811,7 +819,7 @@ def function_setup(  # noqa: PLR0915
                     + litellm.failure_callback
                 )
             )
-            get_set_callbacks = getattr(sys.modules[__name__], 'get_set_callbacks')
+            get_set_callbacks = getattr(sys.modules[__name__], "get_set_callbacks")
             get_set_callbacks()(callback_list=callback_list, function_id=function_id)
         ## ASYNC CALLBACKS
         if len(litellm.input_callback) > 0:
@@ -939,7 +947,7 @@ def function_setup(  # noqa: PLR0915
             elif kwargs.get("messages", None):
                 messages = kwargs["messages"]
             ### PRE-CALL RULES ###
-            Rules = getattr(sys.modules[__name__], 'Rules')
+            Rules = getattr(sys.modules[__name__], "Rules")
             if (
                 Rules.has_pre_call_rules()
                 and isinstance(messages, list)
@@ -947,7 +955,6 @@ def function_setup(  # noqa: PLR0915
                 and isinstance(messages[0], dict)
                 and "content" in messages[0]
             ):
-
                 buffer = StringIO()
                 for m in messages:
                     content = m.get("content", "")
@@ -1041,9 +1048,7 @@ def function_setup(  # noqa: PLR0915
             _file_obj: FileTypes = args[1] if len(args) > 1 else kwargs["file"]
             # Lazy import audio_utils.utils only when needed for transcription calls
             audio_utils = _get_cached_audio_utils()
-            file_checksum = audio_utils.get_audio_file_content_hash(
-                file_obj=_file_obj
-            )
+            file_checksum = audio_utils.get_audio_file_content_hash(file_obj=_file_obj)
             if "metadata" in kwargs:
                 kwargs["metadata"]["file_checksum"] = file_checksum
             else:
@@ -1064,6 +1069,42 @@ def function_setup(  # noqa: PLR0915
                 else kwargs.get("input")
                 or kwargs.get("messages", "default-message-value")
             )
+        elif (
+            call_type == CallTypes.generate_content.value
+            or call_type == CallTypes.agenerate_content.value
+            or call_type == CallTypes.generate_content_stream.value
+            or call_type == CallTypes.agenerate_content_stream.value
+        ):
+            try:
+                from litellm.google_genai.adapters.transformation import (
+                    GoogleGenAIAdapter,
+                )
+                from litellm.litellm_core_utils.prompt_templates.common_utils import (
+                    get_last_user_message,
+                )
+
+                contents_param = args[1] if len(args) > 1 else kwargs.get("contents")
+                model_param = args[0] if len(args) > 0 else kwargs.get("model", "")
+
+                if contents_param:
+                    adapter = GoogleGenAIAdapter()
+                    transformed = adapter.translate_generate_content_to_completion(
+                        model=model_param,
+                        contents=contents_param,
+                        config=kwargs.get("config"),
+                    )
+                    transformed_messages = transformed.get("messages", [])
+                    messages = (
+                        get_last_user_message(transformed_messages)
+                        or "default-message-value"
+                    )
+                else:
+                    messages = "default-message-value"
+            except Exception as e:
+                verbose_logger.debug(
+                    f"Error extracting messages from Google contents: {str(e)}"
+                )
+                messages = "default-message-value"
         else:
             messages = "default-message-value"
         stream = False
@@ -1072,7 +1113,9 @@ def function_setup(  # noqa: PLR0915
             call_type=call_type,
         ):
             stream = True
-        get_litellm_logging_class = getattr(sys.modules[__name__], 'get_litellm_logging_class')
+        get_litellm_logging_class = getattr(
+            sys.modules[__name__], "get_litellm_logging_class"
+        )
         logging_obj = get_litellm_logging_class()(  # Victim for object pool
             model=model,  # type: ignore
             messages=messages,
@@ -1156,8 +1199,10 @@ def _get_wrapper_num_retries(
     if num_retries is None:
         num_retries = litellm.num_retries
     if kwargs.get("retry_policy", None):
-        get_num_retries_from_retry_policy = getattr(sys.modules[__name__], 'get_num_retries_from_retry_policy')
-        reset_retry_policy = getattr(sys.modules[__name__], 'reset_retry_policy')
+        get_num_retries_from_retry_policy = getattr(
+            sys.modules[__name__], "get_num_retries_from_retry_policy"
+        )
+        reset_retry_policy = getattr(sys.modules[__name__], "reset_retry_policy")
         retry_policy_num_retries = get_num_retries_from_retry_policy(
             exception=exception,
             retry_policy=kwargs.get("retry_policy"),
@@ -1185,7 +1230,7 @@ def _get_wrapper_timeout(
 
 
 def check_coroutine(value) -> bool:
-    get_coroutine_checker = getattr(sys.modules[__name__], 'get_coroutine_checker')
+    get_coroutine_checker = getattr(sys.modules[__name__], "get_coroutine_checker")
     return get_coroutine_checker().is_async_callable(value)
 
 
@@ -1344,7 +1389,7 @@ def post_call_processing(
 
 
 def client(original_function):  # noqa: PLR0915
-    Rules = getattr(sys.modules[__name__], 'Rules')
+    Rules = getattr(sys.modules[__name__], "Rules")
     rules_obj = Rules()
 
     @wraps(original_function)
@@ -1530,7 +1575,9 @@ def client(original_function):  # noqa: PLR0915
                     )
                 else:
                     # RETURN RESULT
-                    update_response_metadata = getattr(sys.modules[__name__], 'update_response_metadata')
+                    update_response_metadata = getattr(
+                        sys.modules[__name__], "update_response_metadata"
+                    )
                     update_response_metadata(
                         result=result,
                         logging_obj=logging_obj,
@@ -1574,7 +1621,7 @@ def client(original_function):  # noqa: PLR0915
             # Copy the current context to propagate it to the background thread
             # This is essential for OpenTelemetry span context propagation
             ctx = contextvars.copy_context()
-            executor = getattr(sys.modules[__name__], 'executor')
+            executor = getattr(sys.modules[__name__], "executor")
             executor.submit(
                 ctx.run,
                 logging_obj.success_handler,
@@ -1583,7 +1630,9 @@ def client(original_function):  # noqa: PLR0915
                 end_time,
             )
             # RETURN RESULT
-            update_response_metadata = getattr(sys.modules[__name__], 'update_response_metadata')
+            update_response_metadata = getattr(
+                sys.modules[__name__], "update_response_metadata"
+            )
             update_response_metadata(
                 result=result,
                 logging_obj=logging_obj,
@@ -1600,15 +1649,19 @@ def client(original_function):  # noqa: PLR0915
                     kwargs.get("num_retries", None) or litellm.num_retries or None
                 )
                 if kwargs.get("retry_policy", None):
-                    get_num_retries_from_retry_policy = getattr(sys.modules[__name__], 'get_num_retries_from_retry_policy')
-                    reset_retry_policy = getattr(sys.modules[__name__], 'reset_retry_policy')
+                    get_num_retries_from_retry_policy = getattr(
+                        sys.modules[__name__], "get_num_retries_from_retry_policy"
+                    )
+                    reset_retry_policy = getattr(
+                        sys.modules[__name__], "reset_retry_policy"
+                    )
                     num_retries = get_num_retries_from_retry_policy(
                         exception=e,
                         retry_policy=kwargs.get("retry_policy"),
                     )
-                    kwargs["retry_policy"] = (
-                        reset_retry_policy()
-                    )  # prevent infinite loops
+                    kwargs[
+                        "retry_policy"
+                    ] = reset_retry_policy()  # prevent infinite loops
                 litellm.num_retries = (
                     None  # set retries to None to prevent infinite loops
                 )
@@ -1645,15 +1698,19 @@ def client(original_function):  # noqa: PLR0915
                     kwargs.get("num_retries", None) or litellm.num_retries or None
                 )
                 if kwargs.get("retry_policy", None):
-                    get_num_retries_from_retry_policy = getattr(sys.modules[__name__], 'get_num_retries_from_retry_policy')
-                    reset_retry_policy = getattr(sys.modules[__name__], 'reset_retry_policy')
+                    get_num_retries_from_retry_policy = getattr(
+                        sys.modules[__name__], "get_num_retries_from_retry_policy"
+                    )
+                    reset_retry_policy = getattr(
+                        sys.modules[__name__], "reset_retry_policy"
+                    )
                     num_retries = get_num_retries_from_retry_policy(
                         exception=e,
                         retry_policy=kwargs.get("retry_policy"),
                     )
-                    kwargs["retry_policy"] = (
-                        reset_retry_policy()
-                    )  # prevent infinite loops
+                    kwargs[
+                        "retry_policy"
+                    ] = reset_retry_policy()  # prevent infinite loops
                 litellm.num_retries = (
                     None  # set retries to None to prevent infinite loops
                 )
@@ -1804,7 +1861,9 @@ def client(original_function):  # noqa: PLR0915
                         chunks, messages=kwargs.get("messages", None)
                     )
                 else:
-                    update_response_metadata = getattr(sys.modules[__name__], 'update_response_metadata')
+                    update_response_metadata = getattr(
+                        sys.modules[__name__], "update_response_metadata"
+                    )
                     update_response_metadata(
                         result=result,
                         logging_obj=logging_obj,
@@ -1869,7 +1928,9 @@ def client(original_function):  # noqa: PLR0915
                     end_time=end_time,
                 )
 
-            update_response_metadata = getattr(sys.modules[__name__], 'update_response_metadata')
+            update_response_metadata = getattr(
+                sys.modules[__name__], "update_response_metadata"
+            )
             update_response_metadata(
                 result=result,
                 logging_obj=logging_obj,
@@ -1969,7 +2030,7 @@ def client(original_function):  # noqa: PLR0915
             setattr(e, "timeout", timeout)
             raise e
 
-    get_coroutine_checker = getattr(sys.modules[__name__], 'get_coroutine_checker')
+    get_coroutine_checker = getattr(sys.modules[__name__], "get_coroutine_checker")
     is_coroutine = get_coroutine_checker().is_async_callable(original_function)
 
     # Return the appropriate wrapper based on the original function type
@@ -2330,7 +2391,7 @@ def supports_response_schema(
     """
     ## GET LLM PROVIDER ##
     try:
-        get_llm_provider = getattr(sys.modules[__name__], 'get_llm_provider')
+        get_llm_provider = getattr(sys.modules[__name__], "get_llm_provider")
         model, custom_llm_provider, _, _ = get_llm_provider(
             model=model, custom_llm_provider=custom_llm_provider
         )
@@ -2674,6 +2735,7 @@ def register_model(model_cost: Union[str, dict]):  # noqa: PLR0915
     # Skip get_model_info for these providers during model registration
     _skip_get_model_info_providers = {
         LlmProviders.GITHUB_COPILOT.value,
+        LlmProviders.CHATGPT.value,
     }
 
     for key, value in loaded_model_cost.items():
@@ -2694,10 +2756,10 @@ def register_model(model_cost: Union[str, dict]):  # noqa: PLR0915
         ## override / add new keys to the existing model cost dictionary
         updated_dictionary = _update_dictionary(existing_model, value)
         litellm.model_cost.setdefault(model_cost_key, {}).update(updated_dictionary)
-        
+
         # Invalidate case-insensitive lookup map since model_cost was modified
         _invalidate_model_cost_lowercase_map()
-        
+
         verbose_logger.debug(
             f"added/updated model={model_cost_key} in litellm.model_cost: {model_cost_key}"
         )
@@ -3034,7 +3096,9 @@ def get_optional_params_embeddings(  # noqa: PLR0915
     **kwargs,
 ):
     # Lazy load get_supported_openai_params
-    get_supported_openai_params = getattr(sys.modules[__name__], 'get_supported_openai_params')
+    get_supported_openai_params = getattr(
+        sys.modules[__name__], "get_supported_openai_params"
+    )
 
     # retrieve all parameters passed to the function
     passed_params = locals()
@@ -3308,8 +3372,8 @@ def get_optional_params_embeddings(  # noqa: PLR0915
         )
 
     elif custom_llm_provider == "ollama":
-        if 'dimensions' in non_default_params:
-            optional_params['dimensions']=non_default_params.pop('dimensions')
+        if "dimensions" in non_default_params:
+            optional_params["dimensions"] = non_default_params.pop("dimensions")
         if len(non_default_params.keys()) > 0:
             if (
                 litellm.drop_params is True or drop_params is True
@@ -3575,10 +3639,10 @@ def pre_process_non_default_params(
 
     if "response_format" in non_default_params:
         if provider_config is not None:
-            non_default_params["response_format"] = (
-                provider_config.get_json_schema_from_pydantic_object(
-                    response_format=non_default_params["response_format"]
-                )
+            non_default_params[
+                "response_format"
+            ] = provider_config.get_json_schema_from_pydantic_object(
+                response_format=non_default_params["response_format"]
             )
         else:
             non_default_params["response_format"] = type_to_response_format_param(
@@ -3707,16 +3771,16 @@ def pre_process_optional_params(
                     True  # so that main.py adds the function call to the prompt
                 )
                 if "tools" in non_default_params:
-                    optional_params["functions_unsupported_model"] = (
-                        non_default_params.pop("tools")
-                    )
+                    optional_params[
+                        "functions_unsupported_model"
+                    ] = non_default_params.pop("tools")
                     non_default_params.pop(
                         "tool_choice", None
                     )  # causes ollama requests to hang
                 elif "functions" in non_default_params:
-                    optional_params["functions_unsupported_model"] = (
-                        non_default_params.pop("functions")
-                    )
+                    optional_params[
+                        "functions_unsupported_model"
+                    ] = non_default_params.pop("functions")
             elif (
                 litellm.add_function_to_prompt
             ):  # if user opts to add it to prompt instead
@@ -3840,7 +3904,9 @@ def get_optional_params(  # noqa: PLR0915
                     message=f"{custom_llm_provider} does not support parameters: {list(unsupported_params.keys())}, for model={model}. To drop these, set `litellm.drop_params=True` or for proxy:\n\n`litellm_settings:\n drop_params: true`\n. \n If you want to use these params dynamically send allowed_openai_params={list(unsupported_params.keys())} in your request.",
                 )
 
-    get_supported_openai_params = getattr(sys.modules[__name__], 'get_supported_openai_params')
+    get_supported_openai_params = getattr(
+        sys.modules[__name__], "get_supported_openai_params"
+    )
     supported_params = get_supported_openai_params(
         model=model, custom_llm_provider=custom_llm_provider
     )
@@ -4095,7 +4161,7 @@ def get_optional_params(  # noqa: PLR0915
             ),
         )
     elif custom_llm_provider == "bedrock":
-        BedrockModelInfo = getattr(sys.modules[__name__], 'BedrockModelInfo')
+        BedrockModelInfo = getattr(sys.modules[__name__], "BedrockModelInfo")
         bedrock_route = BedrockModelInfo.get_bedrock_route(model)
         bedrock_base_model = BedrockModelInfo.get_base_model(model)
         if bedrock_route == "converse" or bedrock_route == "converse_like":
@@ -4522,8 +4588,8 @@ def get_optional_params(  # noqa: PLR0915
 
     # Apply nested drops from additional_drop_params
     if additional_drop_params:
-        is_nested_path = getattr(sys.modules[__name__], 'is_nested_path')
-        delete_nested_value = getattr(sys.modules[__name__], 'delete_nested_value')
+        is_nested_path = getattr(sys.modules[__name__], "is_nested_path")
+        delete_nested_value = getattr(sys.modules[__name__], "delete_nested_value")
         nested_paths = [p for p in additional_drop_params if is_nested_path(p)]
         for path in nested_paths:
             optional_params = delete_nested_value(optional_params, path)
@@ -4573,7 +4639,9 @@ def add_provider_specific_params_to_optional_params(
             else:
                 processed_extra_body = initial_extra_body
 
-            _ensure_extra_body_is_safe = getattr(sys.modules[__name__], '_ensure_extra_body_is_safe')
+            _ensure_extra_body_is_safe = getattr(
+                sys.modules[__name__], "_ensure_extra_body_is_safe"
+            )
             optional_params["extra_body"] = _ensure_extra_body_is_safe(
                 extra_body=processed_extra_body
             )
@@ -4866,9 +4934,9 @@ def get_response_string(response_obj: Union[ModelResponse, ModelResponseStream])
             return delta if isinstance(delta, str) else ""
 
     # Handle standard ModelResponse and ModelResponseStream
-    _choices: Union[List[Union[Choices, StreamingChoices]], List[StreamingChoices]] = (
-        response_obj.choices
-    )
+    _choices: Union[
+        List[Union[Choices, StreamingChoices]], List[StreamingChoices]
+    ] = response_obj.choices
 
     # Use list accumulation to avoid O(n^2) string concatenation across choices
     response_parts: List[str] = []
@@ -4986,7 +5054,7 @@ def get_max_tokens(model: str) -> Optional[int]:
                 return litellm.model_cost[model]["max_output_tokens"]
             elif "max_tokens" in litellm.model_cost[model]:
                 return litellm.model_cost[model]["max_tokens"]
-        get_llm_provider = getattr(sys.modules[__name__], 'get_llm_provider')
+        get_llm_provider = getattr(sys.modules[__name__], "get_llm_provider")
         model, custom_llm_provider, _, _ = get_llm_provider(model=model)
         if custom_llm_provider == "huggingface":
             max_tokens = _get_max_position_embeddings(model_name=model)
@@ -5062,7 +5130,7 @@ _model_cost_lowercase_map: Optional[Dict[str, str]] = None
 
 def _invalidate_model_cost_lowercase_map() -> None:
     """Invalidate the case-insensitive lookup map for model_cost.
-    
+
     Call this whenever litellm.model_cost is modified to ensure the map is rebuilt.
     """
     global _model_cost_lowercase_map
@@ -5071,7 +5139,7 @@ def _invalidate_model_cost_lowercase_map() -> None:
 
 def _rebuild_model_cost_lowercase_map() -> Dict[str, str]:
     """Rebuild the case-insensitive lookup map from the current model_cost.
-    
+
     Returns:
         The rebuilt map (guaranteed to be not None).
     """
@@ -5085,9 +5153,9 @@ def _handle_stale_map_entry_rebuild(
 ) -> Optional[str]:
     """
     Handle stale _model_cost_lowercase_map entry (key was popped).
-    
+
     Rebuilds the map and retries the lookup.
-    
+
     Returns:
         The matched key if found after rebuild, None otherwise.
     """
@@ -5104,9 +5172,9 @@ def _handle_new_key_with_scan(
 ) -> Optional[str]:
     """
     Handle new key added to model_cost without invalidating _model_cost_lowercase_map.
-    
+
     Scans model_cost for case-insensitive match and rebuilds the map if found.
-    
+
     Returns:
         The matched key if found, None otherwise.
     """
@@ -5121,20 +5189,20 @@ def _handle_new_key_with_scan(
 def _get_model_cost_key(potential_key: str) -> Optional[str]:
     """
     Get the actual key from model_cost, with case-insensitive fallback.
-    
+
     WARNING: Only O(1) lookup operations are acceptable. O(n) lookups will cause severe
     CPU overhead. This function is called frequently during router operations.
-    
+
     ALLOWED HELPER FUNCTIONS (conditionally called, O(n) operations are acceptable):
     - _rebuild_model_cost_lowercase_map: Rebuilds the lookup map (only when map is None)
     - _handle_stale_map_entry_rebuild: Rebuilds map when stale entry detected (rare case)
-    
+
     If you need to add a new helper function with O(n) operations that is conditionally
     called and confirmed not to cause performance issues, add it to the allowed_helpers
     list in: tests/code_coverage_tests/check_get_model_cost_key_performance.py
     """
     global _model_cost_lowercase_map
-    
+
     # Exact match (O(1))
     if potential_key in litellm.model_cost:
         return potential_key
@@ -5142,20 +5210,20 @@ def _get_model_cost_key(potential_key: str) -> Optional[str]:
     # Case-insensitive lookup via map (O(1))
     if _model_cost_lowercase_map is None:
         _model_cost_lowercase_map = _rebuild_model_cost_lowercase_map()
-    
+
     potential_key_lower = potential_key.lower()
     matched_key = _model_cost_lowercase_map.get(potential_key_lower)
-    
+
     # Verify key exists (O(1) - handles model_cost.pop() case)
     if matched_key is not None and matched_key in litellm.model_cost:
         return matched_key
-    
+
     # Rebuild map if stale entry detected (O(n) rebuild, but only when stale entry found)
     if matched_key is not None:
         matched_key = _handle_stale_map_entry_rebuild(potential_key_lower)
         if matched_key is not None:
             return matched_key
-    
+
     return None
 
 
@@ -5187,9 +5255,12 @@ def _check_provider_match(model_info: dict, custom_llm_provider: Optional[str])
             custom_llm_provider == "litellm_proxy"
         ):  # litellm_proxy is a special case, it's not a provider, it's a proxy for the provider
             return True
-        elif custom_llm_provider == "azure_ai" and model_info["litellm_provider"] in ("azure", "openai"):
-            # Azure AI also works with azure models 
-            # as a last attempt if the model is not on Azure AI, Azure then fallback to OpenAI cost 
+        elif custom_llm_provider == "azure_ai" and model_info["litellm_provider"] in (
+            "azure",
+            "openai",
+        ):
+            # Azure AI also works with azure models
+            # as a last attempt if the model is not on Azure AI, Azure then fallback to OpenAI cost
             # tracking the cost is better than attributing 0 cost to it.
             return True
         else:
@@ -5215,7 +5286,7 @@ def _get_potential_model_names(
     if custom_llm_provider is None:
         # Get custom_llm_provider
         try:
-            get_llm_provider = getattr(sys.modules[__name__], 'get_llm_provider')
+            get_llm_provider = getattr(sys.modules[__name__], "get_llm_provider")
             split_model, custom_llm_provider, _, _ = get_llm_provider(model=model)
         except Exception:
             split_model = model
@@ -5519,6 +5590,13 @@ def _get_model_info_helper(  # noqa: PLR0915
                 input_cost_per_image_token=_model_info.get(
                     "input_cost_per_image_token", None
                 ),
+                input_cost_per_image=_model_info.get("input_cost_per_image", None),
+                input_cost_per_audio_per_second=_model_info.get(
+                    "input_cost_per_audio_per_second", None
+                ),
+                input_cost_per_video_per_second=_model_info.get(
+                    "input_cost_per_video_per_second", None
+                ),
                 input_cost_per_token_batches=_model_info.get(
                     "input_cost_per_token_batches"
                 ),
@@ -5945,7 +6023,7 @@ def validate_environment(  # noqa: PLR0915
         }
     ## EXTRACT LLM PROVIDER - if model name provided
     try:
-        get_llm_provider = getattr(sys.modules[__name__], 'get_llm_provider')
+        get_llm_provider = getattr(sys.modules[__name__], "get_llm_provider")
         _, custom_llm_provider, _, _ = get_llm_provider(model=model)
     except Exception:
         custom_llm_provider = None
@@ -6508,7 +6586,7 @@ def register_prompt_template(
     complete_model = model
     potential_models = [complete_model]
     try:
-        get_llm_provider = getattr(sys.modules[__name__], 'get_llm_provider')
+        get_llm_provider = getattr(sys.modules[__name__], "get_llm_provider")
         model = get_llm_provider(model=model)[0]
         potential_models.append(model)
     except Exception:
@@ -6594,7 +6672,7 @@ class TextCompletionStreamWrapper:
         except StopIteration:
             raise StopIteration
         except Exception as e:
-            exception_type = getattr(sys.modules[__name__], 'exception_type')
+            exception_type = getattr(sys.modules[__name__], "exception_type")
             raise exception_type(
                 model=self.model,
                 custom_llm_provider=self.custom_llm_provider or "",
@@ -7218,14 +7296,20 @@ def _get_base_model_from_metadata(model_call_details=None):
             return _base_model
         metadata = litellm_params.get("metadata", {})
 
-        _get_base_model_from_litellm_call_metadata = getattr(sys.modules[__name__], '_get_base_model_from_litellm_call_metadata')
-        base_model_from_metadata = _get_base_model_from_litellm_call_metadata(metadata=metadata)
+        _get_base_model_from_litellm_call_metadata = getattr(
+            sys.modules[__name__], "_get_base_model_from_litellm_call_metadata"
+        )
+        base_model_from_metadata = _get_base_model_from_litellm_call_metadata(
+            metadata=metadata
+        )
         if base_model_from_metadata is not None:
             return base_model_from_metadata
 
         # Also check litellm_metadata (used by Responses API and other generic API calls)
         litellm_metadata = litellm_params.get("litellm_metadata", {})
-        _get_base_model_from_litellm_call_metadata = getattr(sys.modules[__name__], '_get_base_model_from_litellm_call_metadata')
+        _get_base_model_from_litellm_call_metadata = getattr(
+            sys.modules[__name__], "_get_base_model_from_litellm_call_metadata"
+        )
         return _get_base_model_from_litellm_call_metadata(metadata=litellm_metadata)
     return None
 
@@ -7631,12 +7715,30 @@ class ProviderConfigManager:
             # Format: (factory_function, needs_model_parameter: bool)
             LlmProviders.OPENAI: (lambda: litellm.OpenAIGPTConfig(), False),
             LlmProviders.ANTHROPIC: (lambda: litellm.AnthropicConfig(), False),
-            LlmProviders.AZURE: (lambda model: ProviderConfigManager._get_azure_config(model), True),
-            LlmProviders.AZURE_AI: (lambda model: ProviderConfigManager._get_azure_ai_config(model), True),
-            LlmProviders.VERTEX_AI: (lambda model: ProviderConfigManager._get_vertex_ai_config(model), True),
-            LlmProviders.BEDROCK: (lambda model: ProviderConfigManager._get_bedrock_config(model), True),
-            LlmProviders.COHERE: (lambda model: ProviderConfigManager._get_cohere_config(model), True),
-            LlmProviders.COHERE_CHAT: (lambda model: ProviderConfigManager._get_cohere_config(model), True),
+            LlmProviders.AZURE: (
+                lambda model: ProviderConfigManager._get_azure_config(model),
+                True,
+            ),
+            LlmProviders.AZURE_AI: (
+                lambda model: ProviderConfigManager._get_azure_ai_config(model),
+                True,
+            ),
+            LlmProviders.VERTEX_AI: (
+                lambda model: ProviderConfigManager._get_vertex_ai_config(model),
+                True,
+            ),
+            LlmProviders.BEDROCK: (
+                lambda model: ProviderConfigManager._get_bedrock_config(model),
+                True,
+            ),
+            LlmProviders.COHERE: (
+                lambda model: ProviderConfigManager._get_cohere_config(model),
+                True,
+            ),
+            LlmProviders.COHERE_CHAT: (
+                lambda model: ProviderConfigManager._get_cohere_config(model),
+                True,
+            ),
             # Simple provider mappings (no model parameter needed)
             LlmProviders.DEEPSEEK: (lambda: litellm.DeepSeekChatConfig(), False),
             LlmProviders.GROQ: (lambda: litellm.GroqChatConfig(), False),
@@ -7646,7 +7748,10 @@ class ProviderConfigManager:
             LlmProviders.ZAI: (lambda: litellm.ZAIChatConfig(), False),
             LlmProviders.LAMBDA_AI: (lambda: litellm.LambdaAIChatConfig(), False),
             LlmProviders.LLAMA: (lambda: litellm.LlamaAPIConfig(), False),
-            LlmProviders.TEXT_COMPLETION_OPENAI: (lambda: litellm.OpenAITextCompletionConfig(), False),
+            LlmProviders.TEXT_COMPLETION_OPENAI: (
+                lambda: litellm.OpenAITextCompletionConfig(),
+                False,
+            ),
             LlmProviders.SNOWFLAKE: (lambda: litellm.SnowflakeConfig(), False),
             LlmProviders.CLARIFAI: (lambda: litellm.ClarifaiConfig(), False),
             LlmProviders.ANTHROPIC_TEXT: (lambda: litellm.AnthropicTextConfig(), False),
@@ -7663,12 +7768,16 @@ class ProviderConfigManager:
             LlmProviders.GITHUB: (lambda: litellm.GithubChatConfig(), False),
             LlmProviders.COMPACTIFAI: (lambda: litellm.CompactifAIChatConfig(), False),
             LlmProviders.GITHUB_COPILOT: (lambda: litellm.GithubCopilotConfig(), False),
+            LlmProviders.CHATGPT: (lambda: litellm.ChatGPTConfig(), False),
             LlmProviders.GIGACHAT: (lambda: litellm.GigaChatConfig(), False),
             LlmProviders.RAGFLOW: (lambda: litellm.RAGFlowConfig(), False),
             LlmProviders.CUSTOM: (lambda: litellm.OpenAILikeChatConfig(), False),
             LlmProviders.CUSTOM_OPENAI: (lambda: litellm.OpenAILikeChatConfig(), False),
             LlmProviders.OPENAI_LIKE: (lambda: litellm.OpenAILikeChatConfig(), False),
-            LlmProviders.AIOHTTP_OPENAI: (lambda: litellm.AiohttpOpenAIChatConfig(), False),
+            LlmProviders.AIOHTTP_OPENAI: (
+                lambda: litellm.AiohttpOpenAIChatConfig(),
+                False,
+            ),
             LlmProviders.HOSTED_VLLM: (lambda: litellm.HostedVLLMChatConfig(), False),
             LlmProviders.LLAMAFILE: (lambda: litellm.LlamafileChatConfig(), False),
             LlmProviders.LM_STUDIO: (lambda: litellm.LMStudioChatConfig(), False),
@@ -7677,7 +7786,10 @@ class ProviderConfigManager:
             LlmProviders.HUGGINGFACE: (lambda: litellm.HuggingFaceChatConfig(), False),
             LlmProviders.TOGETHER_AI: (lambda: litellm.TogetherAIConfig(), False),
             LlmProviders.OPENROUTER: (lambda: litellm.OpenrouterConfig(), False),
-            LlmProviders.VERCEL_AI_GATEWAY: (lambda: litellm.VercelAIGatewayConfig(), False),
+            LlmProviders.VERCEL_AI_GATEWAY: (
+                lambda: litellm.VercelAIGatewayConfig(),
+                False,
+            ),
             LlmProviders.COMETAPI: (lambda: litellm.CometAPIConfig(), False),
             LlmProviders.DATAROBOT: (lambda: litellm.DataRobotConfig(), False),
             LlmProviders.GEMINI: (lambda: litellm.GoogleAIStudioGeminiConfig(), False),
@@ -7695,7 +7807,10 @@ class ProviderConfigManager:
             LlmProviders.CEREBRAS: (lambda: litellm.CerebrasConfig(), False),
             LlmProviders.BASETEN: (lambda: litellm.BasetenConfig(), False),
             LlmProviders.VOLCENGINE: (lambda: litellm.VolcEngineConfig(), False),
-            LlmProviders.TEXT_COMPLETION_CODESTRAL: (lambda: litellm.CodestralTextCompletionConfig(), False),
+            LlmProviders.TEXT_COMPLETION_CODESTRAL: (
+                lambda: litellm.CodestralTextCompletionConfig(),
+                False,
+            ),
             LlmProviders.SAMBANOVA: (lambda: litellm.SambanovaConfig(), False),
             LlmProviders.MARITALK: (lambda: litellm.MaritalkConfig(), False),
             LlmProviders.VLLM: (lambda: litellm.VLLMConfig(), False),
@@ -7703,17 +7818,26 @@ class ProviderConfigManager:
             LlmProviders.PREDIBASE: (lambda: litellm.PredibaseConfig(), False),
             LlmProviders.TRITON: (lambda: litellm.TritonConfig(), False),
             LlmProviders.PETALS: (lambda: litellm.PetalsConfig(), False),
-            LlmProviders.SAP_GENERATIVE_AI_HUB: (lambda: litellm.GenAIHubOrchestrationConfig(), False),
+            LlmProviders.SAP_GENERATIVE_AI_HUB: (
+                lambda: litellm.GenAIHubOrchestrationConfig(),
+                False,
+            ),
             LlmProviders.FEATHERLESS_AI: (lambda: litellm.FeatherlessAIConfig(), False),
             LlmProviders.NOVITA: (lambda: litellm.NovitaConfig(), False),
             LlmProviders.NEBIUS: (lambda: litellm.NebiusConfig(), False),
             LlmProviders.WANDB: (lambda: litellm.WandbConfig(), False),
             LlmProviders.DASHSCOPE: (lambda: litellm.DashScopeChatConfig(), False),
             LlmProviders.MOONSHOT: (lambda: litellm.MoonshotChatConfig(), False),
-            LlmProviders.DOCKER_MODEL_RUNNER: (lambda: litellm.DockerModelRunnerChatConfig(), False),
+            LlmProviders.DOCKER_MODEL_RUNNER: (
+                lambda: litellm.DockerModelRunnerChatConfig(),
+                False,
+            ),
             LlmProviders.V0: (lambda: litellm.V0ChatConfig(), False),
             LlmProviders.MORPH: (lambda: litellm.MorphChatConfig(), False),
-            LlmProviders.LITELLM_PROXY: (lambda: litellm.LiteLLMProxyChatConfig(), False),
+            LlmProviders.LITELLM_PROXY: (
+                lambda: litellm.LiteLLMProxyChatConfig(),
+                False,
+            ),
             LlmProviders.GRADIENT_AI: (lambda: litellm.GradientAIConfig(), False),
             LlmProviders.NSCALE: (lambda: litellm.NscaleConfig(), False),
             LlmProviders.HEROKU: (lambda: litellm.HerokuChatConfig(), False),
@@ -7721,7 +7845,10 @@ class ProviderConfigManager:
             LlmProviders.HYPERBOLIC: (lambda: litellm.HyperbolicChatConfig(), False),
             LlmProviders.OVHCLOUD: (lambda: litellm.OVHCloudChatConfig(), False),
             LlmProviders.AMAZON_NOVA: (lambda: litellm.AmazonNovaChatConfig(), False),
-            LlmProviders.LANGGRAPH: (lambda: ProviderConfigManager._get_langgraph_config(), False),
+            LlmProviders.LANGGRAPH: (
+                lambda: ProviderConfigManager._get_langgraph_config(),
+                False,
+            ),
         }
 
     @staticmethod
@@ -7751,6 +7878,7 @@ class ProviderConfigManager:
             from litellm.llms.vertex_ai.vertex_ai_partner_models.gpt_oss.transformation import (
                 VertexAIGPTOSSTransformation,
             )
+
             return VertexAIGPTOSSTransformation()
         elif model in litellm.vertex_mistral_models:
             if "codestral" in model:
@@ -7765,12 +7893,13 @@ class ProviderConfigManager:
     def _get_bedrock_config(model: str) -> BaseConfig:
         """Get Bedrock config based on model."""
         from litellm.llms.bedrock.common_utils import get_bedrock_chat_config
+
         return get_bedrock_chat_config(model=model)
 
     @staticmethod
     def _get_cohere_config(model: str) -> BaseConfig:
         """Get Cohere config based on route."""
-        CohereModelInfo = getattr(sys.modules[__name__], 'CohereModelInfo')
+        CohereModelInfo = getattr(sys.modules[__name__], "CohereModelInfo")
         route = CohereModelInfo.get_cohere_route(model)
         if route == "v2":
             return litellm.CohereV2ChatConfig()
@@ -7780,6 +7909,7 @@ class ProviderConfigManager:
     def _get_langgraph_config() -> BaseConfig:
         """Get LangGraph config."""
         from litellm.llms.langgraph.chat.transformation import LangGraphConfig
+
         return LangGraphConfig()
 
     @staticmethod
@@ -7810,7 +7940,9 @@ class ProviderConfigManager:
 
         # Initialize provider config map lazily (avoids circular imports)
         if ProviderConfigManager._PROVIDER_CONFIG_MAP is None:
-            ProviderConfigManager._PROVIDER_CONFIG_MAP = ProviderConfigManager._build_provider_config_map()
+            ProviderConfigManager._PROVIDER_CONFIG_MAP = (
+                ProviderConfigManager._build_provider_config_map()
+            )
 
         # O(1) dictionary lookup
         config_entry = ProviderConfigManager._PROVIDER_CONFIG_MAP.get(provider)
@@ -7880,6 +8012,7 @@ class ProviderConfigManager:
             from litellm.llms.openrouter.embedding.transformation import (
                 OpenrouterEmbeddingConfig,
             )
+
             return OpenrouterEmbeddingConfig()
         elif litellm.LlmProviders.GIGACHAT == provider:
             return litellm.GigaChatEmbeddingConfig()
@@ -8034,6 +8167,8 @@ class ProviderConfigManager:
             return litellm.XAIResponsesAPIConfig()
         elif litellm.LlmProviders.GITHUB_COPILOT == provider:
             return litellm.GithubCopilotResponsesAPIConfig()
+        elif litellm.LlmProviders.CHATGPT == provider:
+            return litellm.ChatGPTResponsesAPIConfig()
         elif litellm.LlmProviders.LITELLM_PROXY == provider:
             return litellm.LiteLLMProxyResponsesAPIConfig()
         elif litellm.LlmProviders.VOLCENGINE == provider:
@@ -8110,6 +8245,10 @@ class ProviderConfigManager:
             return litellm.ClarifaiConfig()
         elif LlmProviders.BEDROCK == provider:
             return litellm.llms.bedrock.common_utils.BedrockModelInfo()
+        elif LlmProviders.AZURE_AI == provider:
+            from litellm.llms.azure_ai.common_utils import AzureFoundryModelInfo
+
+            return AzureFoundryModelInfo(model=model)
         return None
 
     @staticmethod
@@ -8496,7 +8635,7 @@ class ProviderConfigManager:
 
             return get_vertex_ai_ocr_config(model=model)
 
-        MistralOCRConfig = getattr(sys.modules[__name__], 'MistralOCRConfig')
+        MistralOCRConfig = getattr(sys.modules[__name__], "MistralOCRConfig")
         PROVIDER_TO_CONFIG_MAP = {
             litellm.LlmProviders.MISTRAL: MistralOCRConfig,
         }
@@ -8637,7 +8776,9 @@ def get_end_user_id_for_cost_tracking(
 
     service_type: "litellm_logging" or "prometheus" - used to allow prometheus only disable cost tracking.
     """
-    get_litellm_metadata_from_kwargs = getattr(sys.modules[__name__], 'get_litellm_metadata_from_kwargs')
+    get_litellm_metadata_from_kwargs = getattr(
+        sys.modules[__name__], "get_litellm_metadata_from_kwargs"
+    )
     _metadata = cast(
         dict, get_litellm_metadata_from_kwargs(dict(litellm_params=litellm_params))
     )

model_prices_and_context_window.json

@@ -13465,6 +13465,31 @@
         "supports_vision": true,
         "supports_web_search": true
     },
+    "gemini-2.5-computer-use-preview-10-2025": {
+        "input_cost_per_token": 1.25e-06,
+        "input_cost_per_token_above_200k_tokens": 2.5e-06,
+        "litellm_provider": "vertex_ai-language-models",
+        "max_images_per_prompt": 3000,
+        "max_input_tokens": 128000,
+        "max_output_tokens": 64000,
+        "max_tokens": 64000,
+        "mode": "chat",
+        "output_cost_per_token": 1e-05,
+        "output_cost_per_token_above_200k_tokens": 1.5e-05,
+        "source": "https://cloud.google.com/vertex-ai/generative-ai/docs/computer-use",
+        "supported_modalities": [
+            "text",
+            "image"
+        ],
+        "supported_output_modalities": [
+            "text"
+        ],
+        "supports_computer_use": true,
+        "supports_function_calling": true,
+        "supports_system_messages": true,
+        "supports_tool_choice": true,
+        "supports_vision": true
+    },
     "gemini-embedding-001": {
         "input_cost_per_token": 1.5e-07,
         "litellm_provider": "vertex_ai-embedding-models",
@@ -15950,6 +15975,63 @@
         "max_tokens": 8191,
         "mode": "embedding"
     },
+    "chatgpt/gpt-5.2-codex": {
+        "litellm_provider": "chatgpt",
+        "max_input_tokens": 128000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "mode": "responses",
+        "supported_endpoints": [
+            "/v1/responses"
+        ],
+        "supports_function_calling": true,
+        "supports_parallel_function_calling": true,
+        "supports_response_schema": true,
+        "supports_vision": true
+    },
+    "chatgpt/gpt-5.2": {
+        "litellm_provider": "chatgpt",
+        "max_input_tokens": 128000,
+        "max_output_tokens": 64000,
+        "max_tokens": 64000,
+        "mode": "responses",
+        "supported_endpoints": [
+            "/v1/chat/completions",
+            "/v1/responses"
+        ],
+        "supports_function_calling": true,
+        "supports_parallel_function_calling": true,
+        "supports_response_schema": true,
+        "supports_vision": true
+    },
+    "chatgpt/gpt-5.1-codex-max": {
+        "litellm_provider": "chatgpt",
+        "max_input_tokens": 128000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "mode": "responses",
+        "supported_endpoints": [
+            "/v1/responses"
+        ],
+        "supports_function_calling": true,
+        "supports_parallel_function_calling": true,
+        "supports_response_schema": true,
+        "supports_vision": true
+    },
+    "chatgpt/gpt-5.1-codex-mini": {
+        "litellm_provider": "chatgpt",
+        "max_input_tokens": 128000,
+        "max_output_tokens": 64000,
+        "max_tokens": 64000,
+        "mode": "responses",
+        "supported_endpoints": [
+            "/v1/responses"
+        ],
+        "supports_function_calling": true,
+        "supports_parallel_function_calling": true,
+        "supports_response_schema": true,
+        "supports_vision": true
+    },
     "gigachat/GigaChat-2-Lite": {
         "input_cost_per_token": 0.0,
         "litellm_provider": "gigachat",

provider_endpoints_support.json

@@ -919,6 +919,24 @@
         "interactions": true
       }
     },
+    "chatgpt": {
+      "display_name": "ChatGPT Subscription (`chatgpt`)",
+      "url": "https://docs.litellm.ai/docs/providers/chatgpt",
+      "endpoints": {
+        "chat_completions": true,
+        "messages": true,
+        "responses": true,
+        "embeddings": false,
+        "image_generations": false,
+        "audio_transcriptions": false,
+        "audio_speech": false,
+        "moderations": false,
+        "batches": false,
+        "rerank": false,
+        "a2a": false,
+        "interactions": false
+      }
+    },
     "github": {
       "display_name": "GitHub Models (`github`)",
       "url": "https://docs.litellm.ai/docs/providers/github",