Merge branch 'main' into litellm_ui_claude_code_plugins

cookbook/ai_coding_tool_guides/index.json

@@ -110,7 +110,7 @@
     ]
 },
 {
-    "title": "Use Web Search with Claude Code (across OpenAI/Anthropic/Gemini/etc.)",
+    "title": "Use Web Search with Claude Code (across Bedrock/OpenAI/Gemini/etc.)",
     "description": "This is a guide for using Web Search with Claude Code via LiteLLM.",
     "url": "https://docs.litellm.ai/docs/tutorials/claude_code_websearch",
     "date": "2026-01-17",

deploy/charts/litellm-helm/Chart.yaml

@@ -18,7 +18,7 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 1.0.0
+version: 1.1.0
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to

deploy/charts/litellm-helm/templates/deployment.yaml

@@ -10,7 +10,7 @@ metadata:
     {{- toYaml .Values.deploymentLabels | nindent 4 }}
     {{- end }}
 spec:
-  {{- if not .Values.autoscaling.enabled }}
+  {{- if and (not .Values.keda.enabled) (not .Values.autoscaling.enabled) }}
   replicas: {{ .Values.replicaCount }}
   {{- end }}
   selector:

deploy/charts/litellm-helm/templates/keda.yaml

@@ -0,0 +1,37 @@
+{{- if and .Values.keda.enabled (not .Values.autoscaling.enabled) }}
+apiVersion: keda.sh/v1alpha1
+kind: ScaledObject
+metadata:
+  name: {{ include "litellm.fullname" . }}
+  labels:
+    {{- include "litellm.labels" . | nindent 4 }}
+  {{- if .Values.keda.scaledObject.annotations }}
+  annotations: {{ toYaml .Values.keda.scaledObject.annotations | nindent 4 }}
+  {{- end }}
+spec:
+  scaleTargetRef:
+    name: {{ include "litellm.fullname" . }}
+  pollingInterval: {{ .Values.keda.pollingInterval }}
+  cooldownPeriod: {{ .Values.keda.cooldownPeriod }}
+  minReplicaCount: {{ .Values.keda.minReplicas }}
+  maxReplicaCount: {{ .Values.keda.maxReplicas }}
+{{- with .Values.keda.fallback }}
+  fallback:
+    failureThreshold: {{ .failureThreshold | default 3 }}
+    replicas: {{ .replicas | default $.Values.keda.maxReplicas }}
+{{- end }}
+  triggers:
+{{- with .Values.keda.triggers }}
+  {{- toYaml . | nindent 2 }}
+{{- end }}
+  advanced:
+    restoreToOriginalReplicaCount: {{ .Values.keda.restoreToOriginalReplicaCount }}
+{{- if .Values.keda.behavior }}
+    horizontalPodAutoscalerConfig:
+      behavior:
+{{- with .Values.keda.behavior }}
+{{- toYaml . | nindent 8 }}
+{{- end }}
+
+{{- end }}
+{{- end }}

deploy/charts/litellm-helm/values.yaml

@@ -156,6 +156,40 @@ autoscaling:
   targetCPUUtilizationPercentage: 80
   # targetMemoryUtilizationPercentage: 80
 
+# Autoscaling with keda is mutually exclusive with hpa
+keda:
+  enabled: false
+  minReplicas: 1
+  maxReplicas: 100
+  pollingInterval: 30
+  cooldownPeriod: 300
+  # fallback:
+  #   failureThreshold: 3
+  #   replicas: 11
+  restoreToOriginalReplicaCount: false
+  scaledObject:
+    annotations: {}
+  triggers: []
+  # - type: prometheus
+  #   metadata:
+  #     serverAddress: http://<prometheus-host>:9090
+  #     metricName: http_requests_total
+  #     threshold: '100'
+  #     query: sum(rate(http_requests_total{deployment="my-deployment"}[2m]))
+  behavior: {}
+  # scaleDown:
+  #   stabilizationWindowSeconds: 300
+  #   policies:
+  #   - type: Pods
+  #     value: 1
+  #     periodSeconds: 180
+  # scaleUp:
+  #   stabilizationWindowSeconds: 300
+  #   policies:
+  #   - type: Pods
+  #     value: 2
+  #     periodSeconds: 60
+
 # Additional volumes on the output Deployment definition.
 volumes: []
 # - name: foo

docker/Dockerfile.health_check

@@ -0,0 +1,16 @@
+FROM python:3.11-slim
+
+WORKDIR /app
+
+# Copy health check script and requirements
+COPY scripts/health_check/health_check_client.py /app/health_check_client.py
+COPY scripts/health_check/health_check_requirements.txt /app/requirements.txt
+
+# Install dependencies
+RUN pip install --no-cache-dir -r requirements.txt
+
+# Make script executable
+RUN chmod +x /app/health_check_client.py
+
+# Set entrypoint
+ENTRYPOINT ["python", "/app/health_check_client.py"]

docs/my-website/docs/completion/token_usage.md

@@ -100,7 +100,7 @@ from litellm import cost_per_token
 
 prompt_tokens =  5
 completion_tokens = 10
-prompt_tokens_cost_usd_dollar, completion_tokens_cost_usd_dollar = cost_per_token(model="gpt-3.5-turbo", prompt_tokens=prompt_tokens, completion_tokens=completion_tokens))
+prompt_tokens_cost_usd_dollar, completion_tokens_cost_usd_dollar = cost_per_token(model="gpt-3.5-turbo", prompt_tokens=prompt_tokens, completion_tokens=completion_tokens)
 
 print(prompt_tokens_cost_usd_dollar, completion_tokens_cost_usd_dollar)
 ```
@@ -162,7 +162,7 @@ print(model_cost) # {'gpt-3.5-turbo': {'max_tokens': 4000, 'input_cost_per_token
 
 **Dictionary**
 ```python
-from litellm import register_model
+import litellm
 
 litellm.register_model({
         "gpt-4": {

docs/my-website/docs/providers/openai/text_to_speech.md

@@ -46,7 +46,7 @@ os.environ["OPENAI_API_KEY"] = "sk-.."
 
 async def test_async_speech(): 
     speech_file_path = Path(__file__).parent / "speech.mp3"
-    response = await litellm.aspeech(
+    response = await aspeech(
             model="openai/tts-1",
             voice="alloy",
             input="the quick brown fox jumped over the lazy dogs",

docs/my-website/docs/proxy/config_settings.md

@@ -603,6 +603,7 @@ router_settings:
 | GCS_PATH_SERVICE_ACCOUNT | Path to the Google Cloud service account JSON file
 | GCS_FLUSH_INTERVAL | Flush interval for GCS logging (in seconds). Specify how often you want a log to be sent to GCS. **Default is 20 seconds**
 | GCS_BATCH_SIZE | Batch size for GCS logging. Specify after how many logs you want to flush to GCS. If `BATCH_SIZE` is set to 10, logs are flushed every 10 logs. **Default is 2048**
+| GCS_USE_BATCHED_LOGGING | Enable batched logging for GCS. When enabled (default), multiple log payloads are combined into single GCS object uploads (NDJSON format), dramatically reducing API calls. When disabled, sends each log individually as separate GCS objects (legacy behavior). **Default is true**
 | GCS_PUBSUB_TOPIC_ID | PubSub Topic ID to send LiteLLM SpendLogs to.
 | GCS_PUBSUB_PROJECT_ID | PubSub Project ID to send LiteLLM SpendLogs to.
 | GENERIC_AUTHORIZATION_ENDPOINT | Authorization endpoint for generic OAuth providers

docs/my-website/docs/proxy/deploy.md

@@ -4,6 +4,10 @@ import Image from '@theme/IdealImage';
 
 # Docker, Helm, Terraform
 
+:::info No Limits on LiteLLM OSS
+There are **no limits** on the number of users, keys, or teams you can create on LiteLLM OSS.
+:::
+
 You can find the Dockerfile to build litellm proxy [here](https://github.com/BerriAI/litellm/blob/main/Dockerfile)
 
 > Note: Production requires at least 4 CPU cores and 8 GB RAM.

docs/my-website/docs/text_to_speech.md

@@ -46,7 +46,7 @@ os.environ["OPENAI_API_KEY"] = "sk-.."
 
 async def test_async_speech(): 
     speech_file_path = Path(__file__).parent / "speech.mp3"
-    response = await litellm.aspeech(
+    response = await aspeech(
             model="openai/tts-1",
             voice="alloy",
             input="the quick brown fox jumped over the lazy dogs",

docs/my-website/src/pages/token_usage.md

@@ -27,7 +27,7 @@ from litellm import cost_per_token
 
 prompt_tokens =  5
 completion_tokens = 10
-prompt_tokens_cost_usd_dollar, completion_tokens_cost_usd_dollar = cost_per_token(model="gpt-3.5-turbo", prompt_tokens=prompt_tokens, completion_tokens=completion_tokens))
+prompt_tokens_cost_usd_dollar, completion_tokens_cost_usd_dollar = cost_per_token(model="gpt-3.5-turbo", prompt_tokens=prompt_tokens, completion_tokens=completion_tokens)
 
 print(prompt_tokens_cost_usd_dollar, completion_tokens_cost_usd_dollar)
 ```

litellm/integrations/gcs_bucket/gcs_bucket.py

@@ -1,9 +1,11 @@
 import asyncio
+import hashlib
 import json
 import os
+import time
 from litellm._uuid import uuid
 from datetime import datetime, timedelta, timezone
-from typing import TYPE_CHECKING, Any, Dict, List, Optional
+from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple
 from urllib.parse import quote
 
 from litellm._logging import verbose_logger
@@ -26,19 +28,21 @@ class GCSBucketLogger(GCSBucketBase, AdditionalLoggingUtils):
 
         super().__init__(bucket_name=bucket_name)
 
-        # Init Batch logging settings
-        self.log_queue: List[GCSLogQueueItem] = []
         self.batch_size = int(os.getenv("GCS_BATCH_SIZE", GCS_DEFAULT_BATCH_SIZE))
         self.flush_interval = int(
             os.getenv("GCS_FLUSH_INTERVAL", GCS_DEFAULT_FLUSH_INTERVAL_SECONDS)
         )
-        asyncio.create_task(self.periodic_flush())
+        self.use_batched_logging = (
+            os.getenv("GCS_USE_BATCHED_LOGGING", str(GCS_DEFAULT_USE_BATCHED_LOGGING).lower()).lower() == "true"
+        )
         self.flush_lock = asyncio.Lock()
         super().__init__(
             flush_lock=self.flush_lock,
             batch_size=self.batch_size,
             flush_interval=self.flush_interval,
         )
+        self.log_queue: asyncio.Queue[GCSLogQueueItem] = asyncio.Queue()  # type: ignore[assignment]
+        asyncio.create_task(self.periodic_flush())
         AdditionalLoggingUtils.__init__(self)
 
         if premium_user is not True:
@@ -65,8 +69,7 @@ class GCSBucketLogger(GCSBucketBase, AdditionalLoggingUtils):
             )
             if logging_payload is None:
                 raise ValueError("standard_logging_object not found in kwargs")
-            # Add to logging queue - this will be flushed periodically
-            self.log_queue.append(
+            await self.log_queue.put(
                 GCSLogQueueItem(
                     payload=logging_payload, kwargs=kwargs, response_obj=response_obj
                 )
@@ -89,7 +92,9 @@ class GCSBucketLogger(GCSBucketBase, AdditionalLoggingUtils):
             if logging_payload is None:
                 raise ValueError("standard_logging_object not found in kwargs")
             # Add to logging queue - this will be flushed periodically
-            self.log_queue.append(
+            # Use asyncio.Queue.put() for thread-safe concurrent access
+            # If queue is full, this will block until space is available (backpressure)
+            await self.log_queue.put(
                 GCSLogQueueItem(
                     payload=logging_payload, kwargs=kwargs, response_obj=response_obj
                 )
@@ -98,28 +103,98 @@ class GCSBucketLogger(GCSBucketBase, AdditionalLoggingUtils):
         except Exception as e:
             verbose_logger.exception(f"GCS Bucket logging error: {str(e)}")
 
-    async def async_send_batch(self):
+    def _drain_queue_batch(self) -> List[GCSLogQueueItem]:
         """
-        Process queued logs in batch - sends logs to GCS Bucket
-
-
-        GCS Bucket does not have a Batch endpoint to batch upload logs
-
-        Instead, we
-            - collect the logs to flush every `GCS_FLUSH_INTERVAL` seconds
-            - during async_send_batch, we make 1 POST request per log to GCS Bucket
-
+        Drain items from the queue (non-blocking), respecting batch_size limit.
+        
+        This prevents unbounded queue growth when processing is slower than log accumulation.
+        
+        Returns:
+            List of items to process, up to batch_size items
         """
-        if not self.log_queue:
-            return
+        items_to_process: List[GCSLogQueueItem] = []
+        while len(items_to_process) < self.batch_size:
+            try:
+                items_to_process.append(self.log_queue.get_nowait())
+            except asyncio.QueueEmpty:
+                break
+        return items_to_process
 
-        for log_item in self.log_queue:
-            logging_payload = log_item["payload"]
-            kwargs = log_item["kwargs"]
-            response_obj = log_item.get("response_obj", None) or {}
+    def _generate_batch_object_name(self, date_str: str, batch_id: str) -> str:
+        """
+        Generate object name for a batched log file.
+        Format: {date}/batch-{batch_id}.ndjson
+        """
+        return f"{date_str}/batch-{batch_id}.ndjson"
 
+    def _get_config_key(self, kwargs: Dict[str, Any]) -> str:
+        """
+        Extract a synchronous grouping key from kwargs to group items by GCS config.
+        This allows us to batch items with the same bucket/credentials together.
+        
+        Returns a string key that uniquely identifies the GCS config combination.
+        This key may contain sensitive information (bucket names, paths) - use _sanitize_config_key()
+        for logging purposes.
+        """
+        standard_callback_dynamic_params = kwargs.get("standard_callback_dynamic_params", None) or {}
+        
+        bucket_name = standard_callback_dynamic_params.get("gcs_bucket_name", None) or self.BUCKET_NAME or "default"
+        path_service_account = standard_callback_dynamic_params.get("gcs_path_service_account", None) or self.path_service_account_json or "default"
+        
+        return f"{bucket_name}|{path_service_account}"
+    
+    def _sanitize_config_key(self, config_key: str) -> str:
+        """
+        Create a sanitized version of the config key for logging.
+        Uses a hash to avoid exposing sensitive bucket names or service account paths.
+        
+        Returns a short hash prefix for safe logging.
+        """
+        hash_obj = hashlib.sha256(config_key.encode('utf-8'))
+        return f"config-{hash_obj.hexdigest()[:8]}"
+    
+    def _group_items_by_config(self, items: List[GCSLogQueueItem]) -> Dict[str, List[GCSLogQueueItem]]:
+        """
+        Group items by their GCS config (bucket + credentials).
+        This ensures items with different configs are processed separately.
+        
+        Returns a dict mapping config_key -> list of items with that config.
+        """
+        grouped: Dict[str, List[GCSLogQueueItem]] = {}
+        for item in items:
+            config_key = self._get_config_key(item["kwargs"])
+            if config_key not in grouped:
+                grouped[config_key] = []
+            grouped[config_key].append(item)
+        return grouped
+
+    def _combine_payloads_to_ndjson(self, items: List[GCSLogQueueItem]) -> str:
+        """
+        Combine multiple log payloads into newline-delimited JSON (NDJSON) format.
+        Each line is a valid JSON object representing one log entry.
+        """
+        lines = []
+        for item in items:
+            logging_payload = item["payload"]
+            json_line = json.dumps(logging_payload, default=str, ensure_ascii=False)
+            lines.append(json_line)
+        return "\n".join(lines)
+
+    async def _send_grouped_batch(self, items: List[GCSLogQueueItem], config_key: str) -> Tuple[int, int]:
+        """
+        Send a batch of items that share the same GCS config.
+        
+        Returns:
+            (success_count, error_count)
+        """
+        if not items:
+            return (0, 0)
+        
+        first_kwargs = items[0]["kwargs"]
+        
+        try:
             gcs_logging_config: GCSLoggingConfig = await self.get_gcs_logging_config(
-                kwargs
+                first_kwargs
             )
 
             headers = await self.construct_request_headers(
@@ -127,24 +202,92 @@ class GCSBucketLogger(GCSBucketBase, AdditionalLoggingUtils):
                 service_account_json=gcs_logging_config["path_service_account"],
             )
             bucket_name = gcs_logging_config["bucket_name"]
-            object_name = self._get_object_name(kwargs, logging_payload, response_obj)
+            
+            current_date = self._get_object_date_from_datetime(datetime.now(timezone.utc))
+            batch_id = f"{int(time.time() * 1000)}-{uuid.uuid4().hex[:8]}"
+            object_name = self._generate_batch_object_name(current_date, batch_id)
+            combined_payload = self._combine_payloads_to_ndjson(items)
+            
+            await self._log_json_data_on_gcs(
+                headers=headers,
+                bucket_name=bucket_name,
+                object_name=object_name,
+                logging_payload=combined_payload,
+            )
+            
+            success_count = len(items)
+            error_count = 0
+            return (success_count, error_count)
+            
+        except Exception as e:
+            success_count = 0
+            error_count = len(items)
+            verbose_logger.exception(
+                f"GCS Bucket error logging batch payload to GCS bucket: {str(e)}"
+            )
+            return (success_count, error_count)
 
-            try:
-                await self._log_json_data_on_gcs(
-                    headers=headers,
-                    bucket_name=bucket_name,
-                    object_name=object_name,
-                    logging_payload=logging_payload,
-                )
-            except Exception as e:
-                # don't let one log item fail the entire batch
-                verbose_logger.exception(
-                    f"GCS Bucket error logging payload to GCS bucket: {str(e)}"
-                )
-                pass
+    async def _send_individual_logs(self, items: List[GCSLogQueueItem]) -> None:
+        """
+        Send each log individually as separate GCS objects (legacy behavior).
+        This is used when GCS_USE_BATCHED_LOGGING is disabled.
+        """
+        for item in items:
+            await self._send_single_log_item(item)
 
-        # Clear the queue after processing
-        self.log_queue.clear()
+    async def _send_single_log_item(self, item: GCSLogQueueItem) -> None:
+        """
+        Send a single log item to GCS as an individual object.
+        """
+        try:
+            gcs_logging_config: GCSLoggingConfig = await self.get_gcs_logging_config(
+                item["kwargs"]
+            )
+
+            headers = await self.construct_request_headers(
+                vertex_instance=gcs_logging_config["vertex_instance"],
+                service_account_json=gcs_logging_config["path_service_account"],
+            )
+            bucket_name = gcs_logging_config["bucket_name"]
+            
+            object_name = self._get_object_name(
+                kwargs=item["kwargs"],
+                logging_payload=item["payload"],
+                response_obj=item["response_obj"],
+            )
+            
+            await self._log_json_data_on_gcs(
+                headers=headers,
+                bucket_name=bucket_name,
+                object_name=object_name,
+                logging_payload=item["payload"],
+            )
+        except Exception as e:
+            verbose_logger.exception(
+                f"GCS Bucket error logging individual payload to GCS bucket: {str(e)}"
+            )
+
+    async def async_send_batch(self):
+        """
+        Process queued logs - sends logs to GCS Bucket.
+
+        If `GCS_USE_BATCHED_LOGGING` is enabled (default), batches multiple log payloads
+        into single GCS object uploads (NDJSON format), dramatically reducing API calls.
+
+        If disabled, sends each log individually as separate GCS objects (legacy behavior).
+        """
+        items_to_process = self._drain_queue_batch()
+
+        if not items_to_process:
+            return
+
+        if self.use_batched_logging:
+            grouped_items = self._group_items_by_config(items_to_process)
+            
+            for config_key, group_items in grouped_items.items():
+                await self._send_grouped_batch(group_items, config_key)
+        else:
+            await self._send_individual_logs(items_to_process)
 
     def _get_object_name(
         self, kwargs: Dict, logging_payload: StandardLoggingPayload, response_obj: Any
@@ -186,7 +329,6 @@ class GCSBucketLogger(GCSBucketBase, AdditionalLoggingUtils):
                 "start_time_utc is required for getting a payload from GCS Bucket"
             )
 
-        # Try current day, next day, and previous day
         dates_to_try = [
             start_time_utc,
             start_time_utc + timedelta(days=1),
@@ -230,5 +372,23 @@ class GCSBucketLogger(GCSBucketBase, AdditionalLoggingUtils):
     def _get_object_date_from_datetime(self, datetime_obj: datetime) -> str:
         return datetime_obj.strftime("%Y-%m-%d")
 
+    async def flush_queue(self):
+        """
+        Override flush_queue to work with asyncio.Queue.
+        """
+        await self.async_send_batch()
+        self.last_flush_time = time.time()
+
+    async def periodic_flush(self):
+        """
+        Override periodic_flush to work with asyncio.Queue.
+        """
+        while True:
+            await asyncio.sleep(self.flush_interval)
+            verbose_logger.debug(
+                f"GCS Bucket periodic flush after {self.flush_interval} seconds"
+            )
+            await self.flush_queue()
+
     async def async_health_check(self) -> IntegrationHealthCheckStatus:
         raise NotImplementedError("GCS Bucket does not support health check")

litellm/litellm_core_utils/prompt_templates/common_utils.py

@@ -1071,9 +1071,9 @@ def _extract_reasoning_content(message: dict) -> Tuple[Optional[str], Optional[s
     """
     message_content = message.get("content")
     if "reasoning_content" in message:
-        return message["reasoning_content"], message["content"]
+        return message["reasoning_content"], message_content
     elif "reasoning" in message:
-        return message["reasoning"], message["content"]
+        return message["reasoning"], message_content
     elif isinstance(message_content, str):
         return _parse_content_for_reasoning(message_content)
     return None, message_content

litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py

@@ -50,6 +50,12 @@ class AmazonAnthropicClaudeMessagesConfig(
 
     DEFAULT_BEDROCK_ANTHROPIC_API_VERSION = "bedrock-2023-05-31"
 
+    # Beta header patterns that are not supported by Bedrock Invoke API
+    # These will be filtered out to prevent 400 "invalid beta flag" errors
+    UNSUPPORTED_BEDROCK_INVOKE_BETA_PATTERNS = [
+        "advanced-tool-use",  # Bedrock Invoke doesn't support advanced-tool-use beta headers
+    ]
+
     def __init__(self, **kwargs):
         BaseAnthropicMessagesConfig.__init__(self, **kwargs)
         AmazonInvokeConfig.__init__(self, **kwargs)
@@ -114,7 +120,7 @@ class AmazonAnthropicClaudeMessagesConfig(
         """
         Remove `ttl` field from cache_control in messages.
         Bedrock doesn't support the ttl field in cache_control.
-        
+
         Args:
             anthropic_messages_request: The request dictionary to modify in-place
         """
@@ -129,6 +135,75 @@ class AmazonAnthropicClaudeMessagesConfig(
                                 if isinstance(cache_control, dict) and "ttl" in cache_control:
                                     cache_control.pop("ttl", None)
 
+    def _supports_extended_thinking_on_bedrock(self, model: str) -> bool:
+        """
+        Check if the model supports extended thinking beta headers on Bedrock.
+
+        On 3rd-party platforms (e.g., Amazon Bedrock), extended thinking is only
+        supported on: Claude Opus 4.5, Claude Opus 4.1, Opus 4, or Sonnet 4.
+
+        Ref: https://docs.anthropic.com/en/docs/build-with-claude/extended-thinking
+
+        Args:
+            model: The model name
+
+        Returns:
+            True if the model supports extended thinking on Bedrock
+        """
+        model_lower = model.lower()
+
+        # Supported models on Bedrock for extended thinking
+        supported_patterns = [
+            "opus-4.5", "opus_4.5", "opus-4-5", "opus_4_5",  # Opus 4.5
+            "opus-4.1", "opus_4.1", "opus-4-1", "opus_4_1",  # Opus 4.1
+            "opus-4", "opus_4",                               # Opus 4
+            "sonnet-4", "sonnet_4",                           # Sonnet 4
+        ]
+
+        return any(pattern in model_lower for pattern in supported_patterns)
+
+    def _filter_unsupported_beta_headers_for_bedrock(
+        self, model: str, beta_set: set
+    ) -> None:
+        """
+        Remove beta headers that are not supported on Bedrock for the given model.
+
+        Extended thinking beta headers are only supported on specific Claude 4+ models.
+        Advanced tool use headers are not supported on Bedrock Invoke API.
+        This prevents 400 "invalid beta flag" errors on Bedrock.
+
+        Note: Bedrock Invoke API fails with a 400 error when unsupported beta headers
+        are sent, returning: {"message":"invalid beta flag"}
+
+        Args:
+            model: The model name
+            beta_set: The set of beta headers to filter in-place
+        """
+        beta_headers_to_remove = set()
+
+        # 1. Filter out beta headers that are universally unsupported on Bedrock Invoke
+        for beta in beta_set:
+            for unsupported_pattern in self.UNSUPPORTED_BEDROCK_INVOKE_BETA_PATTERNS:
+                if unsupported_pattern in beta.lower():
+                    beta_headers_to_remove.add(beta)
+                    break
+
+        # 2. Filter out extended thinking headers for models that don't support them
+        extended_thinking_patterns = [
+            "extended-thinking",
+            "interleaved-thinking",
+        ]
+        if not self._supports_extended_thinking_on_bedrock(model):
+            for beta in beta_set:
+                for pattern in extended_thinking_patterns:
+                    if pattern in beta.lower():
+                        beta_headers_to_remove.add(beta)
+                        break
+
+        # Remove all filtered headers
+        for beta in beta_headers_to_remove:
+            beta_set.discard(beta)
+
     def _get_tool_search_beta_header_for_bedrock(
         self,
         model: str,
@@ -139,15 +214,15 @@ class AmazonAnthropicClaudeMessagesConfig(
     ) -> None:
         """
         Adjust tool search beta header for Bedrock.
-        
+
         Bedrock requires a different beta header for tool search on Opus 4 models
         when tool search is used without programmatic tool calling or input examples.
-        
+
         Note: On Amazon Bedrock, server-side tool search is only supported on Claude Opus 4
         with the `tool-search-tool-2025-10-19` beta header.
-        
+
         Ref: https://platform.claude.com/docs/en/agents-and-tools/tool-use/tool-search-tool
-        
+
         Args:
             model: The model name
             tool_search_used: Whether tool search is used
@@ -228,6 +303,12 @@ class AmazonAnthropicClaudeMessagesConfig(
             beta_set=beta_set,
         )
 
+        # Filter out unsupported beta headers for Bedrock (e.g., advanced-tool-use, extended-thinking on non-Opus/Sonnet 4 models)
+        self._filter_unsupported_beta_headers_for_bedrock(
+            model=model,
+            beta_set=beta_set,
+        )
+
         if beta_set:
             anthropic_messages_request["anthropic_beta"] = list(beta_set)
         

litellm/main.py

@@ -7255,4 +7255,4 @@ def __getattr__(name: str) -> Any:
         global _encoding_cache
         _encoding_cache = _encoding
         return _encoding
-    raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
+    raise AttributeError(f"module {__name__!r} has no attribute {name!r}")

litellm/model_prices_and_context_window_backup.json

@@ -18826,13 +18826,14 @@
         "supports_tool_choice": true
     },
     "groq/openai/gpt-oss-120b": {
+        "cache_read_input_token_cost": 7.5e-08,
         "input_cost_per_token": 1.5e-07,
         "litellm_provider": "groq",
         "max_input_tokens": 131072,
         "max_output_tokens": 32766,
         "max_tokens": 32766,
         "mode": "chat",
-        "output_cost_per_token": 7.5e-07,
+        "output_cost_per_token": 6e-07,
         "supports_function_calling": true,
         "supports_parallel_function_calling": true,
         "supports_reasoning": true,
@@ -18841,13 +18842,14 @@
         "supports_web_search": true
     },
     "groq/openai/gpt-oss-20b": {
-        "input_cost_per_token": 1e-07,
+        "cache_read_input_token_cost": 3.75e-08,
+        "input_cost_per_token": 7.5e-08,
         "litellm_provider": "groq",
         "max_input_tokens": 131072,
         "max_output_tokens": 32768,
         "max_tokens": 32768,
         "mode": "chat",
-        "output_cost_per_token": 5e-07,
+        "output_cost_per_token": 3e-07,
         "supports_function_calling": true,
         "supports_parallel_function_calling": true,
         "supports_reasoning": true,
@@ -26718,15 +26720,15 @@
         "tool_use_system_prompt_tokens": 159
     },
     "us.anthropic.claude-opus-4-5-20251101-v1:0": {
-        "cache_creation_input_token_cost": 6.25e-06,
-        "cache_read_input_token_cost": 5e-07,
-        "input_cost_per_token": 5e-06,
+        "cache_creation_input_token_cost": 6.875e-06,
+        "cache_read_input_token_cost": 5.5e-07,
+        "input_cost_per_token": 5.5e-06,
         "litellm_provider": "bedrock_converse",
         "max_input_tokens": 200000,
         "max_output_tokens": 64000,
         "max_tokens": 64000,
         "mode": "chat",
-        "output_cost_per_token": 2.5e-05,
+        "output_cost_per_token": 2.75e-05,
         "search_context_cost_per_query": {
             "search_context_size_high": 0.01,
             "search_context_size_low": 0.01,

litellm/proxy/pass_through_endpoints/pass_through_endpoints.py

@@ -51,6 +51,7 @@ from litellm.proxy._types import (
 from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
 from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing
 from litellm.proxy.common_utils.http_parsing_utils import _read_request_body
+from litellm.proxy.utils import get_server_root_path
 from litellm.secret_managers.main import get_secret_str
 from litellm.types.llms.custom_http import httpxSpecialProvider
 from litellm.types.passthrough_endpoints.pass_through_endpoints import (
@@ -1973,10 +1974,26 @@ class InitPassThroughEndpointHelpers:
         _registered_pass_through_routes.clear()
 
     @staticmethod
-    def get_registered_pass_through_endpoints_keys() -> List[str]:
+    def get_all_registered_pass_through_routes() -> List[str]:
         """Get all registered pass-through endpoints from the registry"""
         return list(_registered_pass_through_routes.keys())
 
+    @staticmethod
+    def _build_full_path_with_root(path: str) -> str:
+        """
+        Build full path by prepending server root path if needed.
+
+        Args:
+            path: The relative path to build
+
+        Returns:
+            Full path with server root prepended (if root is not "/")
+        """
+        root_path = get_server_root_path()
+        if root_path == "/":
+            return path
+        return f"{root_path}{path}"
+
     @staticmethod
     def is_registered_pass_through_route(route: str) -> bool:
         """
@@ -2003,7 +2020,9 @@ class InitPassThroughEndpointHelpers:
             parts = key.split(":", 2)  # Split into [endpoint_id, type, path]
             if len(parts) == 3:
                 route_type = parts[1]
-                registered_path = parts[2]
+                registered_path = InitPassThroughEndpointHelpers._build_full_path_with_root(
+                    parts[2]
+                )
                 if route_type == "exact" and route == registered_path:
                     return True
                 elif route_type == "subpath":
@@ -2021,7 +2040,9 @@ class InitPassThroughEndpointHelpers:
             parts = key.split(":", 2)  # Split into [endpoint_id, type, path]
             if len(parts) == 3:
                 route_type = parts[1]
-                registered_path = parts[2]
+                registered_path = InitPassThroughEndpointHelpers._build_full_path_with_root(
+                    parts[2]
+                )
 
                 if route_type == "exact" and route == registered_path:
                     return _registered_pass_through_routes[key]
@@ -2085,7 +2106,7 @@ async def initialize_pass_through_endpoints(
     # mark the ones that are visited in the list
     # remove the ones that are not visited from the list
     registered_pass_through_endpoints = (
-        InitPassThroughEndpointHelpers.get_registered_pass_through_endpoints_keys()
+        InitPassThroughEndpointHelpers.get_all_registered_pass_through_routes()
     )
 
     visited_endpoints = set()

litellm/types/integrations/gcs_bucket.py

@@ -12,6 +12,7 @@ else:
 
 GCS_DEFAULT_BATCH_SIZE = 2048
 GCS_DEFAULT_FLUSH_INTERVAL_SECONDS = 20
+GCS_DEFAULT_USE_BATCHED_LOGGING = True
 
 
 class GCSLoggingConfig(TypedDict):

model_prices_and_context_window.json

@@ -18826,13 +18826,14 @@
         "supports_tool_choice": true
     },
     "groq/openai/gpt-oss-120b": {
+        "cache_read_input_token_cost": 7.5e-08,
         "input_cost_per_token": 1.5e-07,
         "litellm_provider": "groq",
         "max_input_tokens": 131072,
         "max_output_tokens": 32766,
         "max_tokens": 32766,
         "mode": "chat",
-        "output_cost_per_token": 7.5e-07,
+        "output_cost_per_token": 6e-07,
         "supports_function_calling": true,
         "supports_parallel_function_calling": true,
         "supports_reasoning": true,
@@ -18841,13 +18842,14 @@
         "supports_web_search": true
     },
     "groq/openai/gpt-oss-20b": {
-        "input_cost_per_token": 1e-07,
+        "cache_read_input_token_cost": 3.75e-08,
+        "input_cost_per_token": 7.5e-08,
         "litellm_provider": "groq",
         "max_input_tokens": 131072,
         "max_output_tokens": 32768,
         "max_tokens": 32768,
         "mode": "chat",
-        "output_cost_per_token": 5e-07,
+        "output_cost_per_token": 3e-07,
         "supports_function_calling": true,
         "supports_parallel_function_calling": true,
         "supports_reasoning": true,
@@ -26718,15 +26720,15 @@
         "tool_use_system_prompt_tokens": 159
     },
     "us.anthropic.claude-opus-4-5-20251101-v1:0": {
-        "cache_creation_input_token_cost": 6.25e-06,
-        "cache_read_input_token_cost": 5e-07,
-        "input_cost_per_token": 5e-06,
+        "cache_creation_input_token_cost": 6.875e-06,
+        "cache_read_input_token_cost": 5.5e-07,
+        "input_cost_per_token": 5.5e-06,
         "litellm_provider": "bedrock_converse",
         "max_input_tokens": 200000,
         "max_output_tokens": 64000,
         "max_tokens": 64000,
         "mode": "chat",
-        "output_cost_per_token": 2.5e-05,
+        "output_cost_per_token": 2.75e-05,
         "search_context_cost_per_query": {
             "search_context_size_high": 0.01,
             "search_context_size_low": 0.01,

scripts/health_check/health_check_client.py

@@ -0,0 +1,406 @@
+#!/usr/bin/env python3
+"""
+LiteLLM Health Check Client
+
+A sentinel health check tool that tests all configured models on a LiteLLM proxy.
+This script:
+- Can read models from YAML config file or fetch from proxy API
+- Sends a simple test request to each model concurrently
+- Reports health status for each model
+- Supports both chat/completion and embedding models
+"""
+
+import asyncio
+import json
+import os
+import sys
+import time
+from typing import Dict, List, Optional, Tuple
+
+import httpx
+import yaml
+
+
+class LiteLLMHealthCheckClient:
+    """Client for health checking LiteLLM proxy models."""
+
+    def __init__(
+        self,
+        base_url: str,
+        api_key: str,
+        timeout: int = 120,  # Match Go implementation's 120s timeout
+        completion_prompt: str = "Say this is a test",  # Match Go implementation
+        embedding_text: str = "This is a test for vectorization.",  # Match Go implementation
+    ):
+        """
+        Initialize the health check client.
+
+        Args:
+            base_url: Base URL of the LiteLLM proxy (e.g., https://litellm.example.com)
+            api_key: API key for authentication
+            timeout: Request timeout in seconds (default: 120, matching Go implementation)
+            completion_prompt: Test prompt for chat/completion models
+            embedding_text: Test text for embedding models
+        """
+        self.base_url = base_url.rstrip("/")
+        self.api_key = api_key
+        self.timeout = timeout
+        self.completion_prompt = completion_prompt
+        self.embedding_text = embedding_text
+        self.headers = {
+            "Authorization": f"Bearer {api_key}",
+            "Content-Type": "application/json",
+        }
+
+    def load_models_from_yaml(self, yaml_path: str) -> List[Dict]:
+        """
+        Load models from a YAML config file (similar to Go implementation).
+
+        Args:
+            yaml_path: Path to the YAML config file
+
+        Returns:
+            List of model dictionaries with 'id' and 'mode' keys
+        """
+        try:
+            with open(yaml_path, "r") as f:
+                config = yaml.safe_load(f)
+
+            model_list = config.get("model_list", [])
+            models = []
+
+            for entry in model_list:
+                model_name = entry.get("model_name", "")
+                litellm_params = entry.get("litellm_params", {})
+                model_info = litellm_params.get("model_info", {})
+                mode = model_info.get("mode", "")
+
+                # Use model_name as the ID (this is what gets sent to the API)
+                models.append(
+                    {
+                        "id": model_name,
+                        "mode": mode.lower() if mode else "",
+                        "provider": model_info.get("provider", ""),
+                    }
+                )
+
+            return models
+        except Exception as e:
+            print(f"Error loading models from YAML file {yaml_path}: {e}", file=sys.stderr)
+            return []
+
+    async def fetch_models(self, client: httpx.AsyncClient) -> List[Dict]:
+        """
+        Fetch all available models from the proxy API.
+
+        Returns:
+            List of model dictionaries with 'id' and 'mode' keys
+        """
+        try:
+            # Try /v1/models first (OpenAI-compatible endpoint)
+            response = await client.get(
+                f"{self.base_url}/v1/models",
+                headers=self.headers,
+                timeout=self.timeout,
+            )
+            response.raise_for_status()
+            data = response.json()
+            models_data = data.get("data", [])
+            models = []
+            for m in models_data:
+                models.append({"id": m["id"], "mode": "", "provider": ""})
+            return models
+        except Exception as e:
+            print(f"Error fetching models from /v1/models: {e}", file=sys.stderr)
+            # Fallback to /model/info endpoint which has more details
+            try:
+                response = await client.get(
+                    f"{self.base_url}/model/info",
+                    headers=self.headers,
+                    timeout=self.timeout,
+                )
+                response.raise_for_status()
+                data = response.json()
+                if isinstance(data, dict) and "data" in data:
+                    models_data = data["data"]
+                elif isinstance(data, list):
+                    models_data = data
+                else:
+                    models_data = []
+
+                models = []
+                for m in models_data:
+                    model_info = m.get("model_info", {})
+                    mode = model_info.get("mode", "")
+                    models.append(
+                        {
+                            "id": m.get("model_name", m.get("id", "unknown")),
+                            "mode": mode.lower() if mode else "",
+                            "provider": model_info.get("provider", ""),
+                        }
+                    )
+                return models
+            except Exception as e2:
+                print(f"Error fetching models from /model/info: {e2}", file=sys.stderr)
+                return []
+
+    async def check_model_health(
+        self, client: httpx.AsyncClient, model: Dict
+    ) -> Tuple[str, Dict]:
+        """
+        Check health of a single model by sending a test request.
+
+        Args:
+            client: HTTP client
+            model: Model dictionary with 'id' and 'mode' keys
+
+        Returns:
+            Tuple of (model_id, result_dict)
+        """
+        model_id = model["id"]
+        mode = model.get("mode", "")
+
+        start_time = time.time()
+        result = {
+            "model": model_id,
+            "healthy": False,
+            "error": None,
+            "response_time_ms": None,
+            "mode": mode,
+        }
+
+        try:
+            # Determine if this is an embedding model
+            # Check mode first (from config), then fall back to name-based detection
+            is_embedding = (
+                mode == "embedding"
+                or any(
+                    keyword in model_id.lower()
+                    for keyword in ["embedding", "embed", "text-embedding"]
+                )
+            )
+
+            if is_embedding:
+                # Test embedding endpoint (matching Go implementation)
+                embedding_response = await client.post(
+                    f"{self.base_url}/v1/embeddings",
+                    headers=self.headers,
+                    json={
+                        "model": model_id,
+                        "input": self.embedding_text,
+                    },
+                    timeout=self.timeout,
+                )
+                embedding_response.raise_for_status()
+                embedding_data = embedding_response.json()
+                dimensions = 0
+                if "data" in embedding_data and len(embedding_data["data"]) > 0:
+                    dimensions = len(embedding_data["data"][0].get("embedding", []))
+
+                result["healthy"] = True
+                result["mode"] = "embedding"
+                result["dimensions"] = dimensions
+            else:
+                # Test chat completion endpoint (matching Go implementation)
+                completion_response = await client.post(
+                    f"{self.base_url}/v1/chat/completions",
+                    headers=self.headers,
+                    json={
+                        "model": model_id,
+                        "messages": [
+                            {"role": "user", "content": self.completion_prompt}
+                        ],
+                        "max_tokens": 10,  # Minimal tokens for health check
+                    },
+                    timeout=self.timeout,
+                )
+                completion_response.raise_for_status()
+                completion_data = completion_response.json()
+                response_text = ""
+                if "choices" in completion_data and len(completion_data["choices"]) > 0:
+                    response_text = (
+                        completion_data["choices"][0]
+                        .get("message", {})
+                        .get("content", "")
+                    )
+
+                result["healthy"] = True
+                result["mode"] = "chat"
+                result["response_text"] = response_text[:100]  # Truncate for display
+
+            elapsed_ms = (time.time() - start_time) * 1000
+            result["response_time_ms"] = round(elapsed_ms, 2)
+
+        except httpx.HTTPStatusError as e:
+            result["error"] = f"HTTP {e.response.status_code}: {e.response.text[:200]}"
+        except httpx.TimeoutException:
+            result["error"] = f"Request timeout after {self.timeout}s"
+        except Exception as e:
+            result["error"] = str(e)[:200]
+
+        return model_id, result
+
+    async def run_health_checks(
+        self,
+        models: Optional[List[Dict]] = None,
+        models_only: Optional[List[str]] = None,
+    ) -> Dict[str, Dict]:
+        """
+        Run health checks on all models concurrently.
+
+        Args:
+            models: Optional list of models to check. If None, fetches from proxy.
+            models_only: Optional list of model IDs to check. If set, only these
+                models are health-checked (must exist in the models list).
+
+        Returns:
+            Dictionary mapping model_id to health check result
+        """
+        async with httpx.AsyncClient() as client:
+            if models is None:
+                models = await self.fetch_models(client)
+
+            if not models:
+                print("No models found to health check", file=sys.stderr)
+                return {}
+
+            if models_only:
+                allowlist = {m.strip() for m in models_only if m and m.strip()}
+                models = [m for m in models if m.get("id") in allowlist]
+                print(
+                    f"Filtering to only check {len(models)} models: {', '.join(sorted(allowlist))}",
+                    file=sys.stderr,
+                )
+                if not models:
+                    print(
+                        "No models matched LITELLM_MODELS_ONLY filter",
+                        file=sys.stderr,
+                    )
+                    return {}
+
+            print(f"Running health checks on {len(models)} models...", file=sys.stderr)
+
+            # Run all health checks concurrently
+            tasks = [self.check_model_health(client, model) for model in models]
+            results_list = await asyncio.gather(*tasks, return_exceptions=True)
+
+            # Convert to dictionary format
+            results = {}
+            for result in results_list:
+                if isinstance(result, Exception):
+                    print(
+                        f"Exception in health check task: {result}", file=sys.stderr
+                    )
+                    continue
+                # Type narrowing: after checking it's not an Exception, it's a Tuple
+                if isinstance(result, tuple) and len(result) == 2:
+                    model_id, result_dict = result
+                    results[model_id] = result_dict
+
+            return results
+
+    def print_results(self, results: Dict[str, Dict], json_output: bool = False):
+        """
+        Print health check results.
+
+        Args:
+            results: Dictionary of health check results
+            json_output: If True, output as JSON
+        """
+        if json_output:
+            print(json.dumps(results, indent=2))
+            return
+
+        healthy_count = sum(1 for r in results.values() if r.get("healthy"))
+        unhealthy_count = len(results) - healthy_count
+
+        # Print detailed results for each model (matching Go output format)
+        print(f"\n{'='*60}", file=sys.stderr)
+        print(f"Starting health check queries\n", file=sys.stderr)
+
+        for model_id, result in results.items():
+            if result.get("healthy"):
+                if result.get("mode") == "embedding":
+                    dimensions = result.get("dimensions", 0)
+                    print(
+                        f"---- {model_id} ----\n✅ Success. "
+                        f"Generated embedding vector with {dimensions} dimensions.\n\n",
+                        file=sys.stderr,
+                    )
+                else:
+                    response_text = result.get("response_text", "")
+                    print(
+                        f"---- {model_id} ----\n✅ Success. "
+                        f"Response:\n{response_text}\n\n",
+                        file=sys.stderr,
+                    )
+            else:
+                error = result.get("error", "Unknown error")
+                print(f"---- {model_id} ----\n❌ ERROR: {error}\n\n", file=sys.stderr)
+
+        print(f"{'='*60}", file=sys.stderr)
+        print(f"Health Check Summary", file=sys.stderr)
+        print(f"{'='*60}", file=sys.stderr)
+        print(f"Total models: {len(results)}", file=sys.stderr)
+        print(f"Healthy: {healthy_count}", file=sys.stderr)
+        print(f"Unhealthy: {unhealthy_count}", file=sys.stderr)
+        print(f"{'='*60}\n", file=sys.stderr)
+
+        # Exit with non-zero code if any models are unhealthy
+        if unhealthy_count > 0:
+            sys.exit(1)
+        else:
+            sys.exit(0)
+
+
+async def main():
+    """Main entry point."""
+    base_url = os.environ.get("LITELLM_BASE_URL", "http://localhost:4000")
+    api_key = os.environ.get("LITELLM_API_KEY", "sk-1234")
+    yaml_path = os.environ.get("LITELLM_MODELS_YAML")
+
+    if not base_url:
+        print("Error: LITELLM_BASE_URL environment variable not set", file=sys.stderr)
+        sys.exit(1)
+
+    if not api_key:
+        print("Error: LITELLM_API_KEY environment variable not set", file=sys.stderr)
+        sys.exit(1)
+
+    timeout = int(os.environ.get("LITELLM_TIMEOUT", "120"))  # Match Go's 120s default
+    completion_prompt = os.environ.get(
+        "LITELLM_COMPLETION_PROMPT", "Say this is a test"
+    )
+    embedding_text = os.environ.get(
+        "LITELLM_EMBEDDING_TEXT", "This is a test for vectorization."
+    )
+    json_output = os.environ.get("LITELLM_JSON_OUTPUT", "").lower() == "true"
+    # Optional: only health-check these model IDs (comma-separated). E.g.:
+    # LITELLM_MODELS_ONLY=claude-3.7-sonnet,claude-3.5-sonnet,claude-4.5-haiku
+    models_only_raw = os.environ.get("LITELLM_MODELS_ONLY", "")
+    models_only = [m.strip() for m in models_only_raw.split(",") if m.strip()] or None
+
+    client = LiteLLMHealthCheckClient(
+        base_url=base_url,
+        api_key=api_key,
+        timeout=timeout,
+        completion_prompt=completion_prompt,
+        embedding_text=embedding_text,
+    )
+
+    # Load models from YAML if provided, otherwise fetch from API
+    models = None
+    if yaml_path:
+        models = client.load_models_from_yaml(yaml_path)
+        if models:
+            print(
+                f"Successfully loaded {len(models)} models from {yaml_path}",
+                file=sys.stderr,
+            )
+
+    results = await client.run_health_checks(models=models, models_only=models_only)
+    client.print_results(results, json_output=json_output)
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

scripts/health_check/health_check_client_README.md

@@ -0,0 +1,246 @@
+# LiteLLM Health Check Client
+
+A health check tool for testing all configured models on a LiteLLM proxy. Tests each model with completion/embedding requests and reports health status, errors, and response times.
+
+## Features
+
+- **YAML Config Support**: Reads models from YAML config file OR fetches from proxy API
+- **Smart Mode Detection**: Detects embedding vs chat models from config or model name
+- **Concurrent Testing**: Tests all models concurrently using asyncio
+- **Containerized**: Docker image for easy deployment
+- **Parallel Execution**: Supports parallel execution for stress testing
+- **Configurable**: Customizable timeouts (default 120s) and test prompts
+
+## Quick Start
+
+### As a Python Script
+
+**Option 1: Fetch models from proxy API**
+```bash
+export LITELLM_BASE_URL="https://litellm.example.com"
+export LITELLM_API_KEY="your-api-key"
+python scripts/health_check/health_check_client.py
+```
+
+**Option 2: Use YAML config file**
+```bash
+export LITELLM_BASE_URL="https://litellm.example.com"
+export LITELLM_API_KEY="your-api-key"
+export LITELLM_MODELS_YAML="/path/to/config.yaml"
+python scripts/health_check/health_check_client.py
+```
+
+### As a Docker Container
+
+1. Build the Docker image:
+
+```bash
+docker build -f docker/Dockerfile.health_check -t litellm/litellm-health-check:latest .
+```
+
+2. Run a single health check:
+
+```bash
+docker run --rm \
+  -e LITELLM_BASE_URL="https://litellm.example.com" \
+  -e LITELLM_API_KEY="your-api-key" \
+  litellm/litellm-health-check:latest
+```
+
+### Parallel Execution (Stress Testing)
+
+Run multiple health check containers in parallel:
+
+**PowerShell:**
+```powershell
+$env:LITELLM_BASE_URL="https://litellm.example.com"
+$env:LITELLM_API_KEY="your-api-key"
+.\scripts\health_check\run_parallel_health_checks.ps1 16
+```
+
+**Bash/Shell:**
+```bash
+export LITELLM_BASE_URL="https://litellm.example.com"
+export LITELLM_API_KEY="your-api-key"
+./scripts/health_check/run_parallel_health_checks.sh 16
+```
+
+
+## Configuration
+
+### Environment Variables
+
+- `LITELLM_BASE_URL` (required): Base URL of the LiteLLM proxy
+  - Example: `https://litellm.example.com`
+- `LITELLM_API_KEY` (required): API key for authentication
+- `LITELLM_MODELS_YAML` (optional): Path to YAML config file with model_list
+  - If provided, reads models from YAML instead of fetching from API
+  - Example: `/path/to/config.yaml`
+- `LITELLM_TIMEOUT` (optional): Request timeout in seconds (default: 120)
+- `LITELLM_COMPLETION_PROMPT` (optional): Test prompt for chat/completion models (default: "Say this is a test")
+- `LITELLM_EMBEDDING_TEXT` (optional): Test text for embedding models (default: "This is a test for vectorization.")
+- `LITELLM_JSON_OUTPUT` (optional): Output results as JSON (default: false)
+
+## Output
+
+### Standard Output (Human-Readable)
+
+Example output format:
+
+```
+============================================================
+Starting health check queries
+
+---- gpt-4o ----
+✅ Success. Response:
+This is a test
+
+---- text-embedding-3-small ----
+✅ Success. Generated embedding vector with 1536 dimensions.
+
+---- gpt-5-codex ----
+❌ ERROR: HTTP 503: Service unavailable
+
+============================================================
+Health Check Summary
+============================================================
+Total models: 47
+Healthy: 45
+Unhealthy: 2
+============================================================
+```
+
+Exit code: `0` if all models are healthy, `1` if any models are unhealthy.
+
+### JSON Output
+
+When `LITELLM_JSON_OUTPUT=true`, outputs JSON:
+
+```json
+{
+  "gpt-4o": {
+    "model": "gpt-4o",
+    "healthy": true,
+    "error": null,
+    "response_time_ms": 245.67,
+    "mode": "chat",
+    "response_text": "This is a test"
+  },
+  "text-embedding-3-small": {
+    "model": "text-embedding-3-small",
+    "healthy": true,
+    "error": null,
+    "response_time_ms": 123.45,
+    "mode": "embedding",
+    "dimensions": 1536
+  }
+}
+```
+
+## How It Works
+
+1. **Model Discovery**: 
+   - If `LITELLM_MODELS_YAML` is set: Reads models from YAML config file
+   - Otherwise: Queries `/v1/models` (OpenAI-compatible) or `/model/info` to get all configured models
+2. **Mode Detection**: 
+   - Checks `mode` field from YAML config, or falls back to model name patterns (embedding, embed, text-embedding)
+3. **Concurrent Testing**: 
+   - Chat models: `POST /v1/chat/completions` with configurable prompt (default: "Say this is a test")
+   - Embedding models: `POST /v1/embeddings` with configurable text (default: "This is a test for vectorization.")
+4. **Reporting**: Health status, errors, response times, and response details are reported
+
+## Use Cases
+
+### 1. Regular Health Monitoring
+
+Run as a cron job or scheduled task:
+
+```bash
+# Cron job: Run every 5 minutes
+*/5 * * * * /path/to/health_check.sh
+```
+
+### 2. Load/Stress Testing
+
+Run multiple health checks in parallel:
+
+**PowerShell:**
+```powershell
+.\scripts\health_check\run_parallel_health_checks.ps1 16
+```
+
+### 3. CI/CD Integration
+
+Add to your deployment pipeline:
+
+```yaml
+# GitHub Actions example
+- name: Health Check
+  run: |
+    docker run --rm \
+      -e LITELLM_BASE_URL="${{ secrets.LITELLM_BASE_URL }}" \
+      -e LITELLM_API_KEY="${{ secrets.LITELLM_API_KEY }}" \
+      litellm/litellm-health-check:latest
+```
+
+### 4. Kubernetes Deployment
+
+Deploy as a CronJob:
+
+```yaml
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: litellm-health-check
+spec:
+  schedule: "*/5 * * * *"  # Every 5 minutes
+  jobTemplate:
+    spec:
+      template:
+        spec:
+          containers:
+          - name: health-check
+            image: litellm/litellm-health-check:latest
+            env:
+            - name: LITELLM_BASE_URL
+              value: "https://litellm.example.com"
+            - name: LITELLM_API_KEY
+              valueFrom:
+                secretKeyRef:
+                  name: litellm-secrets
+                  key: api-key
+          restartPolicy: OnFailure
+```
+
+## Troubleshooting
+
+### No Models Found
+
+- Verify `LITELLM_BASE_URL` is correct
+- Check that the API key has permissions to list models
+- Ensure the proxy is running and accessible
+- If using YAML, verify `LITELLM_MODELS_YAML` path is correct
+
+### Timeout Errors
+
+- Increase `LITELLM_TIMEOUT` for slower models (default is 120s)
+- Check network connectivity to the proxy
+- Verify proxy isn't overloaded
+
+### Authentication Errors
+
+- Verify `LITELLM_API_KEY` is correct
+- Check API key has not expired
+- Ensure the key has necessary permissions
+
+## Dependencies
+
+- Python 3.11+
+- httpx (for async HTTP requests)
+- pyyaml (for YAML config file support)
+- Docker or Podman (for containerized execution)
+- PowerShell (for parallel execution script on Windows)
+
+## License
+
+Same as LiteLLM project.

scripts/health_check/health_check_requirements.txt

@@ -0,0 +1,2 @@
+httpx>=0.24.0
+pyyaml>=6.0

scripts/health_check/run_parallel_health_checks.ps1

@@ -0,0 +1,69 @@
+# Parallel LiteLLM Health Check Runner (PowerShell version)
+#
+# This script runs multiple health check containers in parallel.
+#
+# Usage:
+#   $env:LITELLM_BASE_URL="https://litellm.example.com"
+#   $env:LITELLM_API_KEY="your-api-key"
+#   .\run_parallel_health_checks.ps1 [num_parallel_jobs] [image_name]
+#
+# Defaults:
+#   - num_parallel_jobs: 16
+#   - image_name: litellm/litellm-health-check:latest
+
+param(
+    [int]$NumParallelJobs = 16,
+    [string]$ImageName = "litellm/litellm-health-check:latest",
+    [string]$ContainerRuntime = "docker"
+)
+
+# Set defaults for environment variables if not provided
+if (-not $env:LITELLM_BASE_URL) {
+    $env:LITELLM_BASE_URL = "https://litellm-perf-cache-and-router.onrender.com"
+    Write-Warning "LITELLM_BASE_URL not set, using default: $env:LITELLM_BASE_URL"
+}
+
+if (-not $env:LITELLM_API_KEY) {
+    $env:LITELLM_API_KEY = "sk-1234"
+    Write-Warning "LITELLM_API_KEY not set, using default: $env:LITELLM_API_KEY"
+}
+
+# Check if container runtime is available
+$runtimeExists = Get-Command $ContainerRuntime -ErrorAction SilentlyContinue
+if (-not $runtimeExists) {
+    Write-Error "Error: $ContainerRuntime is not installed"
+    exit 1
+}
+
+Write-Host "Running $NumParallelJobs parallel health check containers..." -ForegroundColor Yellow
+Write-Host "Using image: $ImageName" -ForegroundColor Yellow
+Write-Host "Container runtime: $ContainerRuntime" -ForegroundColor Yellow
+Write-Host "LiteLLM Base URL: $env:LITELLM_BASE_URL" -ForegroundColor Cyan
+Write-Host ""
+Write-Host "NOTE: This will run continuously. Press Ctrl+C to stop." -ForegroundColor Red
+Write-Host ""
+Write-Host "Troubleshooting:" -ForegroundColor Yellow
+Write-Host "  - If you see 'All connection attempts failed', check:" -ForegroundColor Yellow
+Write-Host "    1. Is the LiteLLM proxy running on the expected port?" -ForegroundColor Yellow
+Write-Host "    2. Set LITELLM_BASE_URL to the correct URL (e.g., http://host.docker.internal:PORT)" -ForegroundColor Yellow
+Write-Host "    3. On Linux, you may need to use the host IP instead of host.docker.internal" -ForegroundColor Yellow
+Write-Host ""
+
+# Run parallel health checks
+# This creates an infinite loop that keeps spawning containers
+# Each container tests all models, then exits, and a new one starts
+while ($true) {
+    # Start up to NumParallelJobs containers in parallel
+    1..$NumParallelJobs | ForEach-Object -Parallel {
+        $runtime = $using:ContainerRuntime
+        $imageName = $using:ImageName
+        $baseUrl = $env:LITELLM_BASE_URL
+        $apiKey = $env:LITELLM_API_KEY
+        
+        & $runtime run --rm `
+            -e LITELLM_BASE_URL="$baseUrl" `
+            -e LITELLM_API_KEY="$apiKey" `
+            -e LITELLM_JSON_OUTPUT="true" `
+            $imageName
+    } -ThrottleLimit $NumParallelJobs
+}

scripts/health_check/run_parallel_health_checks.sh

@@ -0,0 +1,79 @@
+#!/bin/bash
+# Parallel LiteLLM Health Check Runner (Bash version)
+#
+# This script runs multiple health check containers in parallel.
+#
+# Usage:
+#   export LITELLM_BASE_URL="https://litellm.example.com"
+#   export LITELLM_API_KEY="your-api-key"
+#   ./run_parallel_health_checks.sh [num_parallel_jobs] [image_name] [container_runtime]
+#
+# Defaults:
+#   - num_parallel_jobs: 16
+#   - image_name: litellm/litellm-health-check:latest
+#   - container_runtime: docker
+
+set -e
+
+# Default values
+NUM_PARALLEL_JOBS="${1:-16}"
+IMAGE_NAME="${2:-litellm/litellm-health-check:latest}"
+CONTAINER_RUNTIME="${3:-docker}"
+
+# Set defaults for environment variables if not provided
+if [ -z "$LITELLM_BASE_URL" ]; then
+    export LITELLM_BASE_URL="https://litellm-perf-cache-and-router.onrender.com"
+    echo "Warning: LITELLM_BASE_URL not set, using default: $LITELLM_BASE_URL" >&2
+fi
+
+if [ -z "$LITELLM_API_KEY" ]; then
+    export LITELLM_API_KEY="sk-1234"
+    echo "Warning: LITELLM_API_KEY not set, using default: $LITELLM_API_KEY" >&2
+fi
+
+# Check if container runtime is available
+if ! command -v "$CONTAINER_RUNTIME" &> /dev/null; then
+    echo "Error: $CONTAINER_RUNTIME is not installed" >&2
+    exit 1
+fi
+
+# Print configuration
+echo "Running $NUM_PARALLEL_JOBS parallel health check containers..."
+echo "Using image: $IMAGE_NAME"
+echo "Container runtime: $CONTAINER_RUNTIME"
+echo "LiteLLM Base URL: $LITELLM_BASE_URL"
+echo ""
+echo "NOTE: This will run continuously. Press Ctrl+C to stop."
+echo ""
+echo "Troubleshooting:"
+echo "  - If you see 'All connection attempts failed', check:"
+echo "    1. Is the LiteLLM proxy running on the expected port?"
+echo "    2. Set LITELLM_BASE_URL to the correct URL (e.g., http://host.docker.internal:PORT)"
+echo "    3. On Linux, you may need to use the host IP instead of host.docker.internal"
+echo ""
+
+# Function to run a single health check container
+run_health_check() {
+    "$CONTAINER_RUNTIME" run --rm \
+        -e LITELLM_BASE_URL="$LITELLM_BASE_URL" \
+        -e LITELLM_API_KEY="$LITELLM_API_KEY" \
+        -e LITELLM_JSON_OUTPUT="true" \
+        "$IMAGE_NAME"
+}
+
+# Run parallel health checks
+# This creates an infinite loop that keeps spawning containers
+# Each container tests all models, then exits, and a new one starts
+while true; do
+    # Start containers in parallel using background jobs
+    pids=()
+    for ((i=1; i<=NUM_PARALLEL_JOBS; i++)); do
+        run_health_check &
+        pids+=($!)
+    done
+    
+    # Wait for all background jobs to complete
+    for pid in "${pids[@]}"; do
+        wait "$pid" 2>/dev/null || true
+    done
+done

tests/pass_through_unit_tests/test_bedrock_tool_use_beta_header.py

@@ -0,0 +1,69 @@
+"""
+Simple E2E test for Bedrock with advanced-tool-use beta header.
+
+Tests that LiteLLM correctly filters out the advanced-tool-use-2025-11-20 beta header
+for Bedrock Invoke API, which doesn't support it and returns a 400 "invalid beta flag" error.
+"""
+import os
+import sys
+import pytest
+
+sys.path.insert(0, os.path.abspath("../.."))
+
+import litellm
+
+
+@pytest.mark.asyncio
+async def test_bedrock_sonnet_4_5_with_advanced_tool_use_beta_header():
+    """
+    Simple E2E test: Call Bedrock Sonnet 4.5 with advanced-tool-use beta header.
+
+    This should work without throwing "invalid beta flag" error because LiteLLM
+    filters out the advanced-tool-use beta header for Bedrock Invoke API.
+    """
+    litellm._turn_on_debug()
+    response = await litellm.anthropic.messages.acreate(
+        model="bedrock/invoke/us.anthropic.claude-sonnet-4-5-20250929-v1:0",
+        messages=[{"role": "user", "content": "What is 2+2?"}],
+        max_tokens=100,
+        provider_specific_header={
+            "custom_llm_provider": "bedrock",
+            "extra_headers": {
+                "anthropic-beta": "advanced-tool-use-2025-11-20",
+            },
+        },
+    )
+
+    # Verify response
+    assert response is not None
+    assert "content" in response
+    print(f"✅ Test passed! Response: {response}")
+
+
+@pytest.mark.asyncio
+async def test_bedrock_claude_3_5_with_advanced_tool_use_beta_header_filtered():
+    """
+    Simple E2E test: Call Bedrock Claude 3.5 with advanced-tool-use beta header.
+
+    This should work because the beta header is filtered out by LiteLLM before
+    sending the request to Bedrock Invoke API.
+    """
+
+    response = await litellm.anthropic.messages.acreate(
+        model="bedrock/invoke/us.anthropic.claude-3-5-sonnet-20240620-v1:0",
+        messages=[{"role": "user", "content": "What is 2+2?"}],
+        max_tokens=100,
+        provider_specific_header={
+            "custom_llm_provider": "bedrock",
+            "extra_headers": {
+                "anthropic-beta": "advanced-tool-use-2025-11-20",
+            },
+        },
+    )
+
+    # Verify response
+    assert response is not None
+    assert "content" in response
+    print(f"✅ Test passed! Claude 3.5 response (beta header filtered): {response}")
+
+

tests/test_litellm/proxy/pass_through_endpoints/test_pass_through_endpoints.py

@@ -1897,8 +1897,8 @@ async def test_add_litellm_data_to_request_adds_headers_to_metadata():
     The fix ensures headers are available in data["metadata"]["headers"] so
     guardrails can validate User-Agent, API keys, and other header-based checks.
     """
-    from litellm.proxy.litellm_pre_call_utils import add_litellm_data_to_request
     from litellm.proxy._types import UserAPIKeyAuth
+    from litellm.proxy.litellm_pre_call_utils import add_litellm_data_to_request
 
     # Create mock request with headers including User-Agent
     mock_request = MagicMock(spec=Request)
@@ -1954,3 +1954,150 @@ async def test_add_litellm_data_to_request_adds_headers_to_metadata():
     # Also verify proxy_server_request has headers (original location)
     assert "proxy_server_request" in result
     assert "headers" in result["proxy_server_request"]
+
+
+def test_build_full_path_with_root_default():
+    """
+    Test _build_full_path_with_root with default root path (/)
+    """
+    from litellm.proxy.pass_through_endpoints.pass_through_endpoints import (
+        InitPassThroughEndpointHelpers,
+    )
+
+    with patch("litellm.proxy.pass_through_endpoints.pass_through_endpoints.get_server_root_path") as mock_get_root:
+        # Test with default root path
+        mock_get_root.return_value = "/"
+
+        result = InitPassThroughEndpointHelpers._build_full_path_with_root("/api/v1/endpoint")
+        assert result == "/api/v1/endpoint"
+
+
+def test_build_full_path_with_root_custom():
+    """
+    Test _build_full_path_with_root with custom root path
+    """
+    from litellm.proxy.pass_through_endpoints.pass_through_endpoints import (
+        InitPassThroughEndpointHelpers,
+    )
+
+    with patch("litellm.proxy.pass_through_endpoints.pass_through_endpoints.get_server_root_path") as mock_get_root:
+        # Test with custom root path /proxy
+        mock_get_root.return_value = "/proxy"
+
+        result = InitPassThroughEndpointHelpers._build_full_path_with_root("/api/v1/endpoint")
+        assert result == "/proxy/api/v1/endpoint"
+
+
+def test_build_full_path_with_root_nested():
+    """
+    Test _build_full_path_with_root with nested root path
+    """
+    from litellm.proxy.pass_through_endpoints.pass_through_endpoints import (
+        InitPassThroughEndpointHelpers,
+    )
+
+    with patch("litellm.proxy.pass_through_endpoints.pass_through_endpoints.get_server_root_path") as mock_get_root:
+        # Test with nested root path /api/v2
+        mock_get_root.return_value = "/api/v2"
+
+        result = InitPassThroughEndpointHelpers._build_full_path_with_root("/endpoint")
+        assert result == "/api/v2/endpoint"
+
+
+def test_is_registered_pass_through_route_with_custom_root():
+    """
+    Test is_registered_pass_through_route correctly handles server root path
+
+    When server has a custom root path like /proxy, the registered path
+    should be constructed by prepending the root to match incoming routes.
+    """
+    from litellm.proxy.pass_through_endpoints.pass_through_endpoints import (
+        InitPassThroughEndpointHelpers,
+        _registered_pass_through_routes,
+    )
+
+    # Clear the registry first
+    _registered_pass_through_routes.clear()
+
+    # Register a pass-through route with endpoint format: {endpoint_id}:exact:{path}
+    endpoint_id = "test-endpoint-123"
+    path = "/api/endpoint"
+    route_key = f"{endpoint_id}:exact:{path}"
+    _registered_pass_through_routes[route_key] = {
+        "target": "http://example.com",
+        "headers": {},
+    }
+
+    with patch("litellm.proxy.pass_through_endpoints.pass_through_endpoints.get_server_root_path") as mock_get_root:
+        # Test with custom root path /proxy
+        mock_get_root.return_value = "/proxy"
+
+        # Should match when request route includes the root path
+        assert InitPassThroughEndpointHelpers.is_registered_pass_through_route("/proxy/api/endpoint") is True
+
+        # Should not match when request route doesn't include root path
+        assert InitPassThroughEndpointHelpers.is_registered_pass_through_route("/api/endpoint") is False
+
+        # Test with default root path
+        mock_get_root.return_value = "/"
+
+        # Should match with default root
+        assert InitPassThroughEndpointHelpers.is_registered_pass_through_route("/api/endpoint") is True
+
+        # Should not match with root prepended when root is /
+        assert InitPassThroughEndpointHelpers.is_registered_pass_through_route("/proxy/api/endpoint") is False
+
+    # Clean up
+    _registered_pass_through_routes.clear()
+
+
+def test_get_registered_pass_through_route_with_custom_root():
+    """
+    Test get_registered_pass_through_route correctly handles server root path
+
+    When server has a custom root path, the method should return the correct
+    endpoint configuration by matching the full path including the root.
+    """
+    from litellm.proxy.pass_through_endpoints.pass_through_endpoints import (
+        InitPassThroughEndpointHelpers,
+        _registered_pass_through_routes,
+    )
+
+    # Clear the registry first
+    _registered_pass_through_routes.clear()
+
+    # Register a pass-through route
+    endpoint_id = "test-endpoint-456"
+    path = "/chat/completions"
+    target_config = {
+        "target": "http://api.example.com/v1/chat/completions",
+        "headers": {"Authorization": "Bearer token123"},
+        "forward_headers": True,
+    }
+    route_key = f"{endpoint_id}:exact:{path}"
+    _registered_pass_through_routes[route_key] = target_config
+
+    with patch("litellm.proxy.pass_through_endpoints.pass_through_endpoints.get_server_root_path") as mock_get_root:
+        # Test with custom root path /litellm
+        mock_get_root.return_value = "/litellm"
+
+        # Should return config when request route includes root path
+        result = InitPassThroughEndpointHelpers.get_registered_pass_through_route("/litellm/chat/completions")
+        assert result is not None
+        assert result["target"] == "http://api.example.com/v1/chat/completions"
+        assert result["headers"]["Authorization"] == "Bearer token123"
+
+        # Should return None when route doesn't match
+        result = InitPassThroughEndpointHelpers.get_registered_pass_through_route("/chat/completions")
+        assert result is None
+
+        # Test with default root path
+        mock_get_root.return_value = "/"
+
+        # Should return config with default root
+        result = InitPassThroughEndpointHelpers.get_registered_pass_through_route("/chat/completions")
+        assert result is not None
+        assert result["target"] == "http://api.example.com/v1/chat/completions"
+
+    # Clean up
+    _registered_pass_through_routes.clear()