scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-07 02:59:05 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 103

Merge branch 'main' into litellm_dev_08_31_2025_p1

Browse source
This commit is contained in:
Krish Dholakia 2025-09-01 22:42:53 -07:00 • committed by GitHub
commit 82d6c4d5b1
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
63 changed files with 1837 additions and 441 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
.circleci/config.yml
View file

@ -1477,6 +1477,7 @@ jobs:
docker run -d \
-p 4000:4000 \
-e DATABASE_URL=$PROXY_DATABASE_URL \
-e DEFAULT_NUM_WORKERS_LITELLM_PROXY=1 \
-e DISABLE_SCHEMA_UPDATE="True" \
-v $(pwd)/litellm/proxy/example_config_yaml/bad_schema.prisma:/app/schema.prisma \
-v $(pwd)/litellm/proxy/example_config_yaml/bad_schema.prisma:/app/litellm/proxy/schema.prisma \
@ -2962,6 +2963,7 @@ jobs:
command: |
docker run --name my-app \
-p 4000:4000 \
-e DEFAULT_NUM_WORKERS_LITELLM_PROXY=1 \
-e DATABASE_URL="postgresql://wrong:wrong@wrong:5432/wrong" \
myapp:latest \
--port 4000 > docker_output.log 2>&1 || true

2
deploy/charts/litellm-helm/Chart.yaml
View file

@ -18,7 +18,7 @@ type: application
# This is the chart version. This version number should be incremented each time you make changes
# to the chart and its templates, including the app version.
# Versions are expected to follow Semantic Versioning (https://semver.org/)
version: 0.4.5
version: 0.4.6
# This is the version number of the application being deployed. This version number should be
# incremented each time you make changes to the application. Versions are not expected to

5
deploy/charts/litellm-helm/README.md
View file

@ -41,6 +41,11 @@ If `db.useStackgresOperator` is used (not yet implemented):
| `proxyConfigMap.key` | Key in the ConfigMap that contains the proxy config file. | `"config.yaml"` |
| `proxy_config.*` | See [values.yaml](./values.yaml) for default settings. Rendered into the ConfigMap’s `config.yaml` only when `proxyConfigMap.create=true`. See [example_config_yaml](../../../litellm/proxy/example_config_yaml/) for configuration examples. | `N/A` |
| `extraContainers[]` | An array of additional containers to be deployed as sidecars alongside the LiteLLM Proxy.
| `pdb.enabled` | Enable a PodDisruptionBudget for the LiteLLM proxy Deployment | `false` |
| `pdb.minAvailable` | Minimum number/percentage of pods that must be available during **voluntary** disruptions (choose **one** of minAvailable/maxUnavailable) | `null` |
| `pdb.maxUnavailable` | Maximum number/percentage of pods that can be unavailable during **voluntary** disruptions (choose **one** of minAvailable/maxUnavailable) | `null` |
| `pdb.annotations` | Extra metadata annotations to add to the PDB | `{}` |
| `pdb.labels` | Extra metadata labels to add to the PDB | `{}` |
#### Example `proxy_config` ConfigMap from values (default):

1
deploy/charts/litellm-helm/templates/NOTES.txt
View file

@ -20,3 +20,4 @@
echo "Visit http://127.0.0.1:8080 to use your application"
kubectl --namespace {{ .Release.Namespace }} port-forward $POD_NAME 8080:$CONTAINER_PORT
{{- end }}
PDB: {{ if .Values.pdb.enabled }}enabled{{ else }}disabled{{ end }}. Configure via .Values.pdb.*

33
deploy/charts/litellm-helm/templates/poddisruptionbudget.yaml Normal file
View file

@ -0,0 +1,33 @@
{{- /*
PodDisruptionBudget for LiteLLM proxy
Controlled via .Values.pdb.enabled and .Values.pdb.{minAvailable|maxUnavailable}
Only one of minAvailable / maxUnavailable should be set. If both are set, minAvailable wins.
*/ -}}
{{- if .Values.pdb.enabled }}
apiVersion: policy/v1
kind: PodDisruptionBudget
metadata:
name: {{ include "litellm.fullname" . }}
labels:
{{- include "litellm.labels" . | nindent 4 }}
{{- with .Values.pdb.labels }}
{{- toYaml . | nindent 4 }}
{{- end }}
{{- with .Values.pdb.annotations }}
annotations:
{{- toYaml . | nindent 4 }}
{{- end }}
spec:
selector:
matchLabels:
{{- /* Match the Deployment selector to target the same pod set */ -}}
{{- include "litellm.selectorLabels" . | nindent 6 }}
{{- if .Values.pdb.minAvailable }}
minAvailable: {{ .Values.pdb.minAvailable }}
{{- else if .Values.pdb.maxUnavailable }}
maxUnavailable: {{ .Values.pdb.maxUnavailable }}
{{- else }}
# Safe default if enabled but not configured
maxUnavailable: 1
{{- end }}
{{- end }}

45
deploy/charts/litellm-helm/tests/pdb_tests.yaml Normal file
View file

@ -0,0 +1,45 @@
suite: "pdb enabled"
templates:
- poddisruptionbudget.yaml
tests:
- it: "renders a PDB with maxUnavailable=1"
set:
pdb.enabled: true
pdb.maxUnavailable: 1
asserts:
- hasDocuments: { count: 1 }
- isKind: { of: PodDisruptionBudget }
- equal: { path: apiVersion, value: policy/v1 }
- equal: { path: spec.maxUnavailable, value: 1 }
- equal:
path: spec.selector.matchLabels
value:
app.kubernetes.io/name: litellm
app.kubernetes.io/instance: RELEASE-NAME
---
suite: "pdb disabled"
templates:
- poddisruptionbudget.yaml
tests:
- it: "does not render when disabled"
set:
pdb.enabled: false
asserts:
- hasDocuments: { count: 0 }
---
suite: "pdb minAvailable precedence"
templates:
- poddisruptionbudget.yaml
tests:
- it: "uses minAvailable when both are set"
set:
pdb.enabled: true
pdb.minAvailable: "50%"
pdb.maxUnavailable: 1
asserts:
- isKind: { of: PodDisruptionBudget }
- equal: { path: apiVersion, value: policy/v1 }
- equal: { path: spec.minAvailable, value: "50%" }
- isNull: { path: spec.maxUnavailable }

9
deploy/charts/litellm-helm/values.yaml
View file

@ -240,4 +240,11 @@ extraEnvVars: {
# value: EXTRA_ENV_VAR_VALUE
}
# Pod Disruption Budget
pdb:
enabled: false
# Set exactly one of the following. If both are set, minAvailable takes precedence.
minAvailable: null # e.g. "50%" or 1
maxUnavailable: null # e.g. 1 or "20%"
annotations: {}
labels: {}

1
docs/my-website/docs/exception_mapping.md
View file

@ -12,6 +12,7 @@ All exceptions can be imported from `litellm` - e.g. `from litellm import BadReq
| 400 | UnsupportedParamsError | litellm.BadRequestError | Raised when unsupported params are passed |
| 400 | ContextWindowExceededError| litellm.BadRequestError | Special error type for context window exceeded error messages - enables context window fallbacks |
| 400 | ContentPolicyViolationError| litellm.BadRequestError | Special error type for content policy violation error messages - enables content policy fallbacks |
| 400 | ImageFetchError | litellm.BadRequestError | Raised when there are errors fetching or processing images |
| 400 | InvalidRequestError | openai.BadRequestError | Deprecated error, use BadRequestError instead |
| 401 | AuthenticationError | openai.AuthenticationError |
| 403 | PermissionDeniedError | openai.PermissionDeniedError |

1
docs/my-website/docs/proxy/config_settings.md
View file

@ -431,6 +431,7 @@ router_settings:
| DEFAULT_MOCK_RESPONSE_COMPLETION_TOKEN_COUNT | Default token count for mock response completions. Default is 20
| DEFAULT_MOCK_RESPONSE_PROMPT_TOKEN_COUNT | Default token count for mock response prompts. Default is 10
| DEFAULT_MODEL_CREATED_AT_TIME | Default creation timestamp for models. Default is 1677610602
| DEFAULT_NUM_WORKERS_LITELLM_PROXY | Default number of workers for LiteLLM proxy. Default is 4. **We strongly recommend setting NUM Workers to Number of vCPUs available**
| DEFAULT_PROMPT_INJECTION_SIMILARITY_THRESHOLD | Default threshold for prompt injection similarity. Default is 0.7
| DEFAULT_POLLING_INTERVAL | Default polling interval for schedulers in seconds. Default is 0.03
| DEFAULT_REASONING_EFFORT_DISABLE_THINKING_BUDGET | Default reasoning effort disable thinking budget. Default is 0

5
docs/my-website/docs/proxy/deploy.md
View file

@ -12,10 +12,7 @@ To start using Litellm, run the following commands in a shell:
```bash
# Get the code
git clone https://github.com/BerriAI/litellm
# Go to folder
cd litellm
curl -O https://raw.githubusercontent.com/BerriAI/litellm/main/docker-compose.yml
# Add the master key - you can change this after setup
echo 'LITELLM_MASTER_KEY="sk-1234"' > .env

26
docs/my-website/docs/proxy/docker_quick_start.md
View file

@ -35,6 +35,30 @@ $ pip install 'litellm[proxy]'
</TabItem>
<TabItem value="docker-compose" label="Docker Compose (Proxy + DB)">
Use this docker compose to spin up the proxy with a postgres database running locally.
```bash
# Get the docker compose file
curl -O https://raw.githubusercontent.com/BerriAI/litellm/main/docker-compose.yml
# Add the master key - you can change this after setup
echo 'LITELLM_MASTER_KEY="sk-1234"' > .env
# Add the litellm salt key - you cannot change this after adding a model
# It is used to encrypt / decrypt your LLM API Key credentials
# We recommend - https://1password.com/password-generator/
# password generator to get a random hash for litellm salt key
echo 'LITELLM_SALT_KEY="sk-1234"' >> .env
source .env
# Start
docker-compose up
```
</TabItem>
</Tabs>
## 1. Add a model
@ -43,6 +67,8 @@ Control LiteLLM Proxy with a config.yaml file.
Setup your config.yaml with your azure model.
Note: When using the proxy with a database, you can also **just add models via UI** (UI is available on `/ui` route).
```yaml
model_list:
- model_name: gpt-4o

2
docs/my-website/docs/proxy/request_headers.md
View file

@ -6,6 +6,8 @@ Special headers that are supported by LiteLLM.
`x-litellm-timeout` Optional[float]: The timeout for the request in seconds.
`x-litellm-stream-timeout` Optional[float]: The timeout for getting the first chunk of the response in seconds (only applies for streaming requests). [Demo Video](https://www.loom.com/share/8da67e4845ce431a98c901d4e45db0e5)
`x-litellm-enable-message-redaction`: Optional[bool]: Don't log the message content to logging integrations. Just track spend. [Learn More](./logging#redact-messages-response-content)
`x-litellm-tags`: Optional[str]: A comma separated list (e.g. `tag1,tag2,tag3`) of tags to use for [tag-based routing](./tag_routing) **OR** [spend-tracking](./enterprise.md#tracking-spend-for-custom-tags).

39
enterprise/litellm_enterprise/integrations/prometheus.py
View file

@ -95,13 +95,14 @@ class PrometheusLogger(CustomLogger):
self.litellm_llm_api_time_to_first_token_metric = self._histogram_factory(
"litellm_llm_api_time_to_first_token_metric",
"Time to first token for a models LLM API call",
labelnames=[
"model",
"hashed_api_key",
"api_key_alias",
"team",
"team_alias",
],
# labelnames=[
# "model",
# "hashed_api_key",
# "api_key_alias",
# "team",
# "team_alias",
# ],
labelnames=self.get_labels_for_metric("litellm_llm_api_time_to_first_token_metric"),
buckets=LATENCY_BUCKETS,
)
@ -109,15 +110,7 @@ class PrometheusLogger(CustomLogger):
self.litellm_spend_metric = self._counter_factory(
"litellm_spend_metric",
"Total spend on LLM requests",
labelnames=[
"end_user",
"hashed_api_key",
"api_key_alias",
"model",
"team",
"team_alias",
"user",
],
labelnames=self.get_labels_for_metric("litellm_spend_metric"),
)
# Counter for total_output_tokens
@ -243,25 +236,18 @@ class PrometheusLogger(CustomLogger):
labelnames=["api_provider"],
)
# Get all keys
_logged_llm_labels = [
UserAPIKeyLabelNames.v2_LITELLM_MODEL_NAME.value,
UserAPIKeyLabelNames.MODEL_ID.value,
UserAPIKeyLabelNames.API_BASE.value,
UserAPIKeyLabelNames.API_PROVIDER.value,
]
# Metric for deployment state
self.litellm_deployment_state = self._gauge_factory(
"litellm_deployment_state",
"LLM Deployment Analytics - The state of the deployment: 0 = healthy, 1 = partial outage, 2 = complete outage",
labelnames=_logged_llm_labels,
labelnames=self.get_labels_for_metric("litellm_deployment_state")
)
self.litellm_deployment_cooled_down = self._counter_factory(
"litellm_deployment_cooled_down",
"LLM Deployment Analytics - Number of times a deployment has been cooled down by LiteLLM load balancing logic. exception_status is the status of the exception that caused the deployment to be cooled down",
labelnames=_logged_llm_labels + [EXCEPTION_STATUS],
# labelnames=_logged_llm_labels + [EXCEPTION_STATUS],
labelnames=self.get_labels_for_metric("litellm_deployment_cooled_down")
)
self.litellm_deployment_success_responses = self._counter_factory(
@ -327,6 +313,7 @@ class PrometheusLogger(CustomLogger):
documentation="deprecated - use litellm_proxy_total_requests_metric. Total number of LLM calls to litellm - track total per API Key, team, user",
labelnames=self.get_labels_for_metric("litellm_requests_metric"),
)
except Exception as e:
print_verbose(f"Got exception on init prometheus client {str(e)}")
raise e

1
litellm/__init__.py
View file

@ -1261,6 +1261,7 @@ from .exceptions import (
AuthenticationError,
InvalidRequestError,
BadRequestError,
ImageFetchError,
NotFoundError,
RateLimitError,
ServiceUnavailableError,

1
litellm/constants.py
View file

@ -14,6 +14,7 @@ DEFAULT_S3_BATCH_SIZE = int(os.getenv("DEFAULT_S3_BATCH_SIZE", 512))
DEFAULT_SQS_FLUSH_INTERVAL_SECONDS = int(
os.getenv("DEFAULT_SQS_FLUSH_INTERVAL_SECONDS", 10)
)
DEFAULT_NUM_WORKERS_LITELLM_PROXY = int(os.getenv("DEFAULT_NUM_WORKERS_LITELLM_PROXY", 4))
DEFAULT_SQS_BATCH_SIZE = int(os.getenv("DEFAULT_SQS_BATCH_SIZE", 512))
SQS_SEND_MESSAGE_ACTION = "SendMessage"
SQS_API_VERSION = "2012-11-05"

13
litellm/cost_calculator.py
View file

@ -57,6 +57,7 @@ from litellm.llms.vertex_ai.cost_calculator import (
cost_per_token as google_cost_per_token,
)
from litellm.llms.vertex_ai.cost_calculator import cost_router as google_cost_router
from litellm.llms.xai.cost_calculator import cost_per_token as xai_cost_per_token
from litellm.responses.utils import ResponseAPILoggingUtils
from litellm.types.llms.openai import (
HttpxBinaryResponseContent,
@ -341,6 +342,8 @@ def cost_per_token( # noqa: PLR0915
return deepseek_cost_per_token(model=model, usage=usage_block)
elif custom_llm_provider == "perplexity":
return perplexity_cost_per_token(model=model, usage=usage_block)
elif custom_llm_provider == "xai":
return xai_cost_per_token(model=model, usage=usage_block)
else:
model_info = _cached_get_model_info_helper(
model=model, custom_llm_provider=custom_llm_provider
@ -675,9 +678,9 @@ def completion_cost( # noqa: PLR0915
or isinstance(completion_response, dict)
): # tts returns a custom class
if isinstance(completion_response, dict):
usage_obj: Optional[Union[dict, Usage]] = (
completion_response.get("usage", {})
)
usage_obj: Optional[
Union[dict, Usage]
] = completion_response.get("usage", {})
else:
usage_obj = getattr(completion_response, "usage", {})
if isinstance(usage_obj, BaseModel) and not _is_known_usage_objects(
@ -1279,7 +1282,9 @@ class BaseTokenUsageProcessor:
not hasattr(combined, "completion_tokens_details")
or not combined.completion_tokens_details
):
combined.completion_tokens_details = CompletionTokensDetailsWrapper()
combined.completion_tokens_details = (
CompletionTokensDetailsWrapper()
)
# Check what keys exist in the model's completion_tokens_details
for attr in usage.completion_tokens_details.model_fields:

23
litellm/exceptions.py
View file

@ -153,6 +153,29 @@ class BadRequestError(openai.BadRequestError): # type: ignore
_message += f", LiteLLM Max Retries: {self.max_retries}"
return _message
class ImageFetchError(BadRequestError):
def __init__(
self,
message,
model=None,
llm_provider=None,
response: Optional[httpx.Response] = None,
litellm_debug_info: Optional[str] = None,
max_retries: Optional[int] = None,
num_retries: Optional[int] = None,
body: Optional[dict] = None,
):
super().__init__(
message=message,
model=model,
llm_provider=llm_provider,
response=response,
litellm_debug_info=litellm_debug_info,
max_retries=max_retries,
num_retries=num_retries,
body=body,
)
class UnprocessableEntityError(openai.UnprocessableEntityError): # type: ignore
def __init__(

4
litellm/google_genai/adapters/handler.py
View file

@ -37,6 +37,10 @@ class GenerateContentToCompletionHandler:
completion_kwargs: Dict[str, Any] = dict(completion_request)
# feed metadata for custom callback
if extra_kwargs is not None and "metadata" in extra_kwargs:
completion_kwargs["metadata"] = extra_kwargs["metadata"]
if stream:
completion_kwargs["stream"] = stream

167
litellm/integrations/braintrust_logging.py
View file

@ -1,13 +1,11 @@
# What is this?
## Log success + failure events to Braintrust
import copy
import os
from datetime import datetime
from typing import Dict, Optional
import httpx
from pydantic import BaseModel
import litellm
from litellm import verbose_logger
@ -24,7 +22,6 @@ API_BASE = "https://api.braintrustdata.com/v1"
def get_utc_datetime():
import datetime as dt
from datetime import datetime
if hasattr(dt, "UTC"):
return datetime.now(dt.UTC) # type: ignore
@ -45,9 +42,9 @@ class BraintrustLogger(CustomLogger):
"Authorization": "Bearer " + self.api_key,
"Content-Type": "application/json",
}
self._project_id_cache: Dict[
str, str
] = {} # Cache mapping project names to IDs
self._project_id_cache: Dict[str, str] = (
{}
) # Cache mapping project names to IDs
self.global_braintrust_http_handler = get_async_httpx_client(
llm_provider=httpxSpecialProvider.LoggingCallback
)
@ -108,43 +105,6 @@ class BraintrustLogger(CustomLogger):
except httpx.HTTPStatusError as e:
raise Exception(f"Failed to register project: {e.response.text}")
@staticmethod
def add_metadata_from_header(litellm_params: dict, metadata: dict) -> dict:
"""
Adds metadata from proxy request headers to Braintrust logging if keys start with "braintrust_"
and overwrites litellm_params.metadata if already included.
For example if you want to append your trace to an existing `trace_id` via header, send
`headers: { ..., langfuse_existing_trace_id: your-existing-trace-id }` via proxy request.
"""
if litellm_params is None:
return metadata
if litellm_params.get("proxy_server_request") is None:
return metadata
if metadata is None:
metadata = {}
proxy_headers = (
litellm_params.get("proxy_server_request", {}).get("headers", {}) or {}
)
for metadata_param_key in proxy_headers:
if metadata_param_key.startswith("braintrust"):
trace_param_key = metadata_param_key.replace("braintrust", "", 1)
if trace_param_key in metadata:
verbose_logger.warning(
f"Overwriting Braintrust `{trace_param_key}` from request header"
)
else:
verbose_logger.debug(
f"Found Braintrust `{trace_param_key}` in request header"
)
metadata[trace_param_key] = proxy_headers.get(metadata_param_key)
return metadata
async def create_default_project_and_experiment(self):
project = await self.global_braintrust_http_handler.post(
f"{self.api_base}/project", headers=self.headers, json={"name": "litellm"}
@ -169,7 +129,9 @@ class BraintrustLogger(CustomLogger):
verbose_logger.debug("REACHES BRAINTRUST SUCCESS")
try:
litellm_call_id = kwargs.get("litellm_call_id")
standard_logging_object = kwargs.get("standard_logging_object", {})
prompt = {"messages": kwargs.get("messages")}
output = None
choices = []
if response_obj is not None and (
@ -192,33 +154,13 @@ class BraintrustLogger(CustomLogger):
):
output = response_obj["data"]
litellm_params = kwargs.get("litellm_params", {})
metadata = (
litellm_params.get("metadata", {}) or {}
) # if litellm_params['metadata'] == None
metadata = self.add_metadata_from_header(litellm_params, metadata)
clean_metadata = {}
try:
metadata = copy.deepcopy(
metadata
) # Avoid modifying the original metadata
except Exception:
new_metadata = {}
for key, value in metadata.items():
if (
isinstance(value, list)
or isinstance(value, dict)
or isinstance(value, str)
or isinstance(value, int)
or isinstance(value, float)
):
new_metadata[key] = copy.deepcopy(value)
metadata = new_metadata
litellm_params = kwargs.get("litellm_params", {}) or {}
dynamic_metadata = litellm_params.get("metadata", {}) or {}
# Get project_id from metadata or create default if needed
project_id = metadata.get("project_id")
project_id = dynamic_metadata.get("project_id")
if project_id is None:
project_name = metadata.get("project_name")
project_name = dynamic_metadata.get("project_name")
project_id = (
self.get_project_id_sync(project_name) if project_name else None
)
@ -229,8 +171,9 @@ class BraintrustLogger(CustomLogger):
project_id = self.default_project_id
tags = []
if isinstance(metadata, dict):
for key, value in metadata.items():
if isinstance(dynamic_metadata, dict):
for key, value in dynamic_metadata.items():
# generate langfuse tags - Default Tags sent to Langfuse from LiteLLM Proxy
if (
litellm.langfuse_default_tags is not None
@ -239,25 +182,12 @@ class BraintrustLogger(CustomLogger):
):
tags.append(f"{key}:{value}")
# clean litellm metadata before logging
if key in [
"headers",
"endpoint",
"caching_groups",
"previous_models",
]:
continue
else:
clean_metadata[key] = value
if (
isinstance(value, str) and key not in standard_logging_object
): # support logging dynamic metadata to braintrust
standard_logging_object[key] = value
cost = kwargs.get("response_cost", None)
if cost is not None:
clean_metadata["litellm_response_cost"] = cost
# metadata.model is required for braintrust to calculate the "Estimated cost" metric
litellm_model = kwargs.get("model", None)
if litellm_model is not None:
clean_metadata["model"] = litellm_model
metrics: Optional[dict] = None
usage_obj = getattr(response_obj, "usage", None)
@ -275,12 +205,12 @@ class BraintrustLogger(CustomLogger):
}
# Allow metadata override for span name
span_name = metadata.get("span_name", "Chat Completion")
span_name = dynamic_metadata.get("span_name", "Chat Completion")
request_data = {
"id": litellm_call_id,
"input": prompt["messages"],
"metadata": clean_metadata,
"metadata": standard_logging_object,
"tags": tags,
"span_attributes": {"name": span_name, "type": "llm"},
}
@ -312,6 +242,7 @@ class BraintrustLogger(CustomLogger):
verbose_logger.debug("REACHES BRAINTRUST SUCCESS")
try:
litellm_call_id = kwargs.get("litellm_call_id")
standard_logging_object = kwargs.get("standard_logging_object", {})
prompt = {"messages": kwargs.get("messages")}
output = None
choices = []
@ -336,32 +267,12 @@ class BraintrustLogger(CustomLogger):
output = response_obj["data"]
litellm_params = kwargs.get("litellm_params", {})
metadata = (
litellm_params.get("metadata", {}) or {}
) # if litellm_params['metadata'] == None
metadata = self.add_metadata_from_header(litellm_params, metadata)
clean_metadata = {}
new_metadata = {}
for key, value in metadata.items():
if (
isinstance(value, list)
or isinstance(value, str)
or isinstance(value, int)
or isinstance(value, float)
):
new_metadata[key] = value
elif isinstance(value, BaseModel):
new_metadata[key] = value.model_dump_json()
elif isinstance(value, dict):
for k, v in value.items():
if isinstance(v, datetime):
value[k] = v.isoformat()
new_metadata[key] = value
dynamic_metadata = litellm_params.get("metadata", {}) or {}
# Get project_id from metadata or create default if needed
project_id = metadata.get("project_id")
project_id = dynamic_metadata.get("project_id")
if project_id is None:
project_name = metadata.get("project_name")
project_name = dynamic_metadata.get("project_name")
project_id = (
await self.get_project_id_async(project_name)
if project_name
@ -374,8 +285,9 @@ class BraintrustLogger(CustomLogger):
project_id = self.default_project_id
tags = []
if isinstance(metadata, dict):
for key, value in metadata.items():
if isinstance(dynamic_metadata, dict):
for key, value in dynamic_metadata.items():
# generate langfuse tags - Default Tags sent to Langfuse from LiteLLM Proxy
if (
litellm.langfuse_default_tags is not None
@ -384,25 +296,12 @@ class BraintrustLogger(CustomLogger):
):
tags.append(f"{key}:{value}")
# clean litellm metadata before logging
if key in [
"headers",
"endpoint",
"caching_groups",
"previous_models",
]:
continue
else:
clean_metadata[key] = value
if (
isinstance(value, str) and key not in standard_logging_object
): # support logging dynamic metadata to braintrust
standard_logging_object[key] = value
cost = kwargs.get("response_cost", None)
if cost is not None:
clean_metadata["litellm_response_cost"] = cost
# metadata.model is required for braintrust to calculate the "Estimated cost" metric
litellm_model = kwargs.get("model", None)
if litellm_model is not None:
clean_metadata["model"] = litellm_model
metrics: Optional[dict] = None
usage_obj = getattr(response_obj, "usage", None)
@ -430,13 +329,13 @@ class BraintrustLogger(CustomLogger):
)
# Allow metadata override for span name
span_name = metadata.get("span_name", "Chat Completion")
span_name = dynamic_metadata.get("span_name", "Chat Completion")
request_data = {
"id": litellm_call_id,
"input": prompt["messages"],
"output": output,
"metadata": clean_metadata,
"metadata": standard_logging_object,
"tags": tags,
"span_attributes": {"name": span_name, "type": "llm"},
}

13
litellm/litellm_core_utils/prompt_templates/image_handling.py
View file

@ -17,7 +17,7 @@ in_memory_cache = InMemoryCache(max_size_in_memory=MAX_IMGS_IN_MEMORY)
def _process_image_response(response: Response, url: str) -> str:
if response.status_code != 200:
raise Exception(
raise litellm.ImageFetchError(
f"Error: Unable to fetch image from URL. Status code: {response.status_code}, url={url}"
)
@ -57,9 +57,11 @@ async def async_convert_url_to_base64(url: str) -> str:
try:
response = await client.get(url, follow_redirects=True)
return _process_image_response(response, url)
except litellm.ImageFetchError:
raise
except Exception:
pass
raise Exception(
raise litellm.ImageFetchError(
f"Error: Unable to fetch image from URL after 3 attempts. url={url}"
)
@ -74,10 +76,11 @@ def convert_url_to_base64(url: str) -> str:
try:
response = client.get(url, follow_redirects=True)
return _process_image_response(response, url)
except litellm.ImageFetchError:
raise
except Exception as e:
verbose_logger.exception(e)
# print(e)
pass
raise Exception(
f"Error: Unable to fetch image from URL after 3 attempts. url={url}"
raise litellm.ImageFetchError(
f"Error: Unable to fetch image from URL after 3 attempts. url={url}",
)

1
litellm/litellm_core_utils/safe_json_dumps.py
View file

@ -1,5 +1,6 @@
import json
from typing import Any, Union
from litellm.constants import DEFAULT_MAX_RECURSE_DEPTH

13
litellm/llms/openai/chat/gpt_5_transformation.py
View file

@ -28,7 +28,18 @@ class OpenAIGPT5Config(OpenAIGPTConfig):
base_gpt_series_params.extend(gpt_5_only_params)
if not supports_tool_choice(model=model):
base_gpt_series_params.remove("tool_choice")
return base_gpt_series_params
non_supported_params = [
"logprobs",
"top_p",
"presence_penalty",
"frequency_penalty",
"top_logprobs",
]
return [
param for param in base_gpt_series_params if param not in non_supported_params
]
def map_openai_params(
self,

65
litellm/llms/vertex_ai/common_utils.py
View file

@ -187,6 +187,25 @@ def _check_text_in_content(parts: List[PartType]) -> bool:
return has_text_param
def _fix_enum_empty_strings(schema, depth=0):
"""Fix empty strings in enum values by replacing them with None. Gemini doesn't accept empty strings in enums."""
if depth > DEFAULT_MAX_RECURSE_DEPTH:
raise ValueError(f"Max depth of {DEFAULT_MAX_RECURSE_DEPTH} exceeded while processing schema.")
if "enum" in schema and isinstance(schema["enum"], list):
schema["enum"] = [None if value == "" else value for value in schema["enum"]]
# Reuse existing recursion pattern from convert_anyof_null_to_nullable
properties = schema.get("properties", None)
if properties is not None:
for _, value in properties.items():
_fix_enum_empty_strings(value, depth=depth + 1)
items = schema.get("items", None)
if items is not None:
_fix_enum_empty_strings(items, depth=depth + 1)
def _build_vertex_schema(parameters: dict, add_property_ordering: bool = False):
"""
This is a modified version of https://github.com/google-gemini/generative-ai-python/blob/8f77cc6ac99937cd3a81299ecf79608b91b06bbb/google/generativeai/types/content_types.py#L419
@ -215,6 +234,11 @@ def _build_vertex_schema(parameters: dict, add_property_ordering: bool = False):
# * https://github.com/pydantic/pydantic/discussions/4872
convert_anyof_null_to_nullable(parameters)
_convert_schema_types(parameters)
# Handle empty strings in enum values - Gemini doesn't accept empty strings in enums
_fix_enum_empty_strings(parameters)
# Handle empty items objects
process_items(parameters)
add_object_type(parameters)
@ -439,6 +463,47 @@ def _convert_vertex_datetime_to_openai_datetime(vertex_datetime: str) -> int:
return int(dt.timestamp())
def _convert_schema_types(schema, depth=0):
"""
Convert type arrays and lowercase types for Vertex AI compatibility.
Transforms OpenAI-style schemas to Vertex AI format by converting type arrays
like ["string", "number"] to anyOf format and converting all types to uppercase.
"""
if depth > DEFAULT_MAX_RECURSE_DEPTH:
raise ValueError(
f"Max depth of {DEFAULT_MAX_RECURSE_DEPTH} exceeded while processing schema. Please check the schema for excessive nesting."
)
if not isinstance(schema, dict):
return
# Handle type field
if "type" in schema:
type_val = schema["type"]
if isinstance(type_val, list) and len(type_val) > 1:
# Convert ["string", "number"] -> {"anyOf": [{"type": "STRING"}, {"type": "NUMBER"}]}
schema["anyOf"] = [{"type": t} for t in type_val if isinstance(t, str)]
schema.pop("type")
elif isinstance(type_val, list) and len(type_val) == 1:
schema["type"] = type_val[0]
elif isinstance(type_val, str):
schema["type"] = type_val
# Recursively process nested properties, items, and anyOf
for key in ["properties", "items", "anyOf"]:
if key in schema:
value = schema[key]
if key == "properties" and isinstance(value, dict):
for prop_schema in value.values():
_convert_schema_types(prop_schema, depth + 1)
elif key == "items":
_convert_schema_types(value, depth + 1)
elif key == "anyOf" and isinstance(value, list):
for anyof_schema in value:
_convert_schema_types(anyof_schema, depth + 1)
def get_vertex_project_id_from_url(url: str) -> Optional[str]:
"""
Get the vertex project id from the url

95
litellm/llms/vertex_ai/gemini/transformation.py
View file

@ -105,6 +105,64 @@ def _process_gemini_image(image_url: str, format: Optional[str] = None) -> PartT
raise e
def _snake_to_camel(snake_str: str) -> str:
"""Convert snake_case to camelCase"""
components = snake_str.split("_")
return components[0] + "".join(x.capitalize() for x in components[1:])
def _camel_to_snake(camel_str: str) -> str:
"""Convert camelCase to snake_case"""
import re
return re.sub(r"(?<!^)(?=[A-Z])", "_", camel_str).lower()
def _get_equivalent_key(key: str, available_keys: set) -> Optional[str]:
"""
Get the equivalent key from available keys, checking both camelCase and snake_case variants
"""
if key in available_keys:
return key
# Try camelCase version
camel_key = _snake_to_camel(key)
if camel_key in available_keys:
return camel_key
# Try snake_case version
snake_key = _camel_to_snake(key)
if snake_key in available_keys:
return snake_key
return None
def check_if_part_exists_in_parts(
parts: List[PartType], part: PartType, excluded_keys: List[str] = []
) -> bool:
"""
Check if a part exists in a list of parts
Handles both camelCase and snake_case key variations (e.g., function_call vs functionCall)
"""
keys_to_compare = set(part.keys()) - set(excluded_keys)
for p in parts:
p_keys = set(p.keys())
# Check if all keys in part have equivalent values in p
match_found = True
for key in keys_to_compare:
equivalent_key = _get_equivalent_key(key, p_keys)
if equivalent_key is None or p.get(equivalent_key, None) != part.get(
key, None
):
match_found = False
break
if match_found:
return True
return False
def _gemini_convert_messages_with_history( # noqa: PLR0915
messages: List[AllMessageValues],
) -> List[ContentType]:
@ -236,10 +294,33 @@ def _gemini_convert_messages_with_history( # noqa: PLR0915
assistant_msg = ChatCompletionAssistantMessage(**msg_dict) # type: ignore
_message_content = assistant_msg.get("content", None)
reasoning_content = assistant_msg.get("reasoning_content", None)
thinking_blocks = assistant_msg.get("thinking_blocks")
if reasoning_content is not None:
assistant_content.append(
PartType(thought=True, text=reasoning_content)
)
if thinking_blocks is not None:
for block in thinking_blocks:
block_thinking_str = block.get("thinking")
block_signature = block.get("signature")
if (
block_thinking_str is not None
and block_signature is not None
):
try:
assistant_content.append(
PartType(
thoughtSignature=block_signature,
**json.loads(block_thinking_str),
)
)
except Exception:
assistant_content.append(
PartType(
thoughtSignature=block_signature,
text=block_thinking_str,
)
)
if _message_content is not None and isinstance(_message_content, list):
_parts = []
for element in _message_content:
@ -262,9 +343,17 @@ def _gemini_convert_messages_with_history( # noqa: PLR0915
assistant_msg.get("tool_calls", []) is not None
or assistant_msg.get("function_call") is not None
): # support assistant tool invoke conversion
assistant_content.extend(
convert_to_gemini_tool_call_invoke(assistant_msg)
gemini_tool_call_parts = convert_to_gemini_tool_call_invoke(
assistant_msg
)
## check if gemini_tool_call already exists in assistant_content
for gemini_tool_call_part in gemini_tool_call_parts:
if not check_if_part_exists_in_parts(
assistant_content,
gemini_tool_call_part,
excluded_keys=["thoughtSignature"],
):
assistant_content.append(gemini_tool_call_part)
last_message_with_tool_calls = assistant_msg
msg_i += 1
@ -476,6 +565,7 @@ async def async_transform_request_body(
optional_params=optional_params,
)
def _default_user_message_when_system_message_passed() -> ChatCompletionUserMessage:
"""
Returns a default user message when a "system" message is passed in gemini fails.
@ -484,6 +574,7 @@ def _default_user_message_when_system_message_passed() -> ChatCompletionUserMess
"""
return ChatCompletionUserMessage(content=".", role="user")
def _transform_system_message(
supports_system_message: bool, messages: List[AllMessageValues]
) -> Tuple[Optional[SystemInstructions], List[AllMessageValues]]:

77
litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py
View file

@ -43,6 +43,7 @@ from litellm.types.llms.gemini import BidiGenerateContentServerMessage
from litellm.types.llms.openai import (
AllMessageValues,
ChatCompletionResponseMessage,
ChatCompletionThinkingBlock,
ChatCompletionToolCallChunk,
ChatCompletionToolCallFunctionChunk,
ChatCompletionToolParamFunctionChunk,
@ -792,7 +793,25 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
content_str += _content_str
return content_str, reasoning_content_str
def _extract_thinking_blocks_from_parts(
self, parts: List[HttpxPartType]
) -> List[ChatCompletionThinkingBlock]:
"""Extract thinking blocks from parts if present"""
thinking_blocks: List[ChatCompletionThinkingBlock] = []
for part in parts:
if "thoughtSignature" in part:
part_copy = part.copy()
part_copy.pop("thoughtSignature")
thinking_blocks.append(
ChatCompletionThinkingBlock(
type="thinking",
thinking=json.dumps(part_copy),
signature=part["thoughtSignature"],
)
)
return thinking_blocks
def _extract_image_response_from_parts(
self, parts: List[HttpxPartType]
) -> Optional[ImageURLObject]:
@ -804,10 +823,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
if mime_type.startswith("image/"):
# Convert base64 data to data URI format
data_uri = f"data:{mime_type};base64,{data}"
return ImageURLObject(
url=data_uri,
detail="auto"
)
return ImageURLObject(url=data_uri, detail="auto")
return None
def _extract_audio_response_from_parts(
@ -1127,7 +1143,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
elif web_search_queries:
web_search_requests = len(grounding_metadata)
return web_search_requests
@staticmethod
def _create_streaming_choice(
chat_completion_message: ChatCompletionResponseMessage,
@ -1151,9 +1167,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
index=candidate.get("index", idx),
delta=Delta(
content=chat_completion_message.get("content"),
reasoning_content=chat_completion_message.get(
"reasoning_content"
),
reasoning_content=chat_completion_message.get("reasoning_content"),
tool_calls=tools,
image=image_response,
function_call=functions,
@ -1164,13 +1178,15 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
return choice
@staticmethod
def _extract_candidate_metadata(candidate: Candidates) -> Tuple[List[dict], List[dict], List, List]:
def _extract_candidate_metadata(
candidate: Candidates,
) -> Tuple[List[dict], List[dict], List, List]:
"""
Extract metadata from a single candidate response.
Returns:
grounding_metadata: List[dict]
url_context_metadata: List[dict]
url_context_metadata: List[dict]
safety_ratings: List
citation_metadata: List
"""
@ -1178,7 +1194,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
url_context_metadata: List[dict] = []
safety_ratings: List = []
citation_metadata: List = []
if "groundingMetadata" in candidate:
if isinstance(candidate["groundingMetadata"], list):
grounding_metadata.extend(candidate["groundingMetadata"]) # type: ignore
@ -1194,8 +1210,13 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
if "urlContextMetadata" in candidate:
# Add URL context metadata to grounding metadata
url_context_metadata.append(cast(dict, candidate["urlContextMetadata"]))
return grounding_metadata, url_context_metadata, safety_ratings, citation_metadata
return (
grounding_metadata,
url_context_metadata,
safety_ratings,
citation_metadata,
)
@staticmethod
def _process_candidates(
@ -1227,6 +1248,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
tools: Optional[List[ChatCompletionToolCallChunk]] = []
functions: Optional[ChatCompletionToolCallFunctionChunk] = None
cumulative_tool_call_index: int = 0
thinking_blocks: Optional[List[ChatCompletionThinkingBlock]] = None
for idx, candidate in enumerate(_candidates):
if "content" not in candidate:
@ -1239,7 +1261,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
candidate_safety_ratings,
candidate_citation_metadata,
) = VertexGeminiConfig._extract_candidate_metadata(candidate)
grounding_metadata.extend(candidate_grounding_metadata)
url_context_metadata.extend(candidate_url_context_metadata)
safety_ratings.extend(candidate_safety_ratings)
@ -1264,6 +1286,12 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
)
)
thinking_blocks = (
VertexGeminiConfig()._extract_thinking_blocks_from_parts(
parts=candidate["content"]["parts"]
)
)
if audio_response is not None:
cast(Dict[str, Any], chat_completion_message)[
"audio"
@ -1271,7 +1299,9 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
chat_completion_message["content"] = None # OpenAI spec
if image_response is not None:
# Handle image response - combine with text content into structured format
cast(Dict[str, Any], chat_completion_message)["image"] = image_response
cast(Dict[str, Any], chat_completion_message)[
"image"
] = image_response
if content is not None:
chat_completion_message["content"] = content
@ -1298,15 +1328,18 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
if functions is not None:
chat_completion_message["function_call"] = functions
if thinking_blocks is not None:
chat_completion_message["thinking_blocks"] = thinking_blocks # type: ignore
if isinstance(model_response, ModelResponseStream):
choice = VertexGeminiConfig._create_streaming_choice(
chat_completion_message=chat_completion_message,
candidate=candidate,
idx=idx,
tools=tools,
functions=functions,
candidate=candidate,
idx=idx,
tools=tools,
functions=functions,
chat_completion_logprobs=chat_completion_logprobs,
image_response=image_response
image_response=image_response,
)
model_response.choices.append(choice)
elif isinstance(model_response, ModelResponse):

54
litellm/llms/xai/cost_calculator.py Normal file
View file

@ -0,0 +1,54 @@
"""
Helper util for handling XAI-specific cost calculation
- e.g.: reasoning tokens for grok models
"""
from typing import Tuple, Union
from litellm.types.utils import Usage
from litellm.utils import get_model_info
def cost_per_token(model: str, usage: Usage) -> Tuple[float, float]:
"""
Calculates the cost per token for a given XAI model, prompt tokens, and completion tokens.
Input:
- model: str, the model name without provider prefix
- usage: LiteLLM Usage block, containing XAI-specific usage information
Returns:
Tuple[float, float] - prompt_cost_in_usd, completion_cost_in_usd
"""
## GET MODEL INFO
model_info = get_model_info(model=model, custom_llm_provider="xai")
def _safe_float_cast(
value: Union[str, int, float, None, object], default: float = 0.0
) -> float:
"""Safely cast a value to float with proper type handling for mypy."""
if value is None:
return default
try:
return float(value) # type: ignore
except (ValueError, TypeError):
return default
## CALCULATE INPUT COST
input_cost_per_token = _safe_float_cast(model_info.get("input_cost_per_token"))
prompt_cost: float = (usage.prompt_tokens or 0) * input_cost_per_token
## CALCULATE OUTPUT COST
output_cost_per_token = _safe_float_cast(model_info.get("output_cost_per_token"))
# For XAI models, completion is billed as (visible completion tokens + reasoning tokens)
completion_tokens = int(getattr(usage, "completion_tokens", 0) or 0)
reasoning_tokens = 0
if hasattr(usage, "completion_tokens_details") and usage.completion_tokens_details:
reasoning_tokens = int(
getattr(usage.completion_tokens_details, "reasoning_tokens", 0) or 0
)
completion_cost = (completion_tokens + reasoning_tokens) * output_cost_per_token
return prompt_cost, completion_cost

177
litellm/model_prices_and_context_window_backup.json
View file

@ -5817,16 +5817,6 @@
"supports_response_schema": true,
"supports_tool_choice": true
},
"groq/llama3-8b-8192": {
"max_tokens": 8192,
"max_input_tokens": 8192,
"max_output_tokens": 8192,
"input_cost_per_token": 5e-08,
"output_cost_per_token": 8e-08,
"litellm_provider": "groq",
"mode": "chat",
"supports_tool_choice": true
},
"groq/llama-3.2-1b-preview": {
"max_tokens": 8192,
"max_input_tokens": 8192,
@ -5907,17 +5897,6 @@
"supports_tool_choice": true,
"deprecation_date": "2025-04-14"
},
"groq/llama3-70b-8192": {
"max_tokens": 8192,
"max_input_tokens": 8192,
"max_output_tokens": 8192,
"input_cost_per_token": 5.9e-07,
"output_cost_per_token": 7.9e-07,
"litellm_provider": "groq",
"mode": "chat",
"supports_response_schema": true,
"supports_tool_choice": true
},
"groq/llama-3.1-8b-instant": {
"max_tokens": 8192,
"max_input_tokens": 128000,
@ -11991,6 +11970,108 @@
"mode": "chat",
"supports_tool_choice": true
},
"openrouter/openai/gpt-4.1": {
"max_tokens": 32768,
"max_input_tokens": 1047576,
"max_output_tokens": 32768,
"input_cost_per_token": 2e-06,
"output_cost_per_token": 8e-06,
"cache_read_input_token_cost": 5e-07,
"litellm_provider": "openrouter",
"mode": "chat",
"supports_function_calling": true,
"supports_parallel_function_calling": true,
"supports_response_schema": true,
"supports_vision": true,
"supports_prompt_caching": true,
"supports_system_messages": true,
"supports_tool_choice": true
},
"openrouter/openai/gpt-4.1-2025-04-14": {
"max_tokens": 32768,
"max_input_tokens": 1047576,
"max_output_tokens": 32768,
"input_cost_per_token": 2e-06,
"output_cost_per_token": 8e-06,
"cache_read_input_token_cost": 5e-07,
"litellm_provider": "openrouter",
"mode": "chat",
"supports_function_calling": true,
"supports_parallel_function_calling": true,
"supports_response_schema": true,
"supports_vision": true,
"supports_prompt_caching": true,
"supports_system_messages": true,
"supports_tool_choice": true
},
"openrouter/openai/gpt-4.1-mini": {
"max_tokens": 32768,
"max_input_tokens": 1047576,
"max_output_tokens": 32768,
"input_cost_per_token": 4e-07,
"output_cost_per_token": 1.6e-06,
"cache_read_input_token_cost": 1e-07,
"litellm_provider": "openrouter",
"mode": "chat",
"supports_function_calling": true,
"supports_parallel_function_calling": true,
"supports_response_schema": true,
"supports_vision": true,
"supports_prompt_caching": true,
"supports_system_messages": true,
"supports_tool_choice": true
},
"openrouter/openai/gpt-4.1-mini-2025-04-14": {
"max_tokens": 32768,
"max_input_tokens": 1047576,
"max_output_tokens": 32768,
"input_cost_per_token": 4e-07,
"output_cost_per_token": 1.6e-06,
"cache_read_input_token_cost": 1e-07,
"litellm_provider": "openrouter",
"mode": "chat",
"supports_function_calling": true,
"supports_parallel_function_calling": true,
"supports_response_schema": true,
"supports_vision": true,
"supports_prompt_caching": true,
"supports_system_messages": true,
"supports_tool_choice": true
},
"openrouter/openai/gpt-4.1-nano": {
"max_tokens": 32768,
"max_input_tokens": 1047576,
"max_output_tokens": 32768,
"input_cost_per_token": 1e-07,
"output_cost_per_token": 4e-07,
"cache_read_input_token_cost": 2.5e-08,
"litellm_provider": "openrouter",
"mode": "chat",
"supports_function_calling": true,
"supports_parallel_function_calling": true,
"supports_response_schema": true,
"supports_vision": true,
"supports_prompt_caching": true,
"supports_system_messages": true,
"supports_tool_choice": true
},
"openrouter/openai/gpt-4.1-nano-2025-04-14": {
"max_tokens": 32768,
"max_input_tokens": 1047576,
"max_output_tokens": 32768,
"input_cost_per_token": 1e-07,
"output_cost_per_token": 4e-07,
"cache_read_input_token_cost": 2.5e-08,
"litellm_provider": "openrouter",
"mode": "chat",
"supports_function_calling": true,
"supports_parallel_function_calling": true,
"supports_response_schema": true,
"supports_vision": true,
"supports_prompt_caching": true,
"supports_system_messages": true,
"supports_tool_choice": true
},
"openrouter/openai/gpt-5-mini": {
"max_tokens": 128000,
"max_input_tokens": 400000,
@ -14970,10 +15051,10 @@
"output_cost_per_token": 6e-06,
"max_input_tokens": 262000,
"litellm_provider": "together_ai",
"supports_function_calling": false,
"supports_parallel_function_calling": false,
"supports_function_calling": true,
"supports_parallel_function_calling": true,
"mode": "chat",
"supports_tool_choice": false,
"supports_tool_choice": true,
"source": "https://www.together.ai/models/qwen3-235b-a22b-instruct-2507-fp8"
},
"together_ai/Qwen/Qwen3-Coder-480B-A35B-Instruct-FP8": {
@ -14981,10 +15062,10 @@
"output_cost_per_token": 2e-06,
"max_input_tokens": 256000,
"litellm_provider": "together_ai",
"supports_function_calling": false,
"supports_parallel_function_calling": false,
"supports_function_calling": true,
"supports_parallel_function_calling": true,
"mode": "chat",
"supports_tool_choice": false,
"supports_tool_choice": true,
"source": "https://www.together.ai/models/qwen3-coder-480b-a35b-instruct"
},
"together_ai/Qwen/Qwen3-235B-A22B-Thinking-2507": {
@ -14992,10 +15073,10 @@
"output_cost_per_token": 3e-06,
"max_input_tokens": 256000,
"litellm_provider": "together_ai",
"supports_function_calling": false,
"supports_parallel_function_calling": false,
"supports_function_calling": true,
"supports_parallel_function_calling": true,
"mode": "chat",
"supports_tool_choice": false,
"supports_tool_choice": true,
"source": "https://www.together.ai/models/qwen3-235b-a22b-thinking-2507"
},
"together_ai/Qwen/Qwen3-235B-A22B-fp8-tput": {
@ -15038,10 +15119,10 @@
"output_cost_per_token": 2.19e-06,
"max_input_tokens": 128000,
"litellm_provider": "together_ai",
"supports_function_calling": false,
"supports_parallel_function_calling": false,
"supports_function_calling": true,
"supports_parallel_function_calling": true,
"mode": "chat",
"supports_tool_choice": false,
"supports_tool_choice": true,
"source": "https://www.together.ai/models/deepseek-r1-0528-throughput"
},
"together_ai/mistralai/Mistral-Small-24B-Instruct-2501": {
@ -15066,9 +15147,9 @@
"output_cost_per_token": 6e-07,
"max_input_tokens": 128000,
"litellm_provider": "together_ai",
"supports_function_calling": false,
"supports_tool_choice": false,
"supports_parallel_function_calling": false,
"supports_function_calling": true,
"supports_tool_choice": true,
"supports_parallel_function_calling": true,
"mode": "chat",
"source": "https://www.together.ai/models/gpt-oss-120b"
},
@ -15077,9 +15158,9 @@
"output_cost_per_token": 2e-07,
"max_input_tokens": 128000,
"litellm_provider": "together_ai",
"supports_function_calling": false,
"supports_tool_choice": false,
"supports_parallel_function_calling": false,
"supports_function_calling": true,
"supports_tool_choice": true,
"supports_parallel_function_calling": true,
"mode": "chat",
"source": "https://www.together.ai/models/gpt-oss-20b"
},
@ -15088,12 +15169,24 @@
"output_cost_per_token": 1.1e-06,
"max_input_tokens": 128000,
"litellm_provider": "together_ai",
"supports_function_calling": false,
"supports_tool_choice": false,
"supports_parallel_function_calling": false,
"supports_function_calling": true,
"supports_tool_choice": true,
"supports_parallel_function_calling": true,
"mode": "chat",
"source": "https://www.together.ai/models/glm-4-5-air"
},
"together_ai/deepseek-ai/DeepSeek-V3.1": {
"input_cost_per_token": 0.6e-06,
"output_cost_per_token": 1.7e-06,
"max_tokens": 128000,
"litellm_provider": "together_ai",
"supports_function_calling": true,
"supports_parallel_function_calling": true,
"supports_reasoning": true,
"mode": "chat",
"supports_tool_choice": true,
"source": "https://www.together.ai/models/deepseek-v3-1"
},
"ollama/codegemma": {
"max_tokens": 8192,
"max_input_tokens": 8192,

4
litellm/proxy/_new_secret_config.yaml
View file

@ -21,3 +21,7 @@ model_list:
router_settings:
model_group_alias: {"my-fake-gpt-4": "fake-openai-endpoint"}
litellm_settings:
callbacks: ["otel"]
success_callback: ["braintrust"]

1
litellm/proxy/_types.py
View file

@ -2904,6 +2904,7 @@ class LitellmDataForBackendLLMCall(TypedDict, total=False):
headers: dict
organization: str
timeout: Optional[float]
stream_timeout: Optional[float]
user: Optional[str]
num_retries: Optional[int]

27
litellm/proxy/google_endpoints/endpoints.py
View file

@ -173,15 +173,24 @@ async def google_count_tokens(request: Request, model_name: str):
"""
from litellm.proxy.common_utils.http_parsing_utils import _read_request_body
from litellm.proxy.proxy_server import token_counter as internal_token_counter
from litellm.google_genai.adapters.transformation import GoogleGenAIAdapter
data = await _read_request_body(request=request)
contents = data.get("contents", [])
#Create TokenCountRequest for the internal endpoint
from litellm.proxy._types import TokenCountRequest
# Translate contents to openai format messages using the adapter
messages = (
GoogleGenAIAdapter()
.translate_generate_content_to_completion(model_name, contents)
.get("messages", [])
)
token_request = TokenCountRequest(
model=model_name,
contents=contents
contents=contents,
messages=messages, # compatibility when use openai-like endpoint
)
# Call the internal token counter function with direct request flag set to False
@ -192,11 +201,17 @@ async def google_count_tokens(request: Request, model_name: str):
if token_response is not None:
# cast the response to the well known format
original_response: dict = token_response.original_response or {}
return TokenCountDetailsResponse(
totalTokens=original_response.get("totalTokens", 0),
promptTokensDetails=original_response.get("promptTokensDetails", []),
)
if original_response:
return TokenCountDetailsResponse(
totalTokens=original_response.get("totalTokens", 0),
promptTokensDetails=original_response.get("promptTokensDetails", []),
)
else:
return TokenCountDetailsResponse(
totalTokens=token_response.total_tokens or 0,
promptTokensDetails=[],
)
#########################################################
# Return the response in the well known format
#########################################################

14
litellm/proxy/litellm_pre_call_utils.py
View file

@ -271,6 +271,16 @@ class LiteLLMProxyRequestSetup:
if timeout_header is not None:
return float(timeout_header)
return None
@staticmethod
def _get_stream_timeout_from_request(headers: dict) -> Optional[float]:
"""
Get the `stream_timeout` from the request headers.
"""
stream_timeout_header = headers.get("x-litellm-stream-timeout", None)
if stream_timeout_header is not None:
return float(stream_timeout_header)
return None
@staticmethod
def _get_num_retries_from_request(headers: dict) -> Optional[int]:
@ -439,6 +449,10 @@ class LiteLLMProxyRequestSetup:
timeout = LiteLLMProxyRequestSetup._get_timeout_from_request(headers)
if timeout is not None:
data["timeout"] = timeout
stream_timeout = LiteLLMProxyRequestSetup._get_stream_timeout_from_request(headers)
if stream_timeout is not None:
data["stream_timeout"] = stream_timeout
num_retries = LiteLLMProxyRequestSetup._get_num_retries_from_request(headers)
if num_retries is not None:

6
litellm/proxy/proxy_cli.py
View file

@ -12,6 +12,8 @@ import click
import httpx
from dotenv import load_dotenv
from litellm.constants import DEFAULT_NUM_WORKERS_LITELLM_PROXY
if TYPE_CHECKING:
from fastapi import FastAPI
else:
@ -308,8 +310,8 @@ class ProxyInitializationHelpers:
@click.option("--port", default=4000, help="Port to bind the server to.", envvar="PORT")
@click.option(
"--num_workers",
default=1,
help="Number of uvicorn / gunicorn workers to spin up. By default, 1 uvicorn is used.",
default=DEFAULT_NUM_WORKERS_LITELLM_PROXY,
help="Number of uvicorn / gunicorn workers to spin up. By default, 4 uvicorn workers are used.",
envvar="NUM_WORKERS",
)
@click.option("--api_base", default=None, help="API base URL.")

6
litellm/proxy/proxy_config.yaml
View file

@ -1,4 +1,6 @@
model_list:
- model_name: xai/*
- model_name: db-openai-endpoint
litellm_params:
model: xai/*
model: openai/*
api_base: https://exampleopenaiendpoint-production-0ee2.up.railway.app/
mock_response: "hi"

36
litellm/types/integrations/prometheus.py
View file

@ -154,6 +154,7 @@ class UserAPIKeyLabelNames(Enum):
DEFINED_PROMETHEUS_METRICS = Literal[
"litellm_llm_api_latency_metric",
"litellm_llm_api_time_to_first_token_metric",
"litellm_request_total_latency_metric",
"litellm_overhead_latency_metric",
"litellm_remaining_requests_metric",
@ -162,6 +163,7 @@ DEFINED_PROMETHEUS_METRICS = Literal[
"litellm_proxy_failed_requests_metric",
"litellm_deployment_latency_per_output_token",
"litellm_requests_metric",
"litellm_spend_metric",
"litellm_total_tokens_metric",
"litellm_input_tokens_metric",
"litellm_output_tokens_metric",
@ -173,9 +175,11 @@ DEFINED_PROMETHEUS_METRICS = Literal[
"litellm_remaining_api_key_budget_metric",
"litellm_api_key_max_budget_metric",
"litellm_api_key_budget_remaining_hours_metric",
"litellm_deployment_state",
"litellm_deployment_failure_responses",
"litellm_deployment_total_requests",
"litellm_deployment_success_responses",
"litellm_deployment_cooled_down",
"litellm_pod_lock_manager_size",
"litellm_in_memory_daily_spend_update_queue_size",
"litellm_redis_daily_spend_update_queue_size",
@ -196,6 +200,14 @@ class PrometheusMetricLabels:
UserAPIKeyLabelNames.USER.value,
]
litellm_llm_api_time_to_first_token_metric = [
UserAPIKeyLabelNames.v1_LITELLM_MODEL_NAME.value,
UserAPIKeyLabelNames.API_KEY_HASH.value,
UserAPIKeyLabelNames.API_KEY_ALIAS.value,
UserAPIKeyLabelNames.TEAM.value,
UserAPIKeyLabelNames.TEAM_ALIAS.value,
]
litellm_request_total_latency_metric = [
UserAPIKeyLabelNames.END_USER.value,
UserAPIKeyLabelNames.API_KEY_HASH.value,
@ -282,6 +294,16 @@ class PrometheusMetricLabels:
UserAPIKeyLabelNames.USER_EMAIL.value,
]
litellm_spend_metric = [
UserAPIKeyLabelNames.END_USER.value,
UserAPIKeyLabelNames.API_KEY_HASH.value,
UserAPIKeyLabelNames.API_KEY_ALIAS.value,
UserAPIKeyLabelNames.v1_LITELLM_MODEL_NAME.value,
UserAPIKeyLabelNames.TEAM.value,
UserAPIKeyLabelNames.TEAM_ALIAS.value,
UserAPIKeyLabelNames.USER.value,
]
litellm_input_tokens_metric = [
UserAPIKeyLabelNames.END_USER.value,
UserAPIKeyLabelNames.API_KEY_HASH.value,
@ -315,6 +337,20 @@ class PrometheusMetricLabels:
UserAPIKeyLabelNames.REQUESTED_MODEL.value,
]
litellm_deployment_state = [
UserAPIKeyLabelNames.v2_LITELLM_MODEL_NAME.value,
UserAPIKeyLabelNames.MODEL_ID.value,
UserAPIKeyLabelNames.API_BASE.value,
UserAPIKeyLabelNames.API_PROVIDER.value,
]
litellm_deployment_cooled_down = [
UserAPIKeyLabelNames.v2_LITELLM_MODEL_NAME.value,
UserAPIKeyLabelNames.MODEL_ID.value,
UserAPIKeyLabelNames.API_BASE.value,
UserAPIKeyLabelNames.API_PROVIDER.value,
]
litellm_deployment_successful_fallbacks = [
UserAPIKeyLabelNames.REQUESTED_MODEL.value,
UserAPIKeyLabelNames.FALLBACK_MODEL.value,

5
litellm/types/llms/vertex_ai.py
View file

@ -41,6 +41,7 @@ class PartType(TypedDict, total=False):
function_call: FunctionCall
function_response: FunctionResponse
thought: bool
thoughtSignature: str
class HttpxFunctionCall(TypedDict):
@ -72,6 +73,7 @@ class HttpxPartType(TypedDict, total=False):
executableCode: HttpxExecutableCode
codeExecutionResult: HttpxCodeExecutionResult
thought: bool
thoughtSignature: str
class HttpxContentType(TypedDict, total=False):
@ -245,10 +247,11 @@ class UsageMetadata(TypedDict, total=False):
class TokenCountDetailsResponse(TypedDict):
"""
Response structure for token count details with modality breakdown.
Example:
{'totalTokens': 12, 'promptTokensDetails': [{'modality': 'TEXT', 'tokenCount': 12}]}
"""
totalTokens: int
promptTokensDetails: List[PromptTokensDetails]

177
model_prices_and_context_window.json
View file

@ -5817,16 +5817,6 @@
"supports_response_schema": true,
"supports_tool_choice": true
},
"groq/llama3-8b-8192": {
"max_tokens": 8192,
"max_input_tokens": 8192,
"max_output_tokens": 8192,
"input_cost_per_token": 5e-08,
"output_cost_per_token": 8e-08,
"litellm_provider": "groq",
"mode": "chat",
"supports_tool_choice": true
},
"groq/llama-3.2-1b-preview": {
"max_tokens": 8192,
"max_input_tokens": 8192,
@ -5907,17 +5897,6 @@
"supports_tool_choice": true,
"deprecation_date": "2025-04-14"
},
"groq/llama3-70b-8192": {
"max_tokens": 8192,
"max_input_tokens": 8192,
"max_output_tokens": 8192,
"input_cost_per_token": 5.9e-07,
"output_cost_per_token": 7.9e-07,
"litellm_provider": "groq",
"mode": "chat",
"supports_response_schema": true,
"supports_tool_choice": true
},
"groq/llama-3.1-8b-instant": {
"max_tokens": 8192,
"max_input_tokens": 128000,
@ -11991,6 +11970,108 @@
"mode": "chat",
"supports_tool_choice": true
},
"openrouter/openai/gpt-4.1": {
"max_tokens": 32768,
"max_input_tokens": 1047576,
"max_output_tokens": 32768,
"input_cost_per_token": 2e-06,
"output_cost_per_token": 8e-06,
"cache_read_input_token_cost": 5e-07,
"litellm_provider": "openrouter",
"mode": "chat",
"supports_function_calling": true,
"supports_parallel_function_calling": true,
"supports_response_schema": true,
"supports_vision": true,
"supports_prompt_caching": true,
"supports_system_messages": true,
"supports_tool_choice": true
},
"openrouter/openai/gpt-4.1-2025-04-14": {
"max_tokens": 32768,
"max_input_tokens": 1047576,
"max_output_tokens": 32768,
"input_cost_per_token": 2e-06,
"output_cost_per_token": 8e-06,
"cache_read_input_token_cost": 5e-07,
"litellm_provider": "openrouter",
"mode": "chat",
"supports_function_calling": true,
"supports_parallel_function_calling": true,
"supports_response_schema": true,
"supports_vision": true,
"supports_prompt_caching": true,
"supports_system_messages": true,
"supports_tool_choice": true
},
"openrouter/openai/gpt-4.1-mini": {
"max_tokens": 32768,
"max_input_tokens": 1047576,
"max_output_tokens": 32768,
"input_cost_per_token": 4e-07,
"output_cost_per_token": 1.6e-06,
"cache_read_input_token_cost": 1e-07,
"litellm_provider": "openrouter",
"mode": "chat",
"supports_function_calling": true,
"supports_parallel_function_calling": true,
"supports_response_schema": true,
"supports_vision": true,
"supports_prompt_caching": true,
"supports_system_messages": true,
"supports_tool_choice": true
},
"openrouter/openai/gpt-4.1-mini-2025-04-14": {
"max_tokens": 32768,
"max_input_tokens": 1047576,
"max_output_tokens": 32768,
"input_cost_per_token": 4e-07,
"output_cost_per_token": 1.6e-06,
"cache_read_input_token_cost": 1e-07,
"litellm_provider": "openrouter",
"mode": "chat",
"supports_function_calling": true,
"supports_parallel_function_calling": true,
"supports_response_schema": true,
"supports_vision": true,
"supports_prompt_caching": true,
"supports_system_messages": true,
"supports_tool_choice": true
},
"openrouter/openai/gpt-4.1-nano": {
"max_tokens": 32768,
"max_input_tokens": 1047576,
"max_output_tokens": 32768,
"input_cost_per_token": 1e-07,
"output_cost_per_token": 4e-07,
"cache_read_input_token_cost": 2.5e-08,
"litellm_provider": "openrouter",
"mode": "chat",
"supports_function_calling": true,
"supports_parallel_function_calling": true,
"supports_response_schema": true,
"supports_vision": true,
"supports_prompt_caching": true,
"supports_system_messages": true,
"supports_tool_choice": true
},
"openrouter/openai/gpt-4.1-nano-2025-04-14": {
"max_tokens": 32768,
"max_input_tokens": 1047576,
"max_output_tokens": 32768,
"input_cost_per_token": 1e-07,
"output_cost_per_token": 4e-07,
"cache_read_input_token_cost": 2.5e-08,
"litellm_provider": "openrouter",
"mode": "chat",
"supports_function_calling": true,
"supports_parallel_function_calling": true,
"supports_response_schema": true,
"supports_vision": true,
"supports_prompt_caching": true,
"supports_system_messages": true,
"supports_tool_choice": true
},
"openrouter/openai/gpt-5-mini": {
"max_tokens": 128000,
"max_input_tokens": 400000,
@ -14970,10 +15051,10 @@
"output_cost_per_token": 6e-06,
"max_input_tokens": 262000,
"litellm_provider": "together_ai",
"supports_function_calling": false,
"supports_parallel_function_calling": false,
"supports_function_calling": true,
"supports_parallel_function_calling": true,
"mode": "chat",
"supports_tool_choice": false,
"supports_tool_choice": true,
"source": "https://www.together.ai/models/qwen3-235b-a22b-instruct-2507-fp8"
},
"together_ai/Qwen/Qwen3-Coder-480B-A35B-Instruct-FP8": {
@ -14981,10 +15062,10 @@
"output_cost_per_token": 2e-06,
"max_input_tokens": 256000,
"litellm_provider": "together_ai",
"supports_function_calling": false,
"supports_parallel_function_calling": false,
"supports_function_calling": true,
"supports_parallel_function_calling": true,
"mode": "chat",
"supports_tool_choice": false,
"supports_tool_choice": true,
"source": "https://www.together.ai/models/qwen3-coder-480b-a35b-instruct"
},
"together_ai/Qwen/Qwen3-235B-A22B-Thinking-2507": {
@ -14992,10 +15073,10 @@
"output_cost_per_token": 3e-06,
"max_input_tokens": 256000,
"litellm_provider": "together_ai",
"supports_function_calling": false,
"supports_parallel_function_calling": false,
"supports_function_calling": true,
"supports_parallel_function_calling": true,
"mode": "chat",
"supports_tool_choice": false,
"supports_tool_choice": true,
"source": "https://www.together.ai/models/qwen3-235b-a22b-thinking-2507"
},
"together_ai/Qwen/Qwen3-235B-A22B-fp8-tput": {
@ -15038,10 +15119,10 @@
"output_cost_per_token": 2.19e-06,
"max_input_tokens": 128000,
"litellm_provider": "together_ai",
"supports_function_calling": false,
"supports_parallel_function_calling": false,
"supports_function_calling": true,
"supports_parallel_function_calling": true,
"mode": "chat",
"supports_tool_choice": false,
"supports_tool_choice": true,
"source": "https://www.together.ai/models/deepseek-r1-0528-throughput"
},
"together_ai/mistralai/Mistral-Small-24B-Instruct-2501": {
@ -15066,9 +15147,9 @@
"output_cost_per_token": 6e-07,
"max_input_tokens": 128000,
"litellm_provider": "together_ai",
"supports_function_calling": false,
"supports_tool_choice": false,
"supports_parallel_function_calling": false,
"supports_function_calling": true,
"supports_tool_choice": true,
"supports_parallel_function_calling": true,
"mode": "chat",
"source": "https://www.together.ai/models/gpt-oss-120b"
},
@ -15077,9 +15158,9 @@
"output_cost_per_token": 2e-07,
"max_input_tokens": 128000,
"litellm_provider": "together_ai",
"supports_function_calling": false,
"supports_tool_choice": false,
"supports_parallel_function_calling": false,
"supports_function_calling": true,
"supports_tool_choice": true,
"supports_parallel_function_calling": true,
"mode": "chat",
"source": "https://www.together.ai/models/gpt-oss-20b"
},
@ -15088,12 +15169,24 @@
"output_cost_per_token": 1.1e-06,
"max_input_tokens": 128000,
"litellm_provider": "together_ai",
"supports_function_calling": false,
"supports_tool_choice": false,
"supports_parallel_function_calling": false,
"supports_function_calling": true,
"supports_tool_choice": true,
"supports_parallel_function_calling": true,
"mode": "chat",
"source": "https://www.together.ai/models/glm-4-5-air"
},
"together_ai/deepseek-ai/DeepSeek-V3.1": {
"input_cost_per_token": 0.6e-06,
"output_cost_per_token": 1.7e-06,
"max_tokens": 128000,
"litellm_provider": "together_ai",
"supports_function_calling": true,
"supports_parallel_function_calling": true,
"supports_reasoning": true,
"mode": "chat",
"supports_tool_choice": true,
"source": "https://www.together.ai/models/deepseek-v3-1"
},
"ollama/codegemma": {
"max_tokens": 8192,
"max_input_tokens": 8192,

4
tests/code_coverage_tests/recursive_detector.py
View file

@ -25,7 +25,9 @@ IGNORE_FUNCTIONS = [
"filter_value_from_dict", # max depth set.
"normalize_json_schema_types", # max depth set.
"_extract_fields_recursive", # max depth set.
"_remove_json_schema_refs", # max depth set.
"_remove_json_schema_refs", # max depth set.,
"_convert_schema_types", # max depth set.,
"_fix_enum_empty_strings", # max depth set.,
]

2
tests/litellm_utils_tests/test_utils.py
View file

@ -1054,7 +1054,7 @@ def test_parse_content_for_reasoning(content, expected_reasoning, expected_conte
("gemini/gemini-1.5-pro", True),
("predibase/llama3-8b-instruct", True),
("gpt-3.5-turbo", False),
("groq/llama3-70b-8192", True),
("groq/llama-3.3-70b-versatile", True),
],
)
def test_supports_response_schema(model, expected_bool):

102
tests/llm_translation/base_llm_unit_tests.py
View file

@ -141,6 +141,108 @@ class BaseLLMChatTest(ABC):
# for OpenAI the content contains the JSON schema, so we need to assert that the content is not None
assert response.choices[0].message.content is not None
def test_tool_call_with_property_type_array(self):
litellm._turn_on_debug()
from litellm.utils import supports_function_calling
os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True"
litellm.model_cost = litellm.get_model_cost_map(url="")
base_completion_call_args = self.get_base_completion_call_args()
if not supports_function_calling(base_completion_call_args["model"], None):
print("Model does not support function calling")
pytest.skip("Model does not support function calling")
base_completion_call_args = self.get_base_completion_call_args()
response = self.completion_function(
**base_completion_call_args,
messages = [
{
"role": "user",
"content": "Tell me if the shoe brand Air Jordan has more models than the shoe brand Nike."
}
],
tools = [
{
"type": "function",
"function": {
"name": "shoe_get_id",
"description": "Get information about a show by its ID or name",
"parameters": {
"type": "object",
"properties": {
"shoe_id": {
"type": ["string", "number"],
"description": "The shoe ID or name"
}
},
"required": ["shoe_id"],
"additionalProperties": False,
"$schema": "http://json-schema.org/draft-07/schema#"
}
}
},
]
)
print(response)
print(json.dumps(response, indent=4, default=str))
def test_tool_call_with_empty_enum_property(self):
litellm._turn_on_debug()
from litellm.utils import supports_function_calling
os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True"
litellm.model_cost = litellm.get_model_cost_map(url="")
base_completion_call_args = self.get_base_completion_call_args()
if not supports_function_calling(base_completion_call_args["model"], None):
print("Model does not support function calling")
pytest.skip("Model does not support function calling")
base_completion_call_args = self.get_base_completion_call_args()
response = self.completion_function(
**base_completion_call_args,
messages = [
{
"role": "user",
"content": "Search for the latest iPhone models and tell me which storage options are available."
}
],
tools = [
{
"type": "function",
"function": {
"name": "litellm_product_search",
"description": "Search for product information and specifications.\n\nSupports filtering by category, brand, price range, and availability.\nCan retrieve detailed product specifications, pricing, and stock information.\nSupports different search modes and result formatting options.\n",
"parameters": {
"properties": {
"search_mode": {
"default": "",
"description": "The search strategy to use for finding products.",
"enum": [
"",
"product_search",
"product_search_with_filters",
"product_search_with_sorting",
"product_search_with_pagination",
"product_search_with_aggregation",
],
"title": "Search Mode",
"type": "string"
},
},
"required": [
"search_mode"
],
"title": "product_search_arguments",
"type": "object"
}
}
}
]
)
print(response)
print(json.dumps(response, indent=4, default=str))
def test_streaming(self):
"""Check if litellm handles streaming correctly"""
from litellm.types.utils import ModelResponseStream

166
tests/llm_translation/test_gemini.py
View file

@ -436,7 +436,10 @@ def test_gemini_with_empty_function_call_arguments():
async def test_claude_tool_use_with_gemini():
response = await litellm.anthropic.messages.acreate(
messages=[
{"role": "user", "content": "Hello, can you tell me the weather in Boston. Please respond with a tool call?"}
{
"role": "user",
"content": "Hello, can you tell me the weather in Boston. Please respond with a tool call?",
}
],
model="gemini/gemini-2.5-flash",
stream=True,
@ -578,11 +581,17 @@ def test_gemini_tool_use():
assert stop_reason is not None
assert stop_reason == "tool_calls"
@pytest.mark.asyncio
async def test_gemini_image_generation_async():
litellm._turn_on_debug()
response = await litellm.acompletion(
messages=[{"role": "user", "content": "Generate an image of a banana wearing a costume that says LiteLLM"}],
messages=[
{
"role": "user",
"content": "Generate an image of a banana wearing a costume that says LiteLLM",
}
],
model="gemini/gemini-2.5-flash-image-preview",
)
@ -597,12 +606,16 @@ async def test_gemini_image_generation_async():
assert IMAGE_URL["url"].startswith("data:image/png;base64,")
@pytest.mark.asyncio
async def test_gemini_image_generation_async_stream():
#litellm._turn_on_debug()
# litellm._turn_on_debug()
response = await litellm.acompletion(
messages=[{"role": "user", "content": "Generate an image of a banana wearing a costume that says LiteLLM"}],
messages=[
{
"role": "user",
"content": "Generate an image of a banana wearing a costume that says LiteLLM",
}
],
model="gemini/gemini-2.5-flash-image-preview",
stream=True,
)
@ -611,35 +624,144 @@ async def test_gemini_image_generation_async_stream():
model_response_image = None
async for chunk in response:
print("CHUNK: ", chunk)
if hasattr(chunk.choices[0].delta, "image") and chunk.choices[0].delta.image is not None:
if (
hasattr(chunk.choices[0].delta, "image")
and chunk.choices[0].delta.image is not None
):
model_response_image = chunk.choices[0].delta.image
print("MODEL_RESPONSE_IMAGE: ", model_response_image)
assert model_response_image is not None
assert model_response_image["url"].startswith("data:image/png;base64,")
break
#########################################################
# Important: Validate we did get an image in the response
#########################################################
assert model_response_image is not None
assert model_response_image["url"].startswith("data:image/png;base64,")
def test_system_message_with_no_user_message():
"""
Test that the system message is translated correctly for non-OpenAI providers.
"""
messages = [
{
"role": "system",
"content": "Be a good bot!",
},
]
"""
Test that the system message is translated correctly for non-OpenAI providers.
"""
messages = [
{
"role": "system",
"content": "Be a good bot!",
},
]
response = litellm.completion(
model="gemini/gemini-2.5-flash",
messages=messages,
response = litellm.completion(
model="gemini/gemini-2.5-flash",
messages=messages,
)
assert response is not None
assert response.choices[0].message.content is not None
def get_current_weather(location, unit="fahrenheit"):
"""Get the current weather in a given location"""
if "tokyo" in location.lower():
return json.dumps({"location": "Tokyo", "temperature": "10", "unit": "celsius"})
elif "san francisco" in location.lower():
return json.dumps(
{"location": "San Francisco", "temperature": "72", "unit": "fahrenheit"}
)
assert response is not None
elif "paris" in location.lower():
return json.dumps({"location": "Paris", "temperature": "22", "unit": "celsius"})
else:
return json.dumps({"location": location, "temperature": "unknown"})
assert response.choices[0].message.content is not None
def test_gemini_with_thinking():
from litellm import completion
litellm._turn_on_debug()
litellm.modify_params = True
model = "gemini/gemini-2.5-flash"
messages = [
{
"role": "user",
"content": "What's the weather like in San Francisco, Tokyo, and Paris? - give me 3 responses",
}
]
tools = [
{
"type": "function",
"function": {
"name": "get_current_weather",
"description": "Get the current weather in a given location",
"parameters": {
"type": "object",
"properties": {
"location": {
"type": "string",
"description": "The city and state",
},
"unit": {
"type": "string",
"enum": ["celsius", "fahrenheit"],
},
},
"required": ["location"],
},
},
}
]
response = litellm.completion(
model=model,
messages=messages,
tools=tools,
tool_choice="auto", # auto is default, but we'll be explicit
reasoning_effort="low",
)
print("Response\n", response)
response_message = response.choices[0].message
tool_calls = response_message.tool_calls
print("Expecting there to be 3 tool calls")
assert len(tool_calls) > 0 # this has to call the function for SF, Tokyo and paris
# Step 2: check if the model wanted to call a function
print(f"tool_calls: {tool_calls}")
if tool_calls:
# Step 3: call the function
# Note: the JSON response may not always be valid; be sure to handle errors
available_functions = {
"get_current_weather": get_current_weather,
} # only one function in this example, but you can have multiple
messages.append(response_message) # extend conversation with assistant's reply
print("Response message\n", response_message)
# Step 4: send the info for each function call and function response to the model
for tool_call in tool_calls:
function_name = tool_call.function.name
if function_name not in available_functions:
# the model called a function that does not exist in available_functions - don't try calling anything
return
function_to_call = available_functions[function_name]
function_args = json.loads(tool_call.function.arguments)
function_response = function_to_call(
location=function_args.get("location"),
unit=function_args.get("unit"),
)
messages.append(
{
"tool_call_id": tool_call.id,
"role": "tool",
"name": function_name,
"content": function_response,
}
) # extend conversation with function response
print(f"messages: {messages}")
second_response = litellm.completion(
model=model,
messages=messages,
seed=22,
reasoning_effort="low",
tools=tools,
drop_params=True,
) # get a new response from the model where it can see the function response
print("second response\n", second_response)

2
tests/local_testing/test_completion_cost.py
View file

@ -565,7 +565,7 @@ def test_groq_response_cost_tracking(is_streaming):
response_cost = litellm.response_cost_calculator(
response_object=response,
model="groq/llama3-70b-8192",
model="groq/llama-3.3-70b-versatile",
custom_llm_provider="groq",
call_type=CallTypes.acompletion.value,
optional_params={},

2
tests/local_testing/test_function_calling.py
View file

@ -50,7 +50,7 @@ def get_current_weather(location, unit="fahrenheit"):
"claude-3-haiku-20240307",
"gemini/gemini-1.5-pro",
"anthropic.claude-3-sonnet-20240229-v1:0",
"groq/llama3-8b-8192",
"groq/llama-3.1-8b-instant",
"cohere_chat/command-r",
],
)

4
tests/local_testing/test_lowest_cost_routing.py
View file

@ -31,7 +31,7 @@ async def test_get_available_deployments():
},
{
"model_name": "gpt-3.5-turbo",
"litellm_params": {"model": "groq/llama3-8b-8192"},
"litellm_params": {"model": "groq/llama-3.1-8b-instant"},
"model_info": {"id": "groq-llama"},
},
]
@ -182,7 +182,7 @@ async def test_get_available_endpoints_tpm_rpm_check_async(ans_rpm):
},
{
"model_name": "gpt-3.5-turbo",
"litellm_params": {"model": "groq/llama3-8b-8192"},
"litellm_params": {"model": "groq/llama-3.1-8b-instant"},
"model_info": {"id": "5678", "rpm": non_ans_rpm},
},
]

4
tests/local_testing/test_model_alias_map.py
View file

@ -15,7 +15,7 @@ from litellm import completion, embedding
litellm.set_verbose = True
model_alias_map = {"good-model": "groq/llama3-8b-8192"}
model_alias_map = {"good-model": "groq/llama-3.1-8b-instant"}
def test_model_alias_map(caplog):
@ -35,7 +35,7 @@ def test_model_alias_map(caplog):
for log in captured_logs:
assert "ERROR" not in log
assert "llama3-8b-8192" in response.model
assert "llama-3.1-8b-instant" in response.model
except litellm.ServiceUnavailableError:
pass
except Exception as e:

2
tests/local_testing/test_router.py
View file

@ -119,7 +119,7 @@ async def test_router_provider_wildcard_routing():
print("response 2 = ", response2)
response3 = await router.acompletion(
model="groq/llama3-8b-8192",
model="groq/llama-3.1-8b-instant",
messages=[{"role": "user", "content": "hello"}],
)

6
tests/local_testing/test_router_batch_completion.py
View file

@ -44,7 +44,7 @@ async def test_batch_completion_multiple_models(mode):
{
"model_name": "groq-llama",
"litellm_params": {
"model": "groq/llama3-8b-8192",
"model": "groq/llama-3.1-8b-instant",
},
},
]
@ -143,7 +143,7 @@ async def test_batch_completion_fastest_response_streaming():
{
"model_name": "groq-llama",
"litellm_params": {
"model": "groq/llama3-8b-8192",
"model": "groq/llama-3.1-8b-instant",
},
},
]
@ -179,7 +179,7 @@ async def test_batch_completion_multiple_models_multiple_messages():
{
"model_name": "groq-llama",
"litellm_params": {
"model": "groq/llama3-8b-8192",
"model": "groq/llama-3.1-8b-instant",
},
},
]

4
tests/local_testing/test_spend_calculate_endpoint.py
View file

@ -43,7 +43,7 @@ async def test_spend_calc_model_on_router_messages():
{
"model_name": "special-llama-model",
"litellm_params": {
"model": "groq/llama3-8b-8192",
"model": "groq/llama-3.1-8b-instant",
},
}
]
@ -86,7 +86,7 @@ async def test_spend_calc_using_response():
}
],
"created": "1677652288",
"model": "groq/llama3-8b-8192",
"model": "groq/llama-3.1-8b-instant",
"object": "chat.completion",
"system_fingerprint": "fp_873a560973",
"usage": {

162
tests/test_litellm/integrations/test_braintrust_span_name.py
View file

@ -11,7 +11,7 @@ from litellm.integrations.braintrust_logging import BraintrustLogger
class TestBraintrustSpanName(unittest.TestCase):
"""Test custom span_name functionality in Braintrust logging."""
@patch('litellm.integrations.braintrust_logging.HTTPHandler')
@patch("litellm.integrations.braintrust_logging.HTTPHandler")
def test_default_span_name(self, MockHTTPHandler):
"""Test that default span name is 'Chat Completion' when not provided."""
# Mock HTTP response
@ -22,39 +22,43 @@ class TestBraintrustSpanName(unittest.TestCase):
# Setup
logger = BraintrustLogger(api_key="test-key")
logger.default_project_id = "test-project-id"
# Create a properly structured mock response
response_obj = litellm.ModelResponse(
id="test-id",
object="chat.completion",
created=1234567890,
model="gpt-3.5-turbo",
choices=[{
"index": 0,
"message": {"role": "assistant", "content": "test response"},
"finish_reason": "stop"
}],
usage={"prompt_tokens": 10, "completion_tokens": 20, "total_tokens": 30}
choices=[
{
"index": 0,
"message": {"role": "assistant", "content": "test response"},
"finish_reason": "stop",
}
],
usage={"prompt_tokens": 10, "completion_tokens": 20, "total_tokens": 30},
)
kwargs = {
"litellm_call_id": "test-call-id",
"messages": [{"role": "user", "content": "test"}],
"litellm_params": {"metadata": {}},
"model": "gpt-3.5-turbo",
"response_cost": 0.001
"response_cost": 0.001,
}
# Execute
logger.log_success_event(kwargs, response_obj, datetime.now(), datetime.now())
# Verify
call_args = mock_http_handler.post.call_args
self.assertIsNotNone(call_args)
json_data = call_args.kwargs['json']
self.assertEqual(json_data['events'][0]['span_attributes']['name'], 'Chat Completion')
json_data = call_args.kwargs["json"]
self.assertEqual(
json_data["events"][0]["span_attributes"]["name"], "Chat Completion"
)
@patch('litellm.integrations.braintrust_logging.HTTPHandler')
@patch("litellm.integrations.braintrust_logging.HTTPHandler")
def test_custom_span_name(self, MockHTTPHandler):
"""Test that custom span name is used when provided in metadata."""
# Mock HTTP response
@ -65,39 +69,43 @@ class TestBraintrustSpanName(unittest.TestCase):
# Setup
logger = BraintrustLogger(api_key="test-key")
logger.default_project_id = "test-project-id"
# Create a properly structured mock response
response_obj = litellm.ModelResponse(
id="test-id",
object="chat.completion",
created=1234567890,
model="gpt-3.5-turbo",
choices=[{
"index": 0,
"message": {"role": "assistant", "content": "test response"},
"finish_reason": "stop"
}],
usage={"prompt_tokens": 10, "completion_tokens": 20, "total_tokens": 30}
choices=[
{
"index": 0,
"message": {"role": "assistant", "content": "test response"},
"finish_reason": "stop",
}
],
usage={"prompt_tokens": 10, "completion_tokens": 20, "total_tokens": 30},
)
kwargs = {
"litellm_call_id": "test-call-id",
"messages": [{"role": "user", "content": "test"}],
"litellm_params": {"metadata": {"span_name": "Custom Operation"}},
"model": "gpt-3.5-turbo",
"response_cost": 0.001
"response_cost": 0.001,
}
# Execute
logger.log_success_event(kwargs, response_obj, datetime.now(), datetime.now())
# Verify
call_args = mock_http_handler.post.call_args
self.assertIsNotNone(call_args)
json_data = call_args.kwargs['json']
self.assertEqual(json_data['events'][0]['span_attributes']['name'], 'Custom Operation')
json_data = call_args.kwargs["json"]
self.assertEqual(
json_data["events"][0]["span_attributes"]["name"], "Custom Operation"
)
@patch('litellm.integrations.braintrust_logging.HTTPHandler')
@patch("litellm.integrations.braintrust_logging.HTTPHandler")
def test_span_name_with_other_metadata(self, MockHTTPHandler):
"""Test that span_name works alongside other metadata fields."""
# Mock HTTP response
@ -108,21 +116,23 @@ class TestBraintrustSpanName(unittest.TestCase):
# Setup
logger = BraintrustLogger(api_key="test-key")
logger.default_project_id = "test-project-id"
# Create a properly structured mock response
response_obj = litellm.ModelResponse(
id="test-id",
object="chat.completion",
created=1234567890,
model="gpt-3.5-turbo",
choices=[{
"index": 0,
"message": {"role": "assistant", "content": "test response"},
"finish_reason": "stop"
}],
usage={"prompt_tokens": 10, "completion_tokens": 20, "total_tokens": 30}
choices=[
{
"index": 0,
"message": {"role": "assistant", "content": "test response"},
"finish_reason": "stop",
}
],
usage={"prompt_tokens": 10, "completion_tokens": 20, "total_tokens": 30},
)
kwargs = {
"litellm_call_id": "test-call-id",
"messages": [{"role": "user", "content": "test"}],
@ -132,34 +142,40 @@ class TestBraintrustSpanName(unittest.TestCase):
"project_id": "custom-project",
"user_id": "user123",
"session_id": "session456",
"environment": "production"
"environment": "production",
}
},
"model": "gpt-3.5-turbo",
"response_cost": 0.001
"response_cost": 0.001,
"standard_logging_object": {
"user_id": "user123",
},
}
# Execute
logger.log_success_event(kwargs, response_obj, datetime.now(), datetime.now())
# Verify
call_args = mock_http_handler.post.call_args
self.assertIsNotNone(call_args)
json_data = call_args.kwargs['json']
# Check span name
self.assertEqual(json_data['events'][0]['span_attributes']['name'], 'Multi Metadata Test')
# Check that other metadata is preserved (except for filtered keys)
event_metadata = json_data['events'][0]['metadata']
self.assertEqual(event_metadata['user_id'], 'user123')
self.assertEqual(event_metadata['session_id'], 'session456')
self.assertEqual(event_metadata['environment'], 'production')
# Span name should be in span_attributes, not in metadata
self.assertIn('span_name', event_metadata) # span_name is also kept in metadata
json_data = call_args.kwargs["json"]
@patch('litellm.integrations.braintrust_logging.get_async_httpx_client')
# Check span name
self.assertEqual(
json_data["events"][0]["span_attributes"]["name"], "Multi Metadata Test"
)
# Check that other metadata is preserved (except for filtered keys)
event_metadata = json_data["events"][0]["metadata"]
print(event_metadata)
self.assertEqual(event_metadata["user_id"], "user123")
self.assertEqual(event_metadata["session_id"], "session456")
self.assertEqual(event_metadata["environment"], "production")
# Span name should be in span_attributes, not in metadata
self.assertIn("span_name", event_metadata) # span_name is also kept in metadata
@patch("litellm.integrations.braintrust_logging.get_async_httpx_client")
async def test_async_custom_span_name(self, mock_get_http_handler):
"""Test async logging with custom span name."""
# Mock async HTTP response
@ -170,38 +186,44 @@ class TestBraintrustSpanName(unittest.TestCase):
# Setup
logger = BraintrustLogger(api_key="test-key")
logger.default_project_id = "test-project-id"
# Create a properly structured mock response
response_obj = litellm.ModelResponse(
id="test-id",
object="chat.completion",
created=1234567890,
model="gpt-3.5-turbo",
choices=[{
"index": 0,
"message": {"role": "assistant", "content": "test response"},
"finish_reason": "stop"
}],
usage={"prompt_tokens": 10, "completion_tokens": 20, "total_tokens": 30}
choices=[
{
"index": 0,
"message": {"role": "assistant", "content": "test response"},
"finish_reason": "stop",
}
],
usage={"prompt_tokens": 10, "completion_tokens": 20, "total_tokens": 30},
)
kwargs = {
"litellm_call_id": "test-call-id",
"messages": [{"role": "user", "content": "test"}],
"litellm_params": {"metadata": {"span_name": "Async Custom Operation"}},
"model": "gpt-3.5-turbo",
"response_cost": 0.001
"response_cost": 0.001,
}
# Execute
await logger.async_log_success_event(kwargs, response_obj, datetime.now(), datetime.now())
await logger.async_log_success_event(
kwargs, response_obj, datetime.now(), datetime.now()
)
# Verify
call_args = mock_http_handler.post.call_args
self.assertIsNotNone(call_args)
json_data = call_args.kwargs['json']
self.assertEqual(json_data['events'][0]['span_attributes']['name'], 'Async Custom Operation')
json_data = call_args.kwargs["json"]
self.assertEqual(
json_data["events"][0]["span_attributes"]["name"], "Async Custom Operation"
)
if __name__ == "__main__":
unittest.main()
unittest.main()

41
tests/test_litellm/litellm_core_utils/test_image_handling.py Normal file
View file

@ -0,0 +1,41 @@
import pytest
from httpx import Request, Response
import litellm
from litellm.litellm_core_utils.prompt_templates.image_handling import (
convert_url_to_base64,
)
class DummyClient:
def get(self, url, follow_redirects=True):
return Response(status_code=404, request=Request("GET", url))
def test_invalid_image_url_raises_bad_request(monkeypatch):
monkeypatch.setattr(litellm, "module_level_client", DummyClient())
with pytest.raises(litellm.ImageFetchError) as excinfo:
convert_url_to_base64("https://invalid.example/image.png")
assert "Unable to fetch image" in str(excinfo.value)
def test_completion_with_invalid_image_url(monkeypatch):
monkeypatch.setattr(litellm, "module_level_client", DummyClient())
messages = [
{
"role": "user",
"content": [
{"type": "text", "text": "hi"},
{
"type": "image_url",
"image_url": {"url": "https://invalid.example/image.png"},
},
],
}
]
with pytest.raises(litellm.ImageFetchError) as excinfo:
litellm.completion(
model="gemini/gemini-pro", messages=messages, api_key="test"
)
assert excinfo.value.status_code == 400
assert "Unable to fetch image" in str(excinfo.value)

11
tests/test_litellm/llms/openai/test_gpt5_transformation.py
View file

@ -41,3 +41,14 @@ def test_gpt5_temperature_error(config: OpenAIConfig):
model="gpt-5",
drop_params=False,
)
def test_gpt5_unsupported_params_drop(config: OpenAIConfig):
assert "top_p" not in config.get_supported_openai_params(model="gpt-5")
params = config.map_openai_params(
non_default_params={"top_p": 0.5},
optional_params={},
model="gpt-5",
drop_params=True,
)
assert "top_p" not in params

75
tests/test_litellm/llms/vertex_ai/gemini/test_vertex_ai_gemini_transformation.py Normal file
View file

@ -0,0 +1,75 @@
from litellm.llms.vertex_ai.gemini.transformation import check_if_part_exists_in_parts
def test_check_if_part_exists_in_parts():
parts = [
{"text": "Hello", "thought": True},
{"text": "World", "thought": False},
]
part = {"text": "Hello", "thought": True}
new_part = {"text": "Hello World", "thought": True}
assert check_if_part_exists_in_parts(parts, part)
assert not check_if_part_exists_in_parts(parts, new_part, ["thought"])
assert check_if_part_exists_in_parts(parts, new_part, ["text"])
def test_check_if_part_exists_in_parts_camel_case_snake_case():
"""Test that function handles both camelCase and snake_case key variations"""
# Test snake_case to camelCase matching
parts_with_snake_case = [
{
"function_call": {
"name": "get_current_weather",
"args": {"location": "San Francisco, CA"},
}
},
{"text": "Some other content"},
]
part_with_camel_case = {
"functionCall": {
"name": "get_current_weather",
"args": {"location": "San Francisco, CA"},
}
}
# Should find match between function_call and functionCall
assert check_if_part_exists_in_parts(parts_with_snake_case, part_with_camel_case)
# Test camelCase to snake_case matching
parts_with_camel_case = [
{"functionCall": {"name": "calculate_sum", "args": {"a": 1, "b": 2}}}
]
part_with_snake_case = {
"function_call": {"name": "calculate_sum", "args": {"a": 1, "b": 2}}
}
# Should find match between functionCall and function_call
assert check_if_part_exists_in_parts(parts_with_camel_case, part_with_snake_case)
# Test no match when values differ
part_with_different_values = {
"function_call": {"name": "different_function", "args": {"x": 5}}
}
assert not check_if_part_exists_in_parts(
parts_with_snake_case, part_with_different_values
)
# Test multiple keys with mixed casing
parts_mixed = [
{
"function_call": {"name": "test"},
"thoughtSignature": "reasoning",
"text": "content",
}
]
part_mixed_casing = {
"functionCall": {"name": "test"},
"thought_signature": "reasoning",
"text": "content",
}
assert check_if_part_exists_in_parts(parts_mixed, part_mixed_casing)

124
tests/test_litellm/llms/vertex_ai/test_vertex_ai_common_utils.py
View file

@ -677,3 +677,127 @@ def test_vertex_filter_format_uri():
)
assert "uri" not in json.dumps(new_parameters)
def test_convert_schema_types_type_array_conversion():
"""
Test _convert_schema_types function handles type arrays and case conversion.
This test verifies the fix for the issue where type arrays like ["string", "number"]
would raise an exception in Vertex AI schema validation.
Relevant issue: https://github.com/BerriAI/litellm/issues/14091
"""
from litellm.llms.vertex_ai.common_utils import _convert_schema_types
# Input: OpenAI-style schema with type array (the problematic case)
input_schema = {
"type": "object",
"properties": {
"studio": {
"type": ["string", "number"],
"description": "The studio ID or name"
}
},
"required": ["studio"],
"additionalProperties": False,
"$schema": "http://json-schema.org/draft-07/schema#"
}
# Expected output: Vertex AI compatible schema with anyOf and uppercase types
expected_output = {
"type": "object",
"properties": {
"studio": {
"anyOf": [
{"type": "string"},
{"type": "number"}
],
"description": "The studio ID or name"
}
},
"required": ["studio"],
"additionalProperties": False,
"$schema": "http://json-schema.org/draft-07/schema#"
}
# Apply the transformation
_convert_schema_types(input_schema)
# Verify the transformation
assert input_schema == expected_output
# Verify specific transformations:
# 1. Root level type converted to uppercase
assert input_schema["type"] == "object"
# 2. Type array converted to anyOf format
assert "anyOf" in input_schema["properties"]["studio"]
assert "type" not in input_schema["properties"]["studio"]
# 3. Individual types in anyOf are uppercase
anyof_types = input_schema["properties"]["studio"]["anyOf"]
assert anyof_types[0]["type"] == "string"
assert anyof_types[1]["type"] == "number"
# 4. Other properties preserved
assert input_schema["properties"]["studio"]["description"] == "The studio ID or name"
assert input_schema["required"] == ["studio"]
def test_fix_enum_empty_strings():
"""
Test _fix_enum_empty_strings function replaces empty strings with None in enum arrays.
This test verifies the fix for the issue where Gemini rejects tool definitions
with empty strings in enum values, causing API failures.
Relevant issue: Gemini does not accept empty strings in enum values
"""
from litellm.llms.vertex_ai.common_utils import _fix_enum_empty_strings
# Input: Schema with empty string in enum (the problematic case)
input_schema = {
"type": "object",
"properties": {
"user_agent_type": {
"enum": ["", "desktop", "mobile", "tablet"],
"type": "string",
"description": "Device type for user agent"
}
},
"required": ["user_agent_type"]
}
# Expected output: Empty strings replaced with None
expected_output = {
"type": "object",
"properties": {
"user_agent_type": {
"enum": [None, "desktop", "mobile", "tablet"],
"type": "string",
"description": "Device type for user agent"
}
},
"required": ["user_agent_type"]
}
# Apply the transformation
_fix_enum_empty_strings(input_schema)
# Verify the transformation
assert input_schema == expected_output
# Verify specific transformations:
# 1. Empty string replaced with None
enum_values = input_schema["properties"]["user_agent_type"]["enum"]
assert "" not in enum_values
assert None in enum_values
# 2. Other enum values preserved
assert "desktop" in enum_values
assert "mobile" in enum_values
assert "tablet" in enum_values
# 3. Other properties preserved
assert input_schema["properties"]["user_agent_type"]["type"] == "string"
assert input_schema["properties"]["user_agent_type"]["description"] == "Device type for user agent"

189
tests/test_litellm/llms/xai/test_xai_cost_calculator.py Normal file
View file

@ -0,0 +1,189 @@
"""
Test suite for XAI cost calculation functionality.
"""
import math
import os
import sys
import litellm
from litellm.types.utils import (
CompletionTokensDetailsWrapper,
Usage,
)
sys.path.insert(
0, os.path.abspath("../../..")
) # Adds the parent directory to the system path
from litellm.llms.xai.cost_calculator import cost_per_token
class TestXAICostCalculator:
"""Test suite for XAI cost calculation functionality."""
def setup_method(self):
"""Set up test environment."""
os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True"
litellm.model_cost = litellm.get_model_cost_map(url="")
def test_basic_cost_calculation(self):
"""Test basic cost calculation without reasoning tokens."""
usage = Usage(prompt_tokens=12, completion_tokens=125, total_tokens=137)
prompt_cost, completion_cost = cost_per_token(model="grok-3-mini", usage=usage)
# Expected costs for grok-3-mini:
# Input: 12 tokens * $3e-7 = $0.0000036
# Output: 125 tokens * $5e-7 = $0.0000625
expected_prompt_cost = 12 * 3e-7
expected_completion_cost = 125 * 5e-7
assert math.isclose(prompt_cost, expected_prompt_cost, rel_tol=1e-10)
assert math.isclose(completion_cost, expected_completion_cost, rel_tol=1e-10)
def test_reasoning_tokens_cost_calculation(self):
"""Test cost calculation with reasoning tokens from completion_tokens_details."""
usage = Usage(
prompt_tokens=12,
completion_tokens=125,
total_tokens=1086,
completion_tokens_details=CompletionTokensDetailsWrapper(
accepted_prediction_tokens=0,
audio_tokens=0,
reasoning_tokens=949,
rejected_prediction_tokens=0,
text_tokens=None, # Not set, but doesn't matter for XAI billing
),
)
prompt_cost, completion_cost = cost_per_token(model="grok-3-mini", usage=usage)
# Expected costs for grok-3-mini:
# Input: 12 tokens * $3e-7 = $0.0000036
# Completion: (125 + 949) tokens * $5e-7 = $0.000537
expected_prompt_cost = 12 * 3e-7
expected_completion_cost = (125 + 949) * 5e-7
assert math.isclose(prompt_cost, expected_prompt_cost, rel_tol=1e-10)
assert math.isclose(completion_cost, expected_completion_cost, rel_tol=1e-10)
def test_reasoning_and_text_tokens_cost_calculation(self):
"""Test cost calculation with both reasoning and text tokens."""
usage = Usage(
prompt_tokens=12,
completion_tokens=125,
total_tokens=1086,
completion_tokens_details=CompletionTokensDetailsWrapper(
accepted_prediction_tokens=0,
audio_tokens=0,
reasoning_tokens=949,
rejected_prediction_tokens=0,
text_tokens=76, # Explicitly set (but ignored in XAI billing)
),
)
prompt_cost, completion_cost = cost_per_token(model="grok-3-mini", usage=usage)
# Expected costs for grok-3-mini:
# Input: 12 tokens * $3e-7 = $0.0000036
# Completion: (125 + 949) tokens * $5e-7 = $0.000537
# Note: text_tokens field is ignored, only completion_tokens + reasoning_tokens matters
expected_prompt_cost = 12 * 3e-7
expected_completion_cost = (125 + 949) * 5e-7
assert math.isclose(prompt_cost, expected_prompt_cost, rel_tol=1e-10)
assert math.isclose(completion_cost, expected_completion_cost, rel_tol=1e-10)
def test_grok_4_cost_calculation(self):
"""Test cost calculation for grok-4 model."""
usage = Usage(
prompt_tokens=10,
completion_tokens=200,
total_tokens=210,
completion_tokens_details=CompletionTokensDetailsWrapper(
accepted_prediction_tokens=0,
audio_tokens=0,
reasoning_tokens=150,
rejected_prediction_tokens=0,
text_tokens=50, # Ignored in XAI billing
),
)
prompt_cost, completion_cost = cost_per_token(model="grok-4", usage=usage)
# Expected costs for grok-4:
# Input: 10 tokens * $3e-6 = $0.00003
# Completion: (200 + 150) tokens * $1.5e-5 = $0.00525
expected_prompt_cost = 10 * 3e-6
expected_completion_cost = (200 + 150) * 1.5e-5
assert math.isclose(prompt_cost, expected_prompt_cost, rel_tol=1e-10)
assert math.isclose(completion_cost, expected_completion_cost, rel_tol=1e-10)
def test_grok_3_fast_beta_cost_calculation(self):
"""Test cost calculation for grok-3-fast-beta model."""
usage = Usage(
prompt_tokens=20,
completion_tokens=300,
total_tokens=320,
completion_tokens_details=CompletionTokensDetailsWrapper(
accepted_prediction_tokens=0,
audio_tokens=0,
reasoning_tokens=200,
rejected_prediction_tokens=0,
text_tokens=100, # Ignored in XAI billing
),
)
prompt_cost, completion_cost = cost_per_token(
model="grok-3-fast-beta", usage=usage
)
# Expected costs for grok-3-fast-beta:
# Input: 20 tokens * $5e-6 = $0.0001
# Completion: (300 + 200) tokens * $2.5e-5 = $0.0125
expected_prompt_cost = 20 * 5e-6
expected_completion_cost = (300 + 200) * 2.5e-5
assert math.isclose(prompt_cost, expected_prompt_cost, rel_tol=1e-10)
assert math.isclose(completion_cost, expected_completion_cost, rel_tol=1e-10)
def test_edge_case_no_completion_tokens_details(self):
"""Test cost calculation when completion_tokens_details is not present."""
usage = Usage(prompt_tokens=12, completion_tokens=125, total_tokens=137)
prompt_cost, completion_cost = cost_per_token(model="grok-3-mini", usage=usage)
# Should fall back to basic calculation
expected_prompt_cost = 12 * 3e-7
expected_completion_cost = 125 * 5e-7
assert math.isclose(prompt_cost, expected_prompt_cost, rel_tol=1e-10)
assert math.isclose(completion_cost, expected_completion_cost, rel_tol=1e-10)
def test_edge_case_large_reasoning_tokens(self):
"""Test cost calculation when reasoning_tokens is larger than completion_tokens."""
usage = Usage(
prompt_tokens=12,
completion_tokens=50, # Less than reasoning_tokens
total_tokens=62,
completion_tokens_details=CompletionTokensDetailsWrapper(
accepted_prediction_tokens=0,
audio_tokens=0,
reasoning_tokens=100, # More than completion_tokens
rejected_prediction_tokens=0,
text_tokens=None,
),
)
prompt_cost, completion_cost = cost_per_token(model="grok-3-mini", usage=usage)
# Expected costs:
# Input: 12 tokens * $3e-7 = $0.0000036
# Completion: (50 + 100) tokens * $5e-7 = $0.000075
expected_prompt_cost = 12 * 3e-7
expected_completion_cost = (50 + 100) * 5e-7
assert math.isclose(prompt_cost, expected_prompt_cost, rel_tol=1e-10)
assert math.isclose(completion_cost, expected_completion_cost, rel_tol=1e-10)

0
tests/test_litellm/proxy/google_endpoints/__init__.py Normal file
View file

49
tests/test_litellm/proxy/google_endpoints/test_endpoints.py Normal file
View file

@ -0,0 +1,49 @@
"""
Test for google_endpoints/endpoints.py
"""
import pytest
import sys, os
from dotenv import load_dotenv
from litellm.proxy.google_endpoints.endpoints import google_count_tokens
from litellm.types.llms.vertex_ai import TokenCountDetailsResponse
from starlette.requests import Request
load_dotenv()
sys.path.insert(
0, os.path.abspath("../../../..")
)
@pytest.mark.asyncio
async def test_proxy_gemini_to_openai_like_model_token_counting():
"""
Test the token counting endpoint for proxing gemini to openai-like models.
"""
response: TokenCountDetailsResponse = await google_count_tokens(
request=Request(
scope={
"type": "http",
"parsed_body": (
[
"contents"
],
{
"contents": [
{
"parts": [
{
"text": "Hello, how are you?"
}
]
}
]
}
)
}
),
model_name="volcengine/foo",
)
assert response.get("totalTokens") > 0

90
tests/test_litellm/proxy/test_common_request_processing.py
View file

@ -74,6 +74,96 @@ class TestProxyBaseLLMRequestProcessing:
pytest.fail("litellm_call_id is not a valid UUID")
assert data_passed["litellm_call_id"] == returned_data["litellm_call_id"]
@pytest.mark.asyncio
async def test_stream_timeout_header_processing(self):
"""
Test that x-litellm-stream-timeout header gets processed and added to request data as stream_timeout.
"""
from litellm.proxy.litellm_pre_call_utils import LiteLLMProxyRequestSetup
# Test with stream timeout header
headers_with_timeout = {"x-litellm-stream-timeout": "30.5"}
result = LiteLLMProxyRequestSetup._get_stream_timeout_from_request(headers_with_timeout)
assert result == 30.5
# Test without stream timeout header
headers_without_timeout = {}
result = LiteLLMProxyRequestSetup._get_stream_timeout_from_request(headers_without_timeout)
assert result is None
# Test with invalid header value (should raise ValueError when converting to float)
headers_with_invalid = {"x-litellm-stream-timeout": "invalid"}
with pytest.raises(ValueError):
LiteLLMProxyRequestSetup._get_stream_timeout_from_request(headers_with_invalid)
@pytest.mark.asyncio
async def test_add_litellm_data_to_request_with_stream_timeout_header(self):
"""
Test that x-litellm-stream-timeout header gets processed and added to request data
when calling add_litellm_data_to_request.
"""
from litellm.integrations.opentelemetry import UserAPIKeyAuth
from litellm.proxy.litellm_pre_call_utils import add_litellm_data_to_request
# Create test data with a basic completion request
test_data = {
"model": "gpt-3.5-turbo",
"messages": [{"role": "user", "content": "Hello"}]
}
# Mock request with stream timeout header
mock_request = MagicMock(spec=Request)
mock_request.headers = {"x-litellm-stream-timeout": "45.0"}
mock_request.url.path = "/v1/chat/completions"
mock_request.method = "POST"
mock_request.query_params = {}
mock_request.client = None
# Create a minimal mock with just the required attributes
mock_user_api_key_dict = MagicMock()
mock_user_api_key_dict.api_key = "test_api_key_hash"
mock_user_api_key_dict.tpm_limit = None
mock_user_api_key_dict.rpm_limit = None
mock_user_api_key_dict.max_budget = None
mock_user_api_key_dict.spend = 0
mock_user_api_key_dict.allowed_model_region = None
mock_user_api_key_dict.key_alias = None
mock_user_api_key_dict.user_id = None
mock_user_api_key_dict.team_id = None
mock_user_api_key_dict.metadata = {} # Prevent enterprise feature check
mock_user_api_key_dict.team_metadata = None
mock_user_api_key_dict.org_id = None
mock_user_api_key_dict.team_alias = None
mock_user_api_key_dict.end_user_id = None
mock_user_api_key_dict.user_email = None
mock_user_api_key_dict.request_route = None
mock_user_api_key_dict.team_max_budget = None
mock_user_api_key_dict.team_spend = None
mock_user_api_key_dict.model_max_budget = None
mock_user_api_key_dict.parent_otel_span = None
mock_user_api_key_dict.team_model_aliases = None
general_settings = {}
mock_proxy_config = MagicMock()
# Call the actual function that processes headers and adds data
result_data = await add_litellm_data_to_request(
data=test_data,
request=mock_request,
general_settings=general_settings,
user_api_key_dict=mock_user_api_key_dict,
version=None,
proxy_config=mock_proxy_config,
)
# Verify that stream_timeout was extracted from header and added to request data
assert "stream_timeout" in result_data
assert result_data["stream_timeout"] == 45.0
# Verify that the original test data is preserved
assert result_data["model"] == "gpt-3.5-turbo"
assert result_data["messages"] == [{"role": "user", "content": "Hello"}]
@pytest.mark.asyncio
class TestCommonRequestProcessingHelpers:

6
tests/test_litellm/test_utils.py
View file

@ -979,8 +979,8 @@ class TestProxyFunctionCalling:
# Groq models (mixed support)
("groq/gemma-7b-it", "litellm_proxy/groq/gemma-7b-it", True),
(
"groq/llama3-70b-8192",
"litellm_proxy/groq/llama3-70b-8192",
"groq/llama-3.3-70b-versatile",
"litellm_proxy/groq/llama-3.3-70b-versatile",
False,
), # This model doesn't support function calling
# Cohere models (generally don't support function calling)
@ -1051,7 +1051,7 @@ class TestProxyFunctionCalling:
("litellm_proxy/claude-prod", "anthropic/claude-3-sonnet-20240229", False),
("litellm_proxy/claude-dev", "anthropic/claude-3-haiku-20240307", False),
# Groq with custom names (cannot be resolved)
("litellm_proxy/fast-llama", "groq/llama3-8b-8192", False),
("litellm_proxy/fast-llama", "groq/llama-3.1-8b-instant", False),
("litellm_proxy/groq-gemma", "groq/gemma-7b-it", False),
# Cohere with custom names (cannot be resolved)
("litellm_proxy/cohere-command", "cohere/command-r", False),

2
tests/test_openai_endpoints.py
View file

@ -550,7 +550,7 @@ async def test_proxy_all_models():
async with aiohttp.ClientSession() as session:
# call chat/completions with a model that the key was not created for + the model is not on the config.yaml
await chat_completion(
session=session, key=LITELLM_MASTER_KEY, model="groq/llama3-8b-8192"
session=session, key=LITELLM_MASTER_KEY, model="groq/llama-3.1-8b-instant"
)
await chat_completion(

1
ui/litellm-dashboard/src/components/organizations.tsx
View file

@ -139,6 +139,7 @@ const OrganizationsTable: React.FC<OrganizationsTableProps> = ({
}
await organizationCreateCall(accessToken, values)
NotificationsManager.success("Organization created successfully")
setIsOrgModalVisible(false)
form.resetFields()
// Refresh organizations list

6
ui/litellm-dashboard/src/components/templates/key_info_view.tsx
View file

@ -43,6 +43,7 @@ interface KeyInfoViewProps {
teams: any[] | null
premiumUser: boolean
setAccessToken?: (token: string) => void
backButtonText?: string
}
export default function KeyInfoView({
@ -57,6 +58,7 @@ export default function KeyInfoView({
onDelete,
premiumUser,
setAccessToken,
backButtonText = "Back to Keys",
}: KeyInfoViewProps) {
const [isEditing, setIsEditing] = useState(false)
const [form] = Form.useForm()
@ -92,7 +94,7 @@ export default function KeyInfoView({
return (
<div className="p-4">
<Button icon={ArrowLeftIcon} variant="light" onClick={onClose} className="mb-4">
Back to Keys
{backButtonText}
</Button>
<Text>Key not found</Text>
</div>
@ -261,7 +263,7 @@ export default function KeyInfoView({
<div className="flex justify-between items-center mb-6">
<div>
<Button icon={ArrowLeftIcon} variant="light" onClick={onClose} className="mb-4">
Back to Keys
{backButtonText}
</Button>
<Title>{currentKeyData.key_alias || "API Key"}</Title>

14
ui/litellm-dashboard/src/components/view_logs/RequestResponsePanel.tsx
View file

@ -72,9 +72,9 @@ export function RequestResponsePanel({
};
return (
<div className="grid grid-cols-2 gap-4">
<div className="grid grid-cols-1 lg:grid-cols-2 gap-4 w-full max-w-full overflow-hidden box-border">
{/* Request Side */}
<div className="bg-white rounded-lg shadow">
<div className="bg-white rounded-lg shadow w-full max-w-full overflow-hidden">
<div className="flex justify-between items-center p-4 border-b">
<h3 className="text-lg font-medium">Request</h3>
<button
@ -88,13 +88,13 @@ export function RequestResponsePanel({
</svg>
</button>
</div>
<div className="p-4 overflow-auto max-h-96">
<pre className="text-xs font-mono whitespace-pre-wrap break-all">{JSON.stringify(getRawRequest(), null, 2)}</pre>
<div className="p-4 overflow-auto max-h-96 w-full max-w-full box-border">
<pre className="text-xs font-mono whitespace-pre-wrap break-all w-full max-w-full overflow-hidden break-words">{JSON.stringify(getRawRequest(), null, 2)}</pre>
</div>
</div>
{/* Response Side */}
<div className="bg-white rounded-lg shadow">
<div className="bg-white rounded-lg shadow w-full max-w-full overflow-hidden">
<div className="flex justify-between items-center p-4 border-b">
<h3 className="text-lg font-medium">
Response
@ -116,9 +116,9 @@ export function RequestResponsePanel({
</svg>
</button>
</div>
<div className="p-4 overflow-auto max-h-96 bg-gray-50">
<div className="p-4 overflow-auto max-h-96 bg-gray-50 w-full max-w-full box-border">
{hasResponse ? (
<pre className="text-xs font-mono whitespace-pre-wrap break-all">{JSON.stringify(formattedResponse(), null, 2)}</pre>
<pre className="text-xs font-mono whitespace-pre-wrap break-all w-full max-w-full overflow-hidden break-words">{JSON.stringify(formattedResponse(), null, 2)}</pre>
) : (
<div className="text-gray-500 text-sm italic text-center py-4">Response data not available</div>
)}

49
ui/litellm-dashboard/src/components/view_logs/index.tsx
View file

@ -465,7 +465,7 @@ export default function SpendLogsTable({
const displayLabel = isCustomDate ? getTimeRangeDisplay(isCustomDate, startTime, endTime) : selectedOption?.label
return (
<div className="w-full p-6">
<div className="w-full max-w-screen p-6 overflow-x-hidden box-border">
<TabGroup defaultIndex={0} onIndexChange={(index) => setActiveTab(index === 0 ? "request logs" : "audit logs")}>
<TabList>
<Tab>Request Logs</Tab>
@ -500,6 +500,7 @@ export default function SpendLogsTable({
teams={allTeams}
onClose={() => setSelectedKeyIdInfoView(null)}
premiumUser={premiumUser}
backButtonText="Back to Logs"
/>
) : selectedSessionId ? (
<div className="bg-white rounded-lg shadow">
@ -518,11 +519,11 @@ export default function SpendLogsTable({
onApplyFilters={handleFilterChange}
onResetFilters={handleFilterReset}
/>
<div className="bg-white rounded-lg shadow">
<div className="border-b px-6 py-4">
<div className="flex flex-col md:flex-row items-start md:items-center justify-between space-y-4 md:space-y-0">
<div className="flex flex-wrap items-center gap-3">
<div className="relative w-64">
<div className="bg-white rounded-lg shadow w-full max-w-full box-border">
<div className="border-b px-6 py-4 w-full max-w-full box-border">
<div className="flex flex-col md:flex-row items-start md:items-center justify-between space-y-4 md:space-y-0 w-full max-w-full box-border">
<div className="flex flex-wrap items-center gap-3 w-full max-w-full box-border">
<div className="relative w-64 min-w-0 flex-shrink-0">
<input
type="text"
placeholder="Search by Request ID"
@ -545,8 +546,8 @@ export default function SpendLogsTable({
</svg>
</div>
<div className="flex items-center gap-2">
<div className="relative" ref={quickSelectRef}>
<div className="flex items-center gap-2 min-w-0 flex-shrink">
<div className="relative z-[9999]" ref={quickSelectRef}>
<button
onClick={() => setQuickSelectOpen(!quickSelectOpen)}
className="px-3 py-2 text-sm border rounded-md hover:bg-gray-50 flex items-center gap-2"
@ -563,7 +564,7 @@ export default function SpendLogsTable({
</button>
{quickSelectOpen && (
<div className="absolute right-0 mt-2 w-64 bg-white rounded-lg shadow-lg border p-2 z-50">
<div className="absolute right-0 mt-2 w-64 bg-white rounded-lg shadow-lg border p-2 z-[9999]">
<div className="space-y-1">
{quickSelectOptions.map((option) => (
<button
@ -654,7 +655,7 @@ export default function SpendLogsTable({
</div>
<div className="flex items-center space-x-4">
<span className="text-sm text-gray-700">
<span className="text-sm text-gray-700 whitespace-nowrap">
Showing {logs.isLoading ? "..." : filteredLogs ? (currentPage - 1) * pageSize + 1 : 0} -{" "}
{logs.isLoading
? "..."
@ -664,7 +665,7 @@ export default function SpendLogsTable({
of {logs.isLoading ? "..." : filteredLogs ? filteredLogs.total : 0} results
</span>
<div className="flex items-center space-x-2">
<span className="text-sm text-gray-700">
<span className="text-sm text-gray-700 min-w-[90px]">
Page {logs.isLoading ? "..." : currentPage} of{" "}
{logs.isLoading ? "..." : filteredLogs ? filteredLogs.total_pages : 1}
</span>
@ -801,13 +802,13 @@ export function RequestViewer({ row }: { row: Row<LogEntry> }) {
const totalMaskedEntities = getTotalMaskedEntities()
return (
<div className="p-6 bg-gray-50 space-y-6">
<div className="p-6 bg-gray-50 space-y-6 w-full max-w-full overflow-hidden box-border">
{/* Combined Info Card */}
<div className="bg-white rounded-lg shadow">
<div className="bg-white rounded-lg shadow w-full max-w-full overflow-hidden">
<div className="p-4 border-b">
<h3 className="text-lg font-medium">Request Details</h3>
</div>
<div className="grid grid-cols-2 gap-4 p-4">
<div className="grid grid-cols-1 md:grid-cols-2 gap-4 p-4 w-full max-w-full overflow-hidden">
<div className="space-y-2">
<div className="flex">
<span className="font-medium w-1/3">Request ID:</span>
@ -916,15 +917,17 @@ export function RequestViewer({ row }: { row: Row<LogEntry> }) {
<ConfigInfoMessage show={missingData} />
{/* Request/Response Panel */}
<RequestResponsePanel
row={row}
hasMessages={hasMessages}
hasResponse={hasResponse}
hasError={hasError}
errorInfo={errorInfo}
getRawRequest={getRawRequest}
formattedResponse={formattedResponse}
/>
<div className="w-full max-w-full overflow-hidden">
<RequestResponsePanel
row={row}
hasMessages={hasMessages}
hasResponse={hasResponse}
hasError={hasError}
errorInfo={errorInfo}
getRawRequest={getRawRequest}
formattedResponse={formattedResponse}
/>
</div>
{/* Guardrail Data - Show only if present */}
{hasGuardrailData && <GuardrailViewer data={row.original.metadata!.guardrail_information} />}

10
ui/litellm-dashboard/src/components/view_logs/table.tsx
View file

@ -52,8 +52,8 @@ export function DataTable<TData, TValue>({
});
return (
<div className="rounded-lg custom-border">
<Table className="[&_td]:py-0.5 [&_th]:py-1">
<div className="rounded-lg custom-border overflow-x-auto w-full max-w-full box-border">
<Table className="[&_td]:py-0.5 [&_th]:py-1 table-fixed w-full box-border" style={{minWidth: '800px'}}>
<TableHead>
{table.getHeaderGroups().map((headerGroup) => (
<TableRow key={headerGroup.id}>
@ -100,8 +100,10 @@ export function DataTable<TData, TValue>({
{row.getIsExpanded() && (
<TableRow>
<TableCell colSpan={row.getVisibleCells().length}>
{renderSubComponent({ row })}
<TableCell colSpan={row.getVisibleCells().length} className="p-0">
<div className="w-full max-w-full overflow-hidden box-border">
{renderSubComponent({ row })}
</div>
</TableCell>
</TableRow>
)}