mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-10 22:41:41 +00:00
Merge branch 'BerriAI:main' into main
This commit is contained in:
John Lathouwers
GitHub
commit
d02e560d85
120 changed files with 2631 additions and 587 deletions
|
|
@ -1477,6 +1477,7 @@ jobs:
|
|||
docker run -d \
|
||||
-p 4000:4000 \
|
||||
-e DATABASE_URL=$PROXY_DATABASE_URL \
|
||||
-e DEFAULT_NUM_WORKERS_LITELLM_PROXY=1 \
|
||||
-e DISABLE_SCHEMA_UPDATE="True" \
|
||||
-v $(pwd)/litellm/proxy/example_config_yaml/bad_schema.prisma:/app/schema.prisma \
|
||||
-v $(pwd)/litellm/proxy/example_config_yaml/bad_schema.prisma:/app/litellm/proxy/schema.prisma \
|
||||
|
|
@ -2962,6 +2963,7 @@ jobs:
|
|||
command: |
|
||||
docker run --name my-app \
|
||||
-p 4000:4000 \
|
||||
-e DEFAULT_NUM_WORKERS_LITELLM_PROXY=1 \
|
||||
-e DATABASE_URL="postgresql://wrong:wrong@wrong:5432/wrong" \
|
||||
myapp:latest \
|
||||
--port 4000 > docker_output.log 2>&1 || true
|
||||
|
|
|
|||
|
|
@ -18,7 +18,7 @@ type: application
|
|||
# This is the chart version. This version number should be incremented each time you make changes
|
||||
# to the chart and its templates, including the app version.
|
||||
# Versions are expected to follow Semantic Versioning (https://semver.org/)
|
||||
version: 0.4.5
|
||||
version: 0.4.6
|
||||
|
||||
# This is the version number of the application being deployed. This version number should be
|
||||
# incremented each time you make changes to the application. Versions are not expected to
|
||||
|
|
|
|||
|
|
@ -41,6 +41,11 @@ If `db.useStackgresOperator` is used (not yet implemented):
|
|||
| `proxyConfigMap.key` | Key in the ConfigMap that contains the proxy config file. | `"config.yaml"` |
|
||||
| `proxy_config.*` | See [values.yaml](./values.yaml) for default settings. Rendered into the ConfigMap’s `config.yaml` only when `proxyConfigMap.create=true`. See [example_config_yaml](../../../litellm/proxy/example_config_yaml/) for configuration examples. | `N/A` |
|
||||
| `extraContainers[]` | An array of additional containers to be deployed as sidecars alongside the LiteLLM Proxy.
|
||||
| `pdb.enabled` | Enable a PodDisruptionBudget for the LiteLLM proxy Deployment | `false` |
|
||||
| `pdb.minAvailable` | Minimum number/percentage of pods that must be available during **voluntary** disruptions (choose **one** of minAvailable/maxUnavailable) | `null` |
|
||||
| `pdb.maxUnavailable` | Maximum number/percentage of pods that can be unavailable during **voluntary** disruptions (choose **one** of minAvailable/maxUnavailable) | `null` |
|
||||
| `pdb.annotations` | Extra metadata annotations to add to the PDB | `{}` |
|
||||
| `pdb.labels` | Extra metadata labels to add to the PDB | `{}` |
|
||||
|
||||
#### Example `proxy_config` ConfigMap from values (default):
|
||||
|
||||
|
|
|
|||
|
|
@ -20,3 +20,4 @@
|
|||
echo "Visit http://127.0.0.1:8080 to use your application"
|
||||
kubectl --namespace {{ .Release.Namespace }} port-forward $POD_NAME 8080:$CONTAINER_PORT
|
||||
{{- end }}
|
||||
PDB: {{ if .Values.pdb.enabled }}enabled{{ else }}disabled{{ end }}. Configure via .Values.pdb.*
|
||||
|
|
@ -0,0 +1,33 @@
|
|||
{{- /*
|
||||
PodDisruptionBudget for LiteLLM proxy
|
||||
Controlled via .Values.pdb.enabled and .Values.pdb.{minAvailable|maxUnavailable}
|
||||
Only one of minAvailable / maxUnavailable should be set. If both are set, minAvailable wins.
|
||||
*/ -}}
|
||||
{{- if .Values.pdb.enabled }}
|
||||
apiVersion: policy/v1
|
||||
kind: PodDisruptionBudget
|
||||
metadata:
|
||||
name: {{ include "litellm.fullname" . }}
|
||||
labels:
|
||||
{{- include "litellm.labels" . | nindent 4 }}
|
||||
{{- with .Values.pdb.labels }}
|
||||
{{- toYaml . | nindent 4 }}
|
||||
{{- end }}
|
||||
{{- with .Values.pdb.annotations }}
|
||||
annotations:
|
||||
{{- toYaml . | nindent 4 }}
|
||||
{{- end }}
|
||||
spec:
|
||||
selector:
|
||||
matchLabels:
|
||||
{{- /* Match the Deployment selector to target the same pod set */ -}}
|
||||
{{- include "litellm.selectorLabels" . | nindent 6 }}
|
||||
{{- if .Values.pdb.minAvailable }}
|
||||
minAvailable: {{ .Values.pdb.minAvailable }}
|
||||
{{- else if .Values.pdb.maxUnavailable }}
|
||||
maxUnavailable: {{ .Values.pdb.maxUnavailable }}
|
||||
{{- else }}
|
||||
# Safe default if enabled but not configured
|
||||
maxUnavailable: 1
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
45
deploy/charts/litellm-helm/tests/pdb_tests.yaml
Normal file
45
deploy/charts/litellm-helm/tests/pdb_tests.yaml
Normal file
|
|
@ -0,0 +1,45 @@
|
|||
suite: "pdb enabled"
|
||||
templates:
|
||||
- poddisruptionbudget.yaml
|
||||
tests:
|
||||
- it: "renders a PDB with maxUnavailable=1"
|
||||
set:
|
||||
pdb.enabled: true
|
||||
pdb.maxUnavailable: 1
|
||||
asserts:
|
||||
- hasDocuments: { count: 1 }
|
||||
- isKind: { of: PodDisruptionBudget }
|
||||
- equal: { path: apiVersion, value: policy/v1 }
|
||||
- equal: { path: spec.maxUnavailable, value: 1 }
|
||||
- equal:
|
||||
path: spec.selector.matchLabels
|
||||
value:
|
||||
app.kubernetes.io/name: litellm
|
||||
app.kubernetes.io/instance: RELEASE-NAME
|
||||
|
||||
---
|
||||
suite: "pdb disabled"
|
||||
templates:
|
||||
- poddisruptionbudget.yaml
|
||||
tests:
|
||||
- it: "does not render when disabled"
|
||||
set:
|
||||
pdb.enabled: false
|
||||
asserts:
|
||||
- hasDocuments: { count: 0 }
|
||||
|
||||
---
|
||||
suite: "pdb minAvailable precedence"
|
||||
templates:
|
||||
- poddisruptionbudget.yaml
|
||||
tests:
|
||||
- it: "uses minAvailable when both are set"
|
||||
set:
|
||||
pdb.enabled: true
|
||||
pdb.minAvailable: "50%"
|
||||
pdb.maxUnavailable: 1
|
||||
asserts:
|
||||
- isKind: { of: PodDisruptionBudget }
|
||||
- equal: { path: apiVersion, value: policy/v1 }
|
||||
- equal: { path: spec.minAvailable, value: "50%" }
|
||||
- isNull: { path: spec.maxUnavailable }
|
||||
|
|
@ -240,4 +240,11 @@ extraEnvVars: {
|
|||
# value: EXTRA_ENV_VAR_VALUE
|
||||
}
|
||||
|
||||
|
||||
# Pod Disruption Budget
|
||||
pdb:
|
||||
enabled: false
|
||||
# Set exactly one of the following. If both are set, minAvailable takes precedence.
|
||||
minAvailable: null # e.g. "50%" or 1
|
||||
maxUnavailable: null # e.g. 1 or "20%"
|
||||
annotations: {}
|
||||
labels: {}
|
||||
|
|
|
|||
|
|
@ -12,6 +12,7 @@ All exceptions can be imported from `litellm` - e.g. `from litellm import BadReq
|
|||
| 400 | UnsupportedParamsError | litellm.BadRequestError | Raised when unsupported params are passed |
|
||||
| 400 | ContextWindowExceededError| litellm.BadRequestError | Special error type for context window exceeded error messages - enables context window fallbacks |
|
||||
| 400 | ContentPolicyViolationError| litellm.BadRequestError | Special error type for content policy violation error messages - enables content policy fallbacks |
|
||||
| 400 | ImageFetchError | litellm.BadRequestError | Raised when there are errors fetching or processing images |
|
||||
| 400 | InvalidRequestError | openai.BadRequestError | Deprecated error, use BadRequestError instead |
|
||||
| 401 | AuthenticationError | openai.AuthenticationError |
|
||||
| 403 | PermissionDeniedError | openai.PermissionDeniedError |
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@
|
|||
|
||||
Anyone using the following models with /chat/completions:
|
||||
- `gemini/gemini-2.0-flash-exp-image-generation`
|
||||
- `vertex_ai/gemini-2.5-flash-image-preview`
|
||||
- `vertex_ai/gemini-2.0-flash-exp-image-generation`
|
||||
|
||||
## Key Change
|
||||
|
||||
|
|
@ -40,6 +40,10 @@ response = completion(
|
|||
image_url = response.choices[0].message.image["url"] # "data:image/png;base64,..."
|
||||
```
|
||||
|
||||
### Why the change?
|
||||
|
||||
Because the newer `gemini-2.5-flash-image-preview` model sends both text and image responses in the same response. This interface allows a developer to explicitly access the image or text components of the response. Before a developer would have needed to search through the message content to find the image generated by the model.
|
||||
|
||||
## Usage
|
||||
|
||||
### Using the Python SDK
|
||||
|
|
|
|||
|
|
@ -431,6 +431,7 @@ router_settings:
|
|||
| DEFAULT_MOCK_RESPONSE_COMPLETION_TOKEN_COUNT | Default token count for mock response completions. Default is 20
|
||||
| DEFAULT_MOCK_RESPONSE_PROMPT_TOKEN_COUNT | Default token count for mock response prompts. Default is 10
|
||||
| DEFAULT_MODEL_CREATED_AT_TIME | Default creation timestamp for models. Default is 1677610602
|
||||
| DEFAULT_NUM_WORKERS_LITELLM_PROXY | Default number of workers for LiteLLM proxy. Default is 4. **We strongly recommend setting NUM Workers to Number of vCPUs available**
|
||||
| DEFAULT_PROMPT_INJECTION_SIMILARITY_THRESHOLD | Default threshold for prompt injection similarity. Default is 0.7
|
||||
| DEFAULT_POLLING_INTERVAL | Default polling interval for schedulers in seconds. Default is 0.03
|
||||
| DEFAULT_REASONING_EFFORT_DISABLE_THINKING_BUDGET | Default reasoning effort disable thinking budget. Default is 0
|
||||
|
|
|
|||
|
|
@ -12,10 +12,7 @@ To start using Litellm, run the following commands in a shell:
|
|||
|
||||
```bash
|
||||
# Get the code
|
||||
git clone https://github.com/BerriAI/litellm
|
||||
|
||||
# Go to folder
|
||||
cd litellm
|
||||
curl -O https://raw.githubusercontent.com/BerriAI/litellm/main/docker-compose.yml
|
||||
|
||||
# Add the master key - you can change this after setup
|
||||
echo 'LITELLM_MASTER_KEY="sk-1234"' > .env
|
||||
|
|
|
|||
|
|
@ -35,6 +35,30 @@ $ pip install 'litellm[proxy]'
|
|||
|
||||
</TabItem>
|
||||
|
||||
<TabItem value="docker-compose" label="Docker Compose (Proxy + DB)">
|
||||
|
||||
Use this docker compose to spin up the proxy with a postgres database running locally.
|
||||
|
||||
```bash
|
||||
# Get the docker compose file
|
||||
curl -O https://raw.githubusercontent.com/BerriAI/litellm/main/docker-compose.yml
|
||||
|
||||
# Add the master key - you can change this after setup
|
||||
echo 'LITELLM_MASTER_KEY="sk-1234"' > .env
|
||||
|
||||
# Add the litellm salt key - you cannot change this after adding a model
|
||||
# It is used to encrypt / decrypt your LLM API Key credentials
|
||||
# We recommend - https://1password.com/password-generator/
|
||||
# password generator to get a random hash for litellm salt key
|
||||
echo 'LITELLM_SALT_KEY="sk-1234"' >> .env
|
||||
|
||||
source .env
|
||||
|
||||
# Start
|
||||
docker-compose up
|
||||
```
|
||||
|
||||
</TabItem>
|
||||
</Tabs>
|
||||
|
||||
## 1. Add a model
|
||||
|
|
@ -43,6 +67,8 @@ Control LiteLLM Proxy with a config.yaml file.
|
|||
|
||||
Setup your config.yaml with your azure model.
|
||||
|
||||
Note: When using the proxy with a database, you can also **just add models via UI** (UI is available on `/ui` route).
|
||||
|
||||
```yaml
|
||||
model_list:
|
||||
- model_name: gpt-4o
|
||||
|
|
|
|||
|
|
@ -6,6 +6,8 @@ Special headers that are supported by LiteLLM.
|
|||
|
||||
`x-litellm-timeout` Optional[float]: The timeout for the request in seconds.
|
||||
|
||||
`x-litellm-stream-timeout` Optional[float]: The timeout for getting the first chunk of the response in seconds (only applies for streaming requests). [Demo Video](https://www.loom.com/share/8da67e4845ce431a98c901d4e45db0e5)
|
||||
|
||||
`x-litellm-enable-message-redaction`: Optional[bool]: Don't log the message content to logging integrations. Just track spend. [Learn More](./logging#redact-messages-response-content)
|
||||
|
||||
`x-litellm-tags`: Optional[str]: A comma separated list (e.g. `tag1,tag2,tag3`) of tags to use for [tag-based routing](./tag_routing) **OR** [spend-tracking](./enterprise.md#tracking-spend-for-custom-tags).
|
||||
|
|
|
|||
269
docs/my-website/release_notes/v1.76.1-stable/index.md
Normal file
269
docs/my-website/release_notes/v1.76.1-stable/index.md
Normal file
|
|
@ -0,0 +1,269 @@
|
|||
---
|
||||
title: "v1.76.1-stable - Gemini 2.5 Flash Image"
|
||||
slug: "v1-76-1"
|
||||
date: 2025-08-30T10:00:00
|
||||
authors:
|
||||
- name: Krrish Dholakia
|
||||
title: CEO, LiteLLM
|
||||
url: https://www.linkedin.com/in/krish-d/
|
||||
image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg
|
||||
- name: Ishaan Jaffer
|
||||
title: CTO, LiteLLM
|
||||
url: https://www.linkedin.com/in/reffajnaahsi/
|
||||
image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg
|
||||
|
||||
hide_table_of_contents: false
|
||||
---
|
||||
|
||||
import Image from '@theme/IdealImage';
|
||||
import Tabs from '@theme/Tabs';
|
||||
import TabItem from '@theme/TabItem';
|
||||
|
||||
## Deploy this version
|
||||
|
||||
<Tabs>
|
||||
<TabItem value="docker" label="Docker">
|
||||
|
||||
``` showLineNumbers title="docker run litellm"
|
||||
docker run \
|
||||
-e STORE_MODEL_IN_DB=True \
|
||||
-p 4000:4000 \
|
||||
ghcr.io/berriai/litellm:v1.76.1
|
||||
```
|
||||
</TabItem>
|
||||
|
||||
<TabItem value="pip" label="Pip">
|
||||
|
||||
``` showLineNumbers title="pip install litellm"
|
||||
pip install litellm==1.76.1
|
||||
```
|
||||
|
||||
</TabItem>
|
||||
</Tabs>
|
||||
|
||||
---
|
||||
|
||||
## Key Highlights
|
||||
|
||||
- **Major Performance Improvements** - 6.5x faster LiteLLM Python SDK completion with fastuuid integration.
|
||||
- **New Model Support** - Gemini 2.5 Flash Image Preview, Grok Code Fast, and GPT Realtime models
|
||||
- **Enhanced Provider Support** - DeepSeek-v3.1 pricing on Fireworks AI, Vercel AI Gateway, and improved Anthropic/GitHub Copilot integration
|
||||
- **MCP Improvements** - Better connection testing and SSE MCP tools bug fixes
|
||||
|
||||
## Major Changes
|
||||
- Added support for using Gemini 2.5 Flash Image Preview with /chat/completions. **🚨 Warning** If you were using `gemini-2.0-flash-exp-image-generation` please follow this migration guide.
|
||||
[Gemini Image Generation Migration Guide](../../docs/extras/gemini_img_migration)
|
||||
---
|
||||
|
||||
## Performance Improvements
|
||||
|
||||
This release includes significant performance optimizations:
|
||||
|
||||
- **6.5x faster LiteLLM Python SDK Completion** - Major performance boost for completion operations - [PR #13990](https://github.com/BerriAI/litellm/pull/13990)
|
||||
- **fastuuid Integration** - 2.1x faster UUID generation with +80 RPS improvement for /chat/completions and other LLM endpoints - [PR #13992](https://github.com/BerriAI/litellm/pull/13992), [PR #14016](https://github.com/BerriAI/litellm/pull/14016)
|
||||
- **Optimized Request Logging** - Don't print request params by default for +50 RPS improvement - [PR #14015](https://github.com/BerriAI/litellm/pull/14015)
|
||||
- **Cache Performance** - 21% speedup in InMemoryCache.evict_cache and 45% speedup in `_is_debugging_on` function - [PR #14012](https://github.com/BerriAI/litellm/pull/14012), [PR #13988](https://github.com/BerriAI/litellm/pull/13988)
|
||||
|
||||
---
|
||||
|
||||
## New Models / Updated Models
|
||||
|
||||
#### New Model Support
|
||||
|
||||
| Provider | Model | Context Window | Input ($/1M tokens) | Output ($/1M tokens) | Features |
|
||||
| ----------- | -------------------------------------- | -------------- | ------------------- | -------------------- | -------- |
|
||||
| Google | `gemini-2.5-flash-image-preview` | 1M | $0.30 | $2.50 | Chat completions + image generation ($0.039/image) |
|
||||
| X.AI | `xai/grok-code-fast` | 256K | $0.20 | $1.50 | Code generation |
|
||||
| OpenAI | `gpt-realtime` | 32K | $4.00 | $16.00 | Real-time conversation + audio |
|
||||
| Vercel AI Gateway | `vercel_ai_gateway/openai/o3` | 200K | $2.00 | $8.00 | Advanced reasoning |
|
||||
| Vercel AI Gateway | `vercel_ai_gateway/openai/o3-mini` | 200K | $1.10 | $4.40 | Efficient reasoning |
|
||||
| Vercel AI Gateway | `vercel_ai_gateway/openai/o4-mini` | 200K | $1.10 | $4.40 | Latest mini model |
|
||||
| DeepInfra | `deepinfra/zai-org/GLM-4.5` | 131K | $0.55 | $2.00 | Chat completions |
|
||||
| Perplexity | `perplexity/codellama-34b-instruct` | 16K | $0.35 | $1.40 | Code generation |
|
||||
| Fireworks AI | `fireworks_ai/accounts/fireworks/models/deepseek-v3p1` | 128K | $0.56 | $1.68 | Chat completions |
|
||||
|
||||
**Additional Models Added:** Various other Vercel AI Gateway models were added too. See [models.litellm.ai](https://models.litellm.ai) for the full list.
|
||||
|
||||
#### Features
|
||||
|
||||
- **[Google Gemini](../../docs/providers/gemini)**
|
||||
- Added support for `gemini-2.5-flash-image-preview` with image return capability - [PR #13979](https://github.com/BerriAI/litellm/pull/13979), [PR #13983](https://github.com/BerriAI/litellm/pull/13983)
|
||||
- Support for requests with only system prompt - [PR #14010](https://github.com/BerriAI/litellm/pull/14010)
|
||||
- Fixed invalid model name error for Gemini Imagen models - [PR #13991](https://github.com/BerriAI/litellm/pull/13991)
|
||||
- **[X.AI](../../docs/providers/xai)**
|
||||
- Added `xai/grok-code-fast` model family support - [PR #14054](https://github.com/BerriAI/litellm/pull/14054)
|
||||
- Fixed frequency_penalty parameter for grok-4 models - [PR #14078](https://github.com/BerriAI/litellm/pull/14078)
|
||||
- **[OpenAI](../../docs/providers/openai)**
|
||||
- Added support for gpt-realtime models - [PR #14082](https://github.com/BerriAI/litellm/pull/14082)
|
||||
- Support for reasoning and reasoning_effort parameters by default - [PR #12865](https://github.com/BerriAI/litellm/pull/12865)
|
||||
- **[Fireworks AI](../../docs/providers/fireworks_ai)**
|
||||
- Added DeepSeek-v3.1 pricing - [PR #13958](https://github.com/BerriAI/litellm/pull/13958)
|
||||
- **[DeepInfra](../../docs/providers/deepinfra)**
|
||||
- Fixed reasoning_effort setting for DeepSeek-V3.1 - [PR #14053](https://github.com/BerriAI/litellm/pull/14053)
|
||||
- **[GitHub Copilot](../../docs/providers/github_copilot)**
|
||||
- Added support for thinking and reasoning_effort parameters - [PR #13691](https://github.com/BerriAI/litellm/pull/13691)
|
||||
- Added image headers support - [PR #13955](https://github.com/BerriAI/litellm/pull/13955)
|
||||
- **[Anthropic](../../docs/providers/anthropic)**
|
||||
- Support for custom Anthropic-compatible API endpoints - [PR #13945](https://github.com/BerriAI/litellm/pull/13945)
|
||||
- Fixed /messages fallback from Anthropic API to Bedrock API - [PR #13946](https://github.com/BerriAI/litellm/pull/13946)
|
||||
- **[Nebius](../../docs/providers/nebius)**
|
||||
- Expanded provider models and normalized model IDs - [PR #13965](https://github.com/BerriAI/litellm/pull/13965)
|
||||
- **[Vertex AI](../../docs/providers/vertex)**
|
||||
- Fixed Vertex Mistral streaming issues - [PR #13952](https://github.com/BerriAI/litellm/pull/13952)
|
||||
- Fixed anyOf corner cases for Gemini tool calls - [PR #12797](https://github.com/BerriAI/litellm/pull/12797)
|
||||
- **[Bedrock](../../docs/providers/bedrock)**
|
||||
- Fixed structure output issues - [PR #14005](https://github.com/BerriAI/litellm/pull/14005)
|
||||
- **[OpenRouter](../../docs/providers/openrouter)**
|
||||
- Added GPT-5 family models pricing - [PR #13536](https://github.com/BerriAI/litellm/pull/13536)
|
||||
|
||||
#### New Provider Support
|
||||
|
||||
- **[Vercel AI Gateway](../../docs/providers/vercel_ai_gateway)**
|
||||
- New provider support added - [PR #13144](https://github.com/BerriAI/litellm/pull/13144)
|
||||
- **[DataRobot](../../docs/providers/datarobot)**
|
||||
- Added provider documentation - [PR #14038](https://github.com/BerriAI/litellm/pull/14038), [PR #14074](https://github.com/BerriAI/litellm/pull/14074)
|
||||
|
||||
---
|
||||
|
||||
## LLM API Endpoints
|
||||
|
||||
#### Features
|
||||
|
||||
- **[Images API](../../docs/image_generation)**
|
||||
- Support for multiple images in OpenAI images/edits endpoint - [PR #13916](https://github.com/BerriAI/litellm/pull/13916)
|
||||
- Allow using dynamic `api_key` for image generation requests - [PR #14007](https://github.com/BerriAI/litellm/pull/14007)
|
||||
- **[Responses API](../../docs/response_api)**
|
||||
- Fixed `/responses` endpoint ignoring extra_headers in GitHub Copilot - [PR #13775](https://github.com/BerriAI/litellm/pull/13775)
|
||||
- Added support for new web_search tool - [PR #14083](https://github.com/BerriAI/litellm/pull/14083)
|
||||
- **[Azure Passthrough](../../docs/providers/azure/azure)**
|
||||
- Fixed Azure Passthrough request with streaming - [PR #13831](https://github.com/BerriAI/litellm/pull/13831)
|
||||
|
||||
#### Bugs
|
||||
|
||||
- **General**
|
||||
- Fixed handling of None metadata in batch requests - [PR #13996](https://github.com/BerriAI/litellm/pull/13996)
|
||||
- Fixed token_counter with special token input - [PR #13374](https://github.com/BerriAI/litellm/pull/13374)
|
||||
- Removed incorrect web search support for azure/gpt-4.1 family - [PR #13566](https://github.com/BerriAI/litellm/pull/13566)
|
||||
|
||||
---
|
||||
|
||||
## [MCP Gateway](../../docs/mcp)
|
||||
|
||||
#### Features
|
||||
|
||||
- **SSE MCP Tools**
|
||||
- Bug fix for adding SSE MCP tools - improved connection testing when adding MCPs - [PR #14048](https://github.com/BerriAI/litellm/pull/14048)
|
||||
|
||||
[Read More](../../docs/mcp)
|
||||
|
||||
---
|
||||
|
||||
## Management Endpoints / UI
|
||||
|
||||
#### Features
|
||||
|
||||
- **Team Management**
|
||||
- Allow setting Team Member RPM/TPM limits when creating a team - [PR #13943](https://github.com/BerriAI/litellm/pull/13943)
|
||||
- **UI Improvements**
|
||||
- Fixed Next.js Security Vulnerabilities in UI Dashboard - [PR #14084](https://github.com/BerriAI/litellm/pull/14084)
|
||||
- Fixed collapsible navbar design - [PR #14075](https://github.com/BerriAI/litellm/pull/14075)
|
||||
|
||||
#### Bugs
|
||||
|
||||
- **Authentication**
|
||||
- Fixed Virtual keys with llm_api type causing Internal Server Error for /anthropic/* and other LLM passthrough routes - [PR #14046](https://github.com/BerriAI/litellm/pull/14046)
|
||||
|
||||
---
|
||||
|
||||
## Logging / Guardrail Integrations
|
||||
|
||||
#### Features
|
||||
|
||||
- **[Langfuse OTEL](../../docs/proxy/logging#langfuse)**
|
||||
- Allow using LANGFUSE_OTEL_HOST for configuring host - [PR #14013](https://github.com/BerriAI/litellm/pull/14013)
|
||||
- **[Braintrust](../../docs/proxy/logging#braintrust)**
|
||||
- Added span name metadata feature - [PR #13573](https://github.com/BerriAI/litellm/pull/13573)
|
||||
- Fixed tests to reference moved attributes in `braintrust_logging` module - [PR #13978](https://github.com/BerriAI/litellm/pull/13978)
|
||||
- **[OpenMeter](../../docs/proxy/logging#openmeter)**
|
||||
- Set user from token user_id for OpenMeter integration - [PR #13152](https://github.com/BerriAI/litellm/pull/13152)
|
||||
|
||||
#### New Guardrail Support
|
||||
|
||||
- **[Noma Security](../../docs/proxy/guardrails)**
|
||||
- Added Noma Security guardrail support - [PR #13572](https://github.com/BerriAI/litellm/pull/13572)
|
||||
- **[Pangea](../../docs/proxy/guardrails)**
|
||||
- Updated Pangea Guardrail to support new AIDR endpoint - [PR #13160](https://github.com/BerriAI/litellm/pull/13160)
|
||||
|
||||
---
|
||||
|
||||
## Performance / Loadbalancing / Reliability improvements
|
||||
|
||||
#### Features
|
||||
|
||||
- **Caching**
|
||||
- Verify if cache entry has expired prior to serving it to client - [PR #13933](https://github.com/BerriAI/litellm/pull/13933)
|
||||
- Fixed error saving latency as timedelta on Redis - [PR #14040](https://github.com/BerriAI/litellm/pull/14040)
|
||||
- **Router**
|
||||
- Refactored router to choose weights by 'weight', 'rpm', 'tpm' in one loop for simple_shuffle - [PR #13562](https://github.com/BerriAI/litellm/pull/13562)
|
||||
- **Logging**
|
||||
- Fixed LoggingWorker graceful shutdown to prevent CancelledError warnings - [PR #14050](https://github.com/BerriAI/litellm/pull/14050)
|
||||
- Enhanced logging for containers to log on files both with usual format and json format - [PR #13394](https://github.com/BerriAI/litellm/pull/13394)
|
||||
|
||||
#### Bugs
|
||||
|
||||
- **Dependencies**
|
||||
- Bumped `orjson` version to "3.11.2" - [PR #13969](https://github.com/BerriAI/litellm/pull/13969)
|
||||
|
||||
---
|
||||
|
||||
## General Proxy Improvements
|
||||
|
||||
#### Features
|
||||
|
||||
- **AWS**
|
||||
- Add support for AWS assume_role with a session token - [PR #13919](https://github.com/BerriAI/litellm/pull/13919)
|
||||
- **OCI Provider**
|
||||
- Added oci_key_file as an optional_parameter - [PR #14036](https://github.com/BerriAI/litellm/pull/14036)
|
||||
- **Configuration**
|
||||
- Allow configuration to set threshold before request entry in spend log gets truncated - [PR #14042](https://github.com/BerriAI/litellm/pull/14042)
|
||||
- Enhanced proxy_config configuration: add support for existing configmap in Helm charts - [PR #14041](https://github.com/BerriAI/litellm/pull/14041)
|
||||
- **Docker**
|
||||
- Added back supervisor to non-root image - [PR #13922](https://github.com/BerriAI/litellm/pull/13922)
|
||||
|
||||
|
||||
---
|
||||
|
||||
## New Contributors
|
||||
* @ArthurRenault made their first contribution in [PR #13922](https://github.com/BerriAI/litellm/pull/13922)
|
||||
* @stevenmanton made their first contribution in [PR #13919](https://github.com/BerriAI/litellm/pull/13919)
|
||||
* @uc4w6c made their first contribution in [PR #13914](https://github.com/BerriAI/litellm/pull/13914)
|
||||
* @nielsbosma made their first contribution in [PR #13573](https://github.com/BerriAI/litellm/pull/13573)
|
||||
* @Yuki-Imajuku made their first contribution in [PR #13567](https://github.com/BerriAI/litellm/pull/13567)
|
||||
* @codeflash-ai[bot] made their first contribution in [PR #13988](https://github.com/BerriAI/litellm/pull/13988)
|
||||
* @ColeFrench made their first contribution in [PR #13978](https://github.com/BerriAI/litellm/pull/13978)
|
||||
* @dttran-glo made their first contribution in [PR #13969](https://github.com/BerriAI/litellm/pull/13969)
|
||||
* @manascb1344 made their first contribution in [PR #13965](https://github.com/BerriAI/litellm/pull/13965)
|
||||
* @DorZion made their first contribution in [PR #13572](https://github.com/BerriAI/litellm/pull/13572)
|
||||
* @edwardsamuel made their first contribution in [PR #13536](https://github.com/BerriAI/litellm/pull/13536)
|
||||
* @blahgeek made their first contribution in [PR #13374](https://github.com/BerriAI/litellm/pull/13374)
|
||||
* @Deviad made their first contribution in [PR #13394](https://github.com/BerriAI/litellm/pull/13394)
|
||||
* @XSAM made their first contribution in [PR #13775](https://github.com/BerriAI/litellm/pull/13775)
|
||||
* @KRRT7 made their first contribution in [PR #14012](https://github.com/BerriAI/litellm/pull/14012)
|
||||
* @ikaadil made their first contribution in [PR #13991](https://github.com/BerriAI/litellm/pull/13991)
|
||||
* @timelfrink made their first contribution in [PR #13691](https://github.com/BerriAI/litellm/pull/13691)
|
||||
* @qidu made their first contribution in [PR #13562](https://github.com/BerriAI/litellm/pull/13562)
|
||||
* @nagyv made their first contribution in [PR #13243](https://github.com/BerriAI/litellm/pull/13243)
|
||||
* @xywei made their first contribution in [PR #12885](https://github.com/BerriAI/litellm/pull/12885)
|
||||
* @ericgtkb made their first contribution in [PR #12797](https://github.com/BerriAI/litellm/pull/12797)
|
||||
* @NoWall57 made their first contribution in [PR #13945](https://github.com/BerriAI/litellm/pull/13945)
|
||||
* @lmwang9527 made their first contribution in [PR #14050](https://github.com/BerriAI/litellm/pull/14050)
|
||||
* @WilsonSunBritten made their first contribution in [PR #14042](https://github.com/BerriAI/litellm/pull/14042)
|
||||
* @Const-antine made their first contribution in [PR #14041](https://github.com/BerriAI/litellm/pull/14041)
|
||||
* @dmvieira made their first contribution in [PR #14040](https://github.com/BerriAI/litellm/pull/14040)
|
||||
* @gotsysdba made their first contribution in [PR #14036](https://github.com/BerriAI/litellm/pull/14036)
|
||||
* @moshemorad made their first contribution in [PR #14005](https://github.com/BerriAI/litellm/pull/14005)
|
||||
* @joshualipman123 made their first contribution in [PR #13144](https://github.com/BerriAI/litellm/pull/13144)
|
||||
|
||||
---
|
||||
|
||||
## **[Full Changelog](https://github.com/BerriAI/litellm/compare/v1.76.0-nightly...v1.76.1)**
|
||||
|
|
@ -95,13 +95,14 @@ class PrometheusLogger(CustomLogger):
|
|||
self.litellm_llm_api_time_to_first_token_metric = self._histogram_factory(
|
||||
"litellm_llm_api_time_to_first_token_metric",
|
||||
"Time to first token for a models LLM API call",
|
||||
labelnames=[
|
||||
"model",
|
||||
"hashed_api_key",
|
||||
"api_key_alias",
|
||||
"team",
|
||||
"team_alias",
|
||||
],
|
||||
# labelnames=[
|
||||
# "model",
|
||||
# "hashed_api_key",
|
||||
# "api_key_alias",
|
||||
# "team",
|
||||
# "team_alias",
|
||||
# ],
|
||||
labelnames=self.get_labels_for_metric("litellm_llm_api_time_to_first_token_metric"),
|
||||
buckets=LATENCY_BUCKETS,
|
||||
)
|
||||
|
||||
|
|
@ -109,15 +110,7 @@ class PrometheusLogger(CustomLogger):
|
|||
self.litellm_spend_metric = self._counter_factory(
|
||||
"litellm_spend_metric",
|
||||
"Total spend on LLM requests",
|
||||
labelnames=[
|
||||
"end_user",
|
||||
"hashed_api_key",
|
||||
"api_key_alias",
|
||||
"model",
|
||||
"team",
|
||||
"team_alias",
|
||||
"user",
|
||||
],
|
||||
labelnames=self.get_labels_for_metric("litellm_spend_metric"),
|
||||
)
|
||||
|
||||
# Counter for total_output_tokens
|
||||
|
|
@ -243,25 +236,18 @@ class PrometheusLogger(CustomLogger):
|
|||
labelnames=["api_provider"],
|
||||
)
|
||||
|
||||
# Get all keys
|
||||
_logged_llm_labels = [
|
||||
UserAPIKeyLabelNames.v2_LITELLM_MODEL_NAME.value,
|
||||
UserAPIKeyLabelNames.MODEL_ID.value,
|
||||
UserAPIKeyLabelNames.API_BASE.value,
|
||||
UserAPIKeyLabelNames.API_PROVIDER.value,
|
||||
]
|
||||
|
||||
# Metric for deployment state
|
||||
self.litellm_deployment_state = self._gauge_factory(
|
||||
"litellm_deployment_state",
|
||||
"LLM Deployment Analytics - The state of the deployment: 0 = healthy, 1 = partial outage, 2 = complete outage",
|
||||
labelnames=_logged_llm_labels,
|
||||
labelnames=self.get_labels_for_metric("litellm_deployment_state")
|
||||
)
|
||||
|
||||
self.litellm_deployment_cooled_down = self._counter_factory(
|
||||
"litellm_deployment_cooled_down",
|
||||
"LLM Deployment Analytics - Number of times a deployment has been cooled down by LiteLLM load balancing logic. exception_status is the status of the exception that caused the deployment to be cooled down",
|
||||
labelnames=_logged_llm_labels + [EXCEPTION_STATUS],
|
||||
# labelnames=_logged_llm_labels + [EXCEPTION_STATUS],
|
||||
labelnames=self.get_labels_for_metric("litellm_deployment_cooled_down")
|
||||
)
|
||||
|
||||
self.litellm_deployment_success_responses = self._counter_factory(
|
||||
|
|
@ -327,6 +313,7 @@ class PrometheusLogger(CustomLogger):
|
|||
documentation="deprecated - use litellm_proxy_total_requests_metric. Total number of LLM calls to litellm - track total per API Key, team, user",
|
||||
labelnames=self.get_labels_for_metric("litellm_requests_metric"),
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
print_verbose(f"Got exception on init prometheus client {str(e)}")
|
||||
raise e
|
||||
|
|
|
|||
|
|
@ -1261,6 +1261,7 @@ from .exceptions import (
|
|||
AuthenticationError,
|
||||
InvalidRequestError,
|
||||
BadRequestError,
|
||||
ImageFetchError,
|
||||
NotFoundError,
|
||||
RateLimitError,
|
||||
ServiceUnavailableError,
|
||||
|
|
|
|||
|
|
@ -14,6 +14,7 @@ DEFAULT_S3_BATCH_SIZE = int(os.getenv("DEFAULT_S3_BATCH_SIZE", 512))
|
|||
DEFAULT_SQS_FLUSH_INTERVAL_SECONDS = int(
|
||||
os.getenv("DEFAULT_SQS_FLUSH_INTERVAL_SECONDS", 10)
|
||||
)
|
||||
DEFAULT_NUM_WORKERS_LITELLM_PROXY = int(os.getenv("DEFAULT_NUM_WORKERS_LITELLM_PROXY", 4))
|
||||
DEFAULT_SQS_BATCH_SIZE = int(os.getenv("DEFAULT_SQS_BATCH_SIZE", 512))
|
||||
SQS_SEND_MESSAGE_ACTION = "SendMessage"
|
||||
SQS_API_VERSION = "2012-11-05"
|
||||
|
|
|
|||
|
|
@ -57,6 +57,7 @@ from litellm.llms.vertex_ai.cost_calculator import (
|
|||
cost_per_token as google_cost_per_token,
|
||||
)
|
||||
from litellm.llms.vertex_ai.cost_calculator import cost_router as google_cost_router
|
||||
from litellm.llms.xai.cost_calculator import cost_per_token as xai_cost_per_token
|
||||
from litellm.responses.utils import ResponseAPILoggingUtils
|
||||
from litellm.types.llms.openai import (
|
||||
HttpxBinaryResponseContent,
|
||||
|
|
@ -341,6 +342,8 @@ def cost_per_token( # noqa: PLR0915
|
|||
return deepseek_cost_per_token(model=model, usage=usage_block)
|
||||
elif custom_llm_provider == "perplexity":
|
||||
return perplexity_cost_per_token(model=model, usage=usage_block)
|
||||
elif custom_llm_provider == "xai":
|
||||
return xai_cost_per_token(model=model, usage=usage_block)
|
||||
else:
|
||||
model_info = _cached_get_model_info_helper(
|
||||
model=model, custom_llm_provider=custom_llm_provider
|
||||
|
|
@ -675,9 +678,9 @@ def completion_cost( # noqa: PLR0915
|
|||
or isinstance(completion_response, dict)
|
||||
): # tts returns a custom class
|
||||
if isinstance(completion_response, dict):
|
||||
usage_obj: Optional[Union[dict, Usage]] = (
|
||||
completion_response.get("usage", {})
|
||||
)
|
||||
usage_obj: Optional[
|
||||
Union[dict, Usage]
|
||||
] = completion_response.get("usage", {})
|
||||
else:
|
||||
usage_obj = getattr(completion_response, "usage", {})
|
||||
if isinstance(usage_obj, BaseModel) and not _is_known_usage_objects(
|
||||
|
|
@ -1279,7 +1282,9 @@ class BaseTokenUsageProcessor:
|
|||
not hasattr(combined, "completion_tokens_details")
|
||||
or not combined.completion_tokens_details
|
||||
):
|
||||
combined.completion_tokens_details = CompletionTokensDetailsWrapper()
|
||||
combined.completion_tokens_details = (
|
||||
CompletionTokensDetailsWrapper()
|
||||
)
|
||||
|
||||
# Check what keys exist in the model's completion_tokens_details
|
||||
for attr in usage.completion_tokens_details.model_fields:
|
||||
|
|
|
|||
|
|
@ -153,6 +153,29 @@ class BadRequestError(openai.BadRequestError): # type: ignore
|
|||
_message += f", LiteLLM Max Retries: {self.max_retries}"
|
||||
return _message
|
||||
|
||||
class ImageFetchError(BadRequestError):
|
||||
def __init__(
|
||||
self,
|
||||
message,
|
||||
model=None,
|
||||
llm_provider=None,
|
||||
response: Optional[httpx.Response] = None,
|
||||
litellm_debug_info: Optional[str] = None,
|
||||
max_retries: Optional[int] = None,
|
||||
num_retries: Optional[int] = None,
|
||||
body: Optional[dict] = None,
|
||||
):
|
||||
super().__init__(
|
||||
message=message,
|
||||
model=model,
|
||||
llm_provider=llm_provider,
|
||||
response=response,
|
||||
litellm_debug_info=litellm_debug_info,
|
||||
max_retries=max_retries,
|
||||
num_retries=num_retries,
|
||||
body=body,
|
||||
)
|
||||
|
||||
|
||||
class UnprocessableEntityError(openai.UnprocessableEntityError): # type: ignore
|
||||
def __init__(
|
||||
|
|
|
|||
|
|
@ -37,6 +37,10 @@ class GenerateContentToCompletionHandler:
|
|||
|
||||
completion_kwargs: Dict[str, Any] = dict(completion_request)
|
||||
|
||||
# feed metadata for custom callback
|
||||
if extra_kwargs is not None and "metadata" in extra_kwargs:
|
||||
completion_kwargs["metadata"] = extra_kwargs["metadata"]
|
||||
|
||||
if stream:
|
||||
completion_kwargs["stream"] = stream
|
||||
|
||||
|
|
|
|||
|
|
@ -1,13 +1,11 @@
|
|||
# What is this?
|
||||
## Log success + failure events to Braintrust
|
||||
|
||||
import copy
|
||||
import os
|
||||
from datetime import datetime
|
||||
from typing import Dict, Optional
|
||||
|
||||
import httpx
|
||||
from pydantic import BaseModel
|
||||
|
||||
import litellm
|
||||
from litellm import verbose_logger
|
||||
|
|
@ -24,7 +22,6 @@ API_BASE = "https://api.braintrustdata.com/v1"
|
|||
|
||||
def get_utc_datetime():
|
||||
import datetime as dt
|
||||
from datetime import datetime
|
||||
|
||||
if hasattr(dt, "UTC"):
|
||||
return datetime.now(dt.UTC) # type: ignore
|
||||
|
|
@ -45,9 +42,9 @@ class BraintrustLogger(CustomLogger):
|
|||
"Authorization": "Bearer " + self.api_key,
|
||||
"Content-Type": "application/json",
|
||||
}
|
||||
self._project_id_cache: Dict[
|
||||
str, str
|
||||
] = {} # Cache mapping project names to IDs
|
||||
self._project_id_cache: Dict[str, str] = (
|
||||
{}
|
||||
) # Cache mapping project names to IDs
|
||||
self.global_braintrust_http_handler = get_async_httpx_client(
|
||||
llm_provider=httpxSpecialProvider.LoggingCallback
|
||||
)
|
||||
|
|
@ -108,43 +105,6 @@ class BraintrustLogger(CustomLogger):
|
|||
except httpx.HTTPStatusError as e:
|
||||
raise Exception(f"Failed to register project: {e.response.text}")
|
||||
|
||||
@staticmethod
|
||||
def add_metadata_from_header(litellm_params: dict, metadata: dict) -> dict:
|
||||
"""
|
||||
Adds metadata from proxy request headers to Braintrust logging if keys start with "braintrust_"
|
||||
and overwrites litellm_params.metadata if already included.
|
||||
|
||||
For example if you want to append your trace to an existing `trace_id` via header, send
|
||||
`headers: { ..., langfuse_existing_trace_id: your-existing-trace-id }` via proxy request.
|
||||
"""
|
||||
if litellm_params is None:
|
||||
return metadata
|
||||
|
||||
if litellm_params.get("proxy_server_request") is None:
|
||||
return metadata
|
||||
|
||||
if metadata is None:
|
||||
metadata = {}
|
||||
|
||||
proxy_headers = (
|
||||
litellm_params.get("proxy_server_request", {}).get("headers", {}) or {}
|
||||
)
|
||||
|
||||
for metadata_param_key in proxy_headers:
|
||||
if metadata_param_key.startswith("braintrust"):
|
||||
trace_param_key = metadata_param_key.replace("braintrust", "", 1)
|
||||
if trace_param_key in metadata:
|
||||
verbose_logger.warning(
|
||||
f"Overwriting Braintrust `{trace_param_key}` from request header"
|
||||
)
|
||||
else:
|
||||
verbose_logger.debug(
|
||||
f"Found Braintrust `{trace_param_key}` in request header"
|
||||
)
|
||||
metadata[trace_param_key] = proxy_headers.get(metadata_param_key)
|
||||
|
||||
return metadata
|
||||
|
||||
async def create_default_project_and_experiment(self):
|
||||
project = await self.global_braintrust_http_handler.post(
|
||||
f"{self.api_base}/project", headers=self.headers, json={"name": "litellm"}
|
||||
|
|
@ -169,7 +129,9 @@ class BraintrustLogger(CustomLogger):
|
|||
verbose_logger.debug("REACHES BRAINTRUST SUCCESS")
|
||||
try:
|
||||
litellm_call_id = kwargs.get("litellm_call_id")
|
||||
standard_logging_object = kwargs.get("standard_logging_object", {})
|
||||
prompt = {"messages": kwargs.get("messages")}
|
||||
|
||||
output = None
|
||||
choices = []
|
||||
if response_obj is not None and (
|
||||
|
|
@ -192,33 +154,13 @@ class BraintrustLogger(CustomLogger):
|
|||
):
|
||||
output = response_obj["data"]
|
||||
|
||||
litellm_params = kwargs.get("litellm_params", {})
|
||||
metadata = (
|
||||
litellm_params.get("metadata", {}) or {}
|
||||
) # if litellm_params['metadata'] == None
|
||||
metadata = self.add_metadata_from_header(litellm_params, metadata)
|
||||
clean_metadata = {}
|
||||
try:
|
||||
metadata = copy.deepcopy(
|
||||
metadata
|
||||
) # Avoid modifying the original metadata
|
||||
except Exception:
|
||||
new_metadata = {}
|
||||
for key, value in metadata.items():
|
||||
if (
|
||||
isinstance(value, list)
|
||||
or isinstance(value, dict)
|
||||
or isinstance(value, str)
|
||||
or isinstance(value, int)
|
||||
or isinstance(value, float)
|
||||
):
|
||||
new_metadata[key] = copy.deepcopy(value)
|
||||
metadata = new_metadata
|
||||
litellm_params = kwargs.get("litellm_params", {}) or {}
|
||||
dynamic_metadata = litellm_params.get("metadata", {}) or {}
|
||||
|
||||
# Get project_id from metadata or create default if needed
|
||||
project_id = metadata.get("project_id")
|
||||
project_id = dynamic_metadata.get("project_id")
|
||||
if project_id is None:
|
||||
project_name = metadata.get("project_name")
|
||||
project_name = dynamic_metadata.get("project_name")
|
||||
project_id = (
|
||||
self.get_project_id_sync(project_name) if project_name else None
|
||||
)
|
||||
|
|
@ -229,8 +171,9 @@ class BraintrustLogger(CustomLogger):
|
|||
project_id = self.default_project_id
|
||||
|
||||
tags = []
|
||||
if isinstance(metadata, dict):
|
||||
for key, value in metadata.items():
|
||||
|
||||
if isinstance(dynamic_metadata, dict):
|
||||
for key, value in dynamic_metadata.items():
|
||||
# generate langfuse tags - Default Tags sent to Langfuse from LiteLLM Proxy
|
||||
if (
|
||||
litellm.langfuse_default_tags is not None
|
||||
|
|
@ -239,25 +182,12 @@ class BraintrustLogger(CustomLogger):
|
|||
):
|
||||
tags.append(f"{key}:{value}")
|
||||
|
||||
# clean litellm metadata before logging
|
||||
if key in [
|
||||
"headers",
|
||||
"endpoint",
|
||||
"caching_groups",
|
||||
"previous_models",
|
||||
]:
|
||||
continue
|
||||
else:
|
||||
clean_metadata[key] = value
|
||||
if (
|
||||
isinstance(value, str) and key not in standard_logging_object
|
||||
): # support logging dynamic metadata to braintrust
|
||||
standard_logging_object[key] = value
|
||||
|
||||
cost = kwargs.get("response_cost", None)
|
||||
if cost is not None:
|
||||
clean_metadata["litellm_response_cost"] = cost
|
||||
|
||||
# metadata.model is required for braintrust to calculate the "Estimated cost" metric
|
||||
litellm_model = kwargs.get("model", None)
|
||||
if litellm_model is not None:
|
||||
clean_metadata["model"] = litellm_model
|
||||
|
||||
metrics: Optional[dict] = None
|
||||
usage_obj = getattr(response_obj, "usage", None)
|
||||
|
|
@ -275,12 +205,12 @@ class BraintrustLogger(CustomLogger):
|
|||
}
|
||||
|
||||
# Allow metadata override for span name
|
||||
span_name = metadata.get("span_name", "Chat Completion")
|
||||
|
||||
span_name = dynamic_metadata.get("span_name", "Chat Completion")
|
||||
|
||||
request_data = {
|
||||
"id": litellm_call_id,
|
||||
"input": prompt["messages"],
|
||||
"metadata": clean_metadata,
|
||||
"metadata": standard_logging_object,
|
||||
"tags": tags,
|
||||
"span_attributes": {"name": span_name, "type": "llm"},
|
||||
}
|
||||
|
|
@ -312,6 +242,7 @@ class BraintrustLogger(CustomLogger):
|
|||
verbose_logger.debug("REACHES BRAINTRUST SUCCESS")
|
||||
try:
|
||||
litellm_call_id = kwargs.get("litellm_call_id")
|
||||
standard_logging_object = kwargs.get("standard_logging_object", {})
|
||||
prompt = {"messages": kwargs.get("messages")}
|
||||
output = None
|
||||
choices = []
|
||||
|
|
@ -336,32 +267,12 @@ class BraintrustLogger(CustomLogger):
|
|||
output = response_obj["data"]
|
||||
|
||||
litellm_params = kwargs.get("litellm_params", {})
|
||||
metadata = (
|
||||
litellm_params.get("metadata", {}) or {}
|
||||
) # if litellm_params['metadata'] == None
|
||||
metadata = self.add_metadata_from_header(litellm_params, metadata)
|
||||
clean_metadata = {}
|
||||
new_metadata = {}
|
||||
for key, value in metadata.items():
|
||||
if (
|
||||
isinstance(value, list)
|
||||
or isinstance(value, str)
|
||||
or isinstance(value, int)
|
||||
or isinstance(value, float)
|
||||
):
|
||||
new_metadata[key] = value
|
||||
elif isinstance(value, BaseModel):
|
||||
new_metadata[key] = value.model_dump_json()
|
||||
elif isinstance(value, dict):
|
||||
for k, v in value.items():
|
||||
if isinstance(v, datetime):
|
||||
value[k] = v.isoformat()
|
||||
new_metadata[key] = value
|
||||
dynamic_metadata = litellm_params.get("metadata", {}) or {}
|
||||
|
||||
# Get project_id from metadata or create default if needed
|
||||
project_id = metadata.get("project_id")
|
||||
project_id = dynamic_metadata.get("project_id")
|
||||
if project_id is None:
|
||||
project_name = metadata.get("project_name")
|
||||
project_name = dynamic_metadata.get("project_name")
|
||||
project_id = (
|
||||
await self.get_project_id_async(project_name)
|
||||
if project_name
|
||||
|
|
@ -374,8 +285,9 @@ class BraintrustLogger(CustomLogger):
|
|||
project_id = self.default_project_id
|
||||
|
||||
tags = []
|
||||
if isinstance(metadata, dict):
|
||||
for key, value in metadata.items():
|
||||
|
||||
if isinstance(dynamic_metadata, dict):
|
||||
for key, value in dynamic_metadata.items():
|
||||
# generate langfuse tags - Default Tags sent to Langfuse from LiteLLM Proxy
|
||||
if (
|
||||
litellm.langfuse_default_tags is not None
|
||||
|
|
@ -384,25 +296,12 @@ class BraintrustLogger(CustomLogger):
|
|||
):
|
||||
tags.append(f"{key}:{value}")
|
||||
|
||||
# clean litellm metadata before logging
|
||||
if key in [
|
||||
"headers",
|
||||
"endpoint",
|
||||
"caching_groups",
|
||||
"previous_models",
|
||||
]:
|
||||
continue
|
||||
else:
|
||||
clean_metadata[key] = value
|
||||
if (
|
||||
isinstance(value, str) and key not in standard_logging_object
|
||||
): # support logging dynamic metadata to braintrust
|
||||
standard_logging_object[key] = value
|
||||
|
||||
cost = kwargs.get("response_cost", None)
|
||||
if cost is not None:
|
||||
clean_metadata["litellm_response_cost"] = cost
|
||||
|
||||
# metadata.model is required for braintrust to calculate the "Estimated cost" metric
|
||||
litellm_model = kwargs.get("model", None)
|
||||
if litellm_model is not None:
|
||||
clean_metadata["model"] = litellm_model
|
||||
|
||||
metrics: Optional[dict] = None
|
||||
usage_obj = getattr(response_obj, "usage", None)
|
||||
|
|
@ -430,13 +329,13 @@ class BraintrustLogger(CustomLogger):
|
|||
)
|
||||
|
||||
# Allow metadata override for span name
|
||||
span_name = metadata.get("span_name", "Chat Completion")
|
||||
|
||||
span_name = dynamic_metadata.get("span_name", "Chat Completion")
|
||||
|
||||
request_data = {
|
||||
"id": litellm_call_id,
|
||||
"input": prompt["messages"],
|
||||
"output": output,
|
||||
"metadata": clean_metadata,
|
||||
"metadata": standard_logging_object,
|
||||
"tags": tags,
|
||||
"span_attributes": {"name": span_name, "type": "llm"},
|
||||
}
|
||||
|
|
|
|||
|
|
@ -17,7 +17,7 @@ in_memory_cache = InMemoryCache(max_size_in_memory=MAX_IMGS_IN_MEMORY)
|
|||
|
||||
def _process_image_response(response: Response, url: str) -> str:
|
||||
if response.status_code != 200:
|
||||
raise Exception(
|
||||
raise litellm.ImageFetchError(
|
||||
f"Error: Unable to fetch image from URL. Status code: {response.status_code}, url={url}"
|
||||
)
|
||||
|
||||
|
|
@ -57,9 +57,11 @@ async def async_convert_url_to_base64(url: str) -> str:
|
|||
try:
|
||||
response = await client.get(url, follow_redirects=True)
|
||||
return _process_image_response(response, url)
|
||||
except litellm.ImageFetchError:
|
||||
raise
|
||||
except Exception:
|
||||
pass
|
||||
raise Exception(
|
||||
raise litellm.ImageFetchError(
|
||||
f"Error: Unable to fetch image from URL after 3 attempts. url={url}"
|
||||
)
|
||||
|
||||
|
|
@ -74,10 +76,11 @@ def convert_url_to_base64(url: str) -> str:
|
|||
try:
|
||||
response = client.get(url, follow_redirects=True)
|
||||
return _process_image_response(response, url)
|
||||
except litellm.ImageFetchError:
|
||||
raise
|
||||
except Exception as e:
|
||||
verbose_logger.exception(e)
|
||||
# print(e)
|
||||
pass
|
||||
raise Exception(
|
||||
f"Error: Unable to fetch image from URL after 3 attempts. url={url}"
|
||||
raise litellm.ImageFetchError(
|
||||
f"Error: Unable to fetch image from URL after 3 attempts. url={url}",
|
||||
)
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
import json
|
||||
from typing import Any, Union
|
||||
|
||||
from litellm.constants import DEFAULT_MAX_RECURSE_DEPTH
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -137,6 +137,7 @@ class OllamaChatConfig(BaseConfig):
|
|||
"tool_choice",
|
||||
"functions",
|
||||
"response_format",
|
||||
"reasoning_effort",
|
||||
]
|
||||
|
||||
def map_openai_params(
|
||||
|
|
@ -175,6 +176,8 @@ class OllamaChatConfig(BaseConfig):
|
|||
if value.get("json_schema") and value["json_schema"].get("schema"):
|
||||
optional_params["format"] = value["json_schema"]["schema"]
|
||||
### FUNCTION CALLING LOGIC ###
|
||||
if param == "reasoning_effort" and value is not None:
|
||||
optional_params["think"] = True
|
||||
if param == "tools":
|
||||
## CHECK IF MODEL SUPPORTS TOOL CALLING ##
|
||||
try:
|
||||
|
|
@ -212,9 +215,9 @@ class OllamaChatConfig(BaseConfig):
|
|||
litellm.add_function_to_prompt = (
|
||||
True # so that main.py adds the function call to the prompt
|
||||
)
|
||||
optional_params[
|
||||
"functions_unsupported_model"
|
||||
] = non_default_params.get("functions")
|
||||
optional_params["functions_unsupported_model"] = (
|
||||
non_default_params.get("functions")
|
||||
)
|
||||
non_default_params.pop("tool_choice", None) # causes ollama requests to hang
|
||||
non_default_params.pop("functions", None) # causes ollama requests to hang
|
||||
return optional_params
|
||||
|
|
@ -346,11 +349,31 @@ class OllamaChatConfig(BaseConfig):
|
|||
|
||||
## RESPONSE OBJECT
|
||||
model_response.choices[0].finish_reason = "stop"
|
||||
response_json_message = response_json.get("message")
|
||||
if response_json_message is not None:
|
||||
if "thinking" in response_json_message:
|
||||
# remap 'thinking' to 'reasoning_content'
|
||||
response_json_message["reasoning_content"] = response_json_message[
|
||||
"thinking"
|
||||
]
|
||||
del response_json_message["thinking"]
|
||||
elif response_json_message.get("content") is not None:
|
||||
# parse reasoning content from content
|
||||
from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import (
|
||||
_parse_content_for_reasoning,
|
||||
)
|
||||
|
||||
reasoning_content, content = _parse_content_for_reasoning(
|
||||
response_json_message["content"]
|
||||
)
|
||||
response_json_message["reasoning_content"] = reasoning_content
|
||||
response_json_message["content"] = content
|
||||
|
||||
if (
|
||||
request_data.get("format", "") == "json"
|
||||
and litellm_params.get("function_name") is not None
|
||||
):
|
||||
function_call = json.loads(response_json["message"]["content"])
|
||||
function_call = json.loads(response_json_message["content"])
|
||||
message = litellm.Message(
|
||||
content=None,
|
||||
tool_calls=[
|
||||
|
|
@ -367,11 +390,13 @@ class OllamaChatConfig(BaseConfig):
|
|||
"type": "function",
|
||||
}
|
||||
],
|
||||
reasoning_content=response_json_message.get("reasoning_content"),
|
||||
)
|
||||
model_response.choices[0].message = message # type: ignore
|
||||
model_response.choices[0].finish_reason = "tool_calls"
|
||||
else:
|
||||
_message = litellm.Message(**response_json["message"])
|
||||
|
||||
_message = litellm.Message(**response_json_message)
|
||||
model_response.choices[0].message = _message # type: ignore
|
||||
model_response.created = int(time.time())
|
||||
model_response.model = "ollama_chat/" + model
|
||||
|
|
@ -412,6 +437,9 @@ class OllamaChatConfig(BaseConfig):
|
|||
|
||||
|
||||
class OllamaChatCompletionResponseIterator(BaseModelResponseIterator):
|
||||
started_reasoning_content: bool = False
|
||||
finished_reasoning_content: bool = False
|
||||
|
||||
def _is_function_call_complete(self, function_args: Union[str, dict]) -> bool:
|
||||
if isinstance(function_args, dict):
|
||||
return True
|
||||
|
|
@ -465,8 +493,38 @@ class OllamaChatCompletionResponseIterator(BaseModelResponseIterator):
|
|||
if is_function_call_complete:
|
||||
tool_call["id"] = str(uuid.uuid4())
|
||||
|
||||
# PROCESS REASONING CONTENT
|
||||
reasoning_content: Optional[str] = None
|
||||
content: Optional[str] = None
|
||||
if chunk["message"].get("thinking") is not None:
|
||||
if self.started_reasoning_content is False:
|
||||
reasoning_content = chunk["message"].get("thinking")
|
||||
self.started_reasoning_content = True
|
||||
elif self.finished_reasoning_content is False:
|
||||
reasoning_content = chunk["message"].get("thinking")
|
||||
self.finished_reasoning_content = True
|
||||
elif chunk["message"].get("content") is not None:
|
||||
message_content = chunk["message"].get("content")
|
||||
if "<think>" in message_content:
|
||||
message_content = message_content.replace("<think>", "")
|
||||
|
||||
self.started_reasoning_content = True
|
||||
|
||||
if "</think>" in message_content and self.started_reasoning_content:
|
||||
message_content = message_content.replace("</think>", "")
|
||||
self.finished_reasoning_content = True
|
||||
|
||||
if (
|
||||
self.started_reasoning_content
|
||||
and not self.finished_reasoning_content
|
||||
):
|
||||
reasoning_content = message_content
|
||||
else:
|
||||
content = message_content
|
||||
|
||||
delta = Delta(
|
||||
content=chunk["message"].get("content", ""),
|
||||
content=content,
|
||||
reasoning_content=reasoning_content,
|
||||
tool_calls=tool_calls,
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -19,13 +19,13 @@ from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMExcepti
|
|||
from litellm.secret_managers.main import get_secret_str
|
||||
from litellm.types.llms.openai import AllMessageValues, ChatCompletionUsageBlock
|
||||
from litellm.types.utils import (
|
||||
Delta,
|
||||
GenericStreamingChunk,
|
||||
ModelInfoBase,
|
||||
ModelResponse,
|
||||
ModelResponseStream,
|
||||
ProviderField,
|
||||
StreamingChoices,
|
||||
Delta,
|
||||
)
|
||||
|
||||
from ..common_utils import OllamaError, _convert_image
|
||||
|
|
@ -92,9 +92,9 @@ class OllamaConfig(BaseConfig):
|
|||
repeat_penalty: Optional[float] = None
|
||||
temperature: Optional[float] = None
|
||||
seed: Optional[int] = None
|
||||
stop: Optional[
|
||||
list
|
||||
] = None # stop is a list based on this - https://github.com/ollama/ollama/pull/442
|
||||
stop: Optional[list] = (
|
||||
None # stop is a list based on this - https://github.com/ollama/ollama/pull/442
|
||||
)
|
||||
tfs_z: Optional[float] = None
|
||||
num_predict: Optional[int] = None
|
||||
top_k: Optional[int] = None
|
||||
|
|
@ -154,6 +154,7 @@ class OllamaConfig(BaseConfig):
|
|||
"stop",
|
||||
"response_format",
|
||||
"max_completion_tokens",
|
||||
"reasoning_effort",
|
||||
]
|
||||
|
||||
def map_openai_params(
|
||||
|
|
@ -166,19 +167,21 @@ class OllamaConfig(BaseConfig):
|
|||
for param, value in non_default_params.items():
|
||||
if param == "max_tokens" or param == "max_completion_tokens":
|
||||
optional_params["num_predict"] = value
|
||||
if param == "stream":
|
||||
elif param == "stream":
|
||||
optional_params["stream"] = value
|
||||
if param == "temperature":
|
||||
elif param == "temperature":
|
||||
optional_params["temperature"] = value
|
||||
if param == "seed":
|
||||
elif param == "seed":
|
||||
optional_params["seed"] = value
|
||||
if param == "top_p":
|
||||
elif param == "top_p":
|
||||
optional_params["top_p"] = value
|
||||
if param == "frequency_penalty":
|
||||
elif param == "frequency_penalty":
|
||||
optional_params["frequency_penalty"] = value
|
||||
if param == "stop":
|
||||
elif param == "stop":
|
||||
optional_params["stop"] = value
|
||||
if param == "response_format" and isinstance(value, dict):
|
||||
elif param == "reasoning_effort" and value is not None:
|
||||
optional_params["think"] = True
|
||||
elif param == "response_format" and isinstance(value, dict):
|
||||
if value["type"] == "json_object":
|
||||
optional_params["format"] = "json"
|
||||
elif value["type"] == "json_schema":
|
||||
|
|
@ -258,12 +261,17 @@ class OllamaConfig(BaseConfig):
|
|||
api_key: Optional[str] = None,
|
||||
json_mode: Optional[bool] = None,
|
||||
) -> ModelResponse:
|
||||
from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import (
|
||||
_parse_content_for_reasoning,
|
||||
)
|
||||
|
||||
response_json = raw_response.json()
|
||||
## RESPONSE OBJECT
|
||||
model_response.choices[0].finish_reason = "stop"
|
||||
if request_data.get("format", "") == "json":
|
||||
# Check if response field exists and is not empty before parsing JSON
|
||||
response_text = response_json.get("response", "")
|
||||
|
||||
if not response_text or not response_text.strip():
|
||||
# Handle empty response gracefully - set empty content
|
||||
message = litellm.Message(content="")
|
||||
|
|
@ -288,7 +296,9 @@ class OllamaConfig(BaseConfig):
|
|||
"id": f"call_{str(uuid.uuid4())}",
|
||||
"function": {
|
||||
"name": function_call["name"],
|
||||
"arguments": json.dumps(function_call["arguments"]),
|
||||
"arguments": json.dumps(
|
||||
function_call["arguments"]
|
||||
),
|
||||
},
|
||||
"type": "function",
|
||||
}
|
||||
|
|
@ -305,11 +315,26 @@ class OllamaConfig(BaseConfig):
|
|||
model_response.choices[0].finish_reason = "stop"
|
||||
except json.JSONDecodeError:
|
||||
# If JSON parsing fails, treat as regular text response
|
||||
message = litellm.Message(content=response_text)
|
||||
## output parse reasoning content from response_text
|
||||
reasoning_content: Optional[str] = None
|
||||
content: Optional[str] = None
|
||||
if response_text is not None:
|
||||
reasoning_content, content = _parse_content_for_reasoning(
|
||||
response_text
|
||||
)
|
||||
message = litellm.Message(
|
||||
content=content, reasoning_content=reasoning_content
|
||||
)
|
||||
model_response.choices[0].message = message # type: ignore
|
||||
model_response.choices[0].finish_reason = "stop"
|
||||
else:
|
||||
model_response.choices[0].message.content = response_json["response"] # type: ignore
|
||||
response_text = response_json.get("response", "")
|
||||
content = None
|
||||
reasoning_content = None
|
||||
if response_text is not None:
|
||||
reasoning_content, content = _parse_content_for_reasoning(response_text)
|
||||
model_response.choices[0].message.content = content # type: ignore
|
||||
model_response.choices[0].message.reasoning_content = reasoning_content # type: ignore
|
||||
model_response.created = int(time.time())
|
||||
model_response.model = "ollama/" + model
|
||||
_prompt = request_data.get("prompt", "")
|
||||
|
|
@ -434,12 +459,21 @@ class OllamaConfig(BaseConfig):
|
|||
|
||||
|
||||
class OllamaTextCompletionResponseIterator(BaseModelResponseIterator):
|
||||
def __init__(
|
||||
self, streaming_response, sync_stream: bool, json_mode: Optional[bool] = False
|
||||
):
|
||||
super().__init__(streaming_response, sync_stream, json_mode)
|
||||
self.started_reasoning_content: bool = False
|
||||
self.finished_reasoning_content: bool = False
|
||||
|
||||
def _handle_string_chunk(
|
||||
self, str_line: str
|
||||
) -> Union[GenericStreamingChunk, ModelResponseStream]:
|
||||
return self.chunk_parser(json.loads(str_line))
|
||||
|
||||
def chunk_parser(self, chunk: dict) -> Union[GenericStreamingChunk, ModelResponseStream]:
|
||||
def chunk_parser(
|
||||
self, chunk: dict
|
||||
) -> Union[GenericStreamingChunk, ModelResponseStream]:
|
||||
try:
|
||||
if "error" in chunk:
|
||||
raise Exception(f"Ollama Error - {chunk}")
|
||||
|
|
@ -469,12 +503,42 @@ class OllamaTextCompletionResponseIterator(BaseModelResponseIterator):
|
|||
)
|
||||
elif chunk["response"]:
|
||||
text = chunk["response"]
|
||||
return GenericStreamingChunk(
|
||||
text=text,
|
||||
is_finished=is_finished,
|
||||
finish_reason="stop",
|
||||
reasoning_content: Optional[str] = None
|
||||
content: Optional[str] = None
|
||||
if text is not None:
|
||||
if "<think>" in text:
|
||||
text = text.replace("<think>", "")
|
||||
self.started_reasoning_content = True
|
||||
elif "</think>" in text:
|
||||
text = text.replace("</think>", "")
|
||||
self.finished_reasoning_content = True
|
||||
|
||||
if (
|
||||
self.started_reasoning_content
|
||||
and not self.finished_reasoning_content
|
||||
):
|
||||
reasoning_content = text
|
||||
else:
|
||||
content = text
|
||||
|
||||
return ModelResponseStream(
|
||||
choices=[
|
||||
StreamingChoices(
|
||||
index=0,
|
||||
delta=Delta(
|
||||
reasoning_content=reasoning_content, content=content
|
||||
),
|
||||
)
|
||||
],
|
||||
finish_reason=finish_reason,
|
||||
usage=None,
|
||||
)
|
||||
# return GenericStreamingChunk(
|
||||
# text=text,
|
||||
# is_finished=is_finished,
|
||||
# finish_reason="stop",
|
||||
# usage=None,
|
||||
# )
|
||||
elif "thinking" in chunk and not chunk["response"]:
|
||||
# Return reasoning content as ModelResponseStream so UIs can render it
|
||||
thinking_content = chunk.get("thinking") or ""
|
||||
|
|
|
|||
|
|
@ -28,7 +28,18 @@ class OpenAIGPT5Config(OpenAIGPTConfig):
|
|||
base_gpt_series_params.extend(gpt_5_only_params)
|
||||
if not supports_tool_choice(model=model):
|
||||
base_gpt_series_params.remove("tool_choice")
|
||||
return base_gpt_series_params
|
||||
|
||||
non_supported_params = [
|
||||
"logprobs",
|
||||
"top_p",
|
||||
"presence_penalty",
|
||||
"frequency_penalty",
|
||||
"top_logprobs",
|
||||
]
|
||||
|
||||
return [
|
||||
param for param in base_gpt_series_params if param not in non_supported_params
|
||||
]
|
||||
|
||||
def map_openai_params(
|
||||
self,
|
||||
|
|
|
|||
|
|
@ -187,6 +187,25 @@ def _check_text_in_content(parts: List[PartType]) -> bool:
|
|||
return has_text_param
|
||||
|
||||
|
||||
def _fix_enum_empty_strings(schema, depth=0):
|
||||
"""Fix empty strings in enum values by replacing them with None. Gemini doesn't accept empty strings in enums."""
|
||||
if depth > DEFAULT_MAX_RECURSE_DEPTH:
|
||||
raise ValueError(f"Max depth of {DEFAULT_MAX_RECURSE_DEPTH} exceeded while processing schema.")
|
||||
|
||||
if "enum" in schema and isinstance(schema["enum"], list):
|
||||
schema["enum"] = [None if value == "" else value for value in schema["enum"]]
|
||||
|
||||
# Reuse existing recursion pattern from convert_anyof_null_to_nullable
|
||||
properties = schema.get("properties", None)
|
||||
if properties is not None:
|
||||
for _, value in properties.items():
|
||||
_fix_enum_empty_strings(value, depth=depth + 1)
|
||||
|
||||
items = schema.get("items", None)
|
||||
if items is not None:
|
||||
_fix_enum_empty_strings(items, depth=depth + 1)
|
||||
|
||||
|
||||
def _build_vertex_schema(parameters: dict, add_property_ordering: bool = False):
|
||||
"""
|
||||
This is a modified version of https://github.com/google-gemini/generative-ai-python/blob/8f77cc6ac99937cd3a81299ecf79608b91b06bbb/google/generativeai/types/content_types.py#L419
|
||||
|
|
@ -215,6 +234,11 @@ def _build_vertex_schema(parameters: dict, add_property_ordering: bool = False):
|
|||
# * https://github.com/pydantic/pydantic/discussions/4872
|
||||
convert_anyof_null_to_nullable(parameters)
|
||||
|
||||
_convert_schema_types(parameters)
|
||||
|
||||
# Handle empty strings in enum values - Gemini doesn't accept empty strings in enums
|
||||
_fix_enum_empty_strings(parameters)
|
||||
|
||||
# Handle empty items objects
|
||||
process_items(parameters)
|
||||
add_object_type(parameters)
|
||||
|
|
@ -439,6 +463,47 @@ def _convert_vertex_datetime_to_openai_datetime(vertex_datetime: str) -> int:
|
|||
return int(dt.timestamp())
|
||||
|
||||
|
||||
def _convert_schema_types(schema, depth=0):
|
||||
"""
|
||||
Convert type arrays and lowercase types for Vertex AI compatibility.
|
||||
|
||||
Transforms OpenAI-style schemas to Vertex AI format by converting type arrays
|
||||
like ["string", "number"] to anyOf format and converting all types to uppercase.
|
||||
"""
|
||||
if depth > DEFAULT_MAX_RECURSE_DEPTH:
|
||||
raise ValueError(
|
||||
f"Max depth of {DEFAULT_MAX_RECURSE_DEPTH} exceeded while processing schema. Please check the schema for excessive nesting."
|
||||
)
|
||||
|
||||
if not isinstance(schema, dict):
|
||||
return
|
||||
|
||||
|
||||
# Handle type field
|
||||
if "type" in schema:
|
||||
type_val = schema["type"]
|
||||
if isinstance(type_val, list) and len(type_val) > 1:
|
||||
# Convert ["string", "number"] -> {"anyOf": [{"type": "STRING"}, {"type": "NUMBER"}]}
|
||||
schema["anyOf"] = [{"type": t} for t in type_val if isinstance(t, str)]
|
||||
schema.pop("type")
|
||||
elif isinstance(type_val, list) and len(type_val) == 1:
|
||||
schema["type"] = type_val[0]
|
||||
elif isinstance(type_val, str):
|
||||
schema["type"] = type_val
|
||||
|
||||
# Recursively process nested properties, items, and anyOf
|
||||
for key in ["properties", "items", "anyOf"]:
|
||||
if key in schema:
|
||||
value = schema[key]
|
||||
if key == "properties" and isinstance(value, dict):
|
||||
for prop_schema in value.values():
|
||||
_convert_schema_types(prop_schema, depth + 1)
|
||||
elif key == "items":
|
||||
_convert_schema_types(value, depth + 1)
|
||||
elif key == "anyOf" and isinstance(value, list):
|
||||
for anyof_schema in value:
|
||||
_convert_schema_types(anyof_schema, depth + 1)
|
||||
|
||||
def get_vertex_project_id_from_url(url: str) -> Optional[str]:
|
||||
"""
|
||||
Get the vertex project id from the url
|
||||
|
|
|
|||
|
|
@ -105,6 +105,64 @@ def _process_gemini_image(image_url: str, format: Optional[str] = None) -> PartT
|
|||
raise e
|
||||
|
||||
|
||||
def _snake_to_camel(snake_str: str) -> str:
|
||||
"""Convert snake_case to camelCase"""
|
||||
components = snake_str.split("_")
|
||||
return components[0] + "".join(x.capitalize() for x in components[1:])
|
||||
|
||||
|
||||
def _camel_to_snake(camel_str: str) -> str:
|
||||
"""Convert camelCase to snake_case"""
|
||||
import re
|
||||
|
||||
return re.sub(r"(?<!^)(?=[A-Z])", "_", camel_str).lower()
|
||||
|
||||
|
||||
def _get_equivalent_key(key: str, available_keys: set) -> Optional[str]:
|
||||
"""
|
||||
Get the equivalent key from available keys, checking both camelCase and snake_case variants
|
||||
"""
|
||||
if key in available_keys:
|
||||
return key
|
||||
|
||||
# Try camelCase version
|
||||
camel_key = _snake_to_camel(key)
|
||||
if camel_key in available_keys:
|
||||
return camel_key
|
||||
|
||||
# Try snake_case version
|
||||
snake_key = _camel_to_snake(key)
|
||||
if snake_key in available_keys:
|
||||
return snake_key
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def check_if_part_exists_in_parts(
|
||||
parts: List[PartType], part: PartType, excluded_keys: List[str] = []
|
||||
) -> bool:
|
||||
"""
|
||||
Check if a part exists in a list of parts
|
||||
Handles both camelCase and snake_case key variations (e.g., function_call vs functionCall)
|
||||
"""
|
||||
keys_to_compare = set(part.keys()) - set(excluded_keys)
|
||||
for p in parts:
|
||||
p_keys = set(p.keys())
|
||||
# Check if all keys in part have equivalent values in p
|
||||
match_found = True
|
||||
for key in keys_to_compare:
|
||||
equivalent_key = _get_equivalent_key(key, p_keys)
|
||||
if equivalent_key is None or p.get(equivalent_key, None) != part.get(
|
||||
key, None
|
||||
):
|
||||
match_found = False
|
||||
break
|
||||
|
||||
if match_found:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def _gemini_convert_messages_with_history( # noqa: PLR0915
|
||||
messages: List[AllMessageValues],
|
||||
) -> List[ContentType]:
|
||||
|
|
@ -236,10 +294,33 @@ def _gemini_convert_messages_with_history( # noqa: PLR0915
|
|||
assistant_msg = ChatCompletionAssistantMessage(**msg_dict) # type: ignore
|
||||
_message_content = assistant_msg.get("content", None)
|
||||
reasoning_content = assistant_msg.get("reasoning_content", None)
|
||||
thinking_blocks = assistant_msg.get("thinking_blocks")
|
||||
if reasoning_content is not None:
|
||||
assistant_content.append(
|
||||
PartType(thought=True, text=reasoning_content)
|
||||
)
|
||||
if thinking_blocks is not None:
|
||||
for block in thinking_blocks:
|
||||
block_thinking_str = block.get("thinking")
|
||||
block_signature = block.get("signature")
|
||||
if (
|
||||
block_thinking_str is not None
|
||||
and block_signature is not None
|
||||
):
|
||||
try:
|
||||
assistant_content.append(
|
||||
PartType(
|
||||
thoughtSignature=block_signature,
|
||||
**json.loads(block_thinking_str),
|
||||
)
|
||||
)
|
||||
except Exception:
|
||||
assistant_content.append(
|
||||
PartType(
|
||||
thoughtSignature=block_signature,
|
||||
text=block_thinking_str,
|
||||
)
|
||||
)
|
||||
if _message_content is not None and isinstance(_message_content, list):
|
||||
_parts = []
|
||||
for element in _message_content:
|
||||
|
|
@ -262,9 +343,17 @@ def _gemini_convert_messages_with_history( # noqa: PLR0915
|
|||
assistant_msg.get("tool_calls", []) is not None
|
||||
or assistant_msg.get("function_call") is not None
|
||||
): # support assistant tool invoke conversion
|
||||
assistant_content.extend(
|
||||
convert_to_gemini_tool_call_invoke(assistant_msg)
|
||||
gemini_tool_call_parts = convert_to_gemini_tool_call_invoke(
|
||||
assistant_msg
|
||||
)
|
||||
## check if gemini_tool_call already exists in assistant_content
|
||||
for gemini_tool_call_part in gemini_tool_call_parts:
|
||||
if not check_if_part_exists_in_parts(
|
||||
assistant_content,
|
||||
gemini_tool_call_part,
|
||||
excluded_keys=["thoughtSignature"],
|
||||
):
|
||||
assistant_content.append(gemini_tool_call_part)
|
||||
last_message_with_tool_calls = assistant_msg
|
||||
|
||||
msg_i += 1
|
||||
|
|
@ -476,6 +565,7 @@ async def async_transform_request_body(
|
|||
optional_params=optional_params,
|
||||
)
|
||||
|
||||
|
||||
def _default_user_message_when_system_message_passed() -> ChatCompletionUserMessage:
|
||||
"""
|
||||
Returns a default user message when a "system" message is passed in gemini fails.
|
||||
|
|
@ -484,6 +574,7 @@ def _default_user_message_when_system_message_passed() -> ChatCompletionUserMess
|
|||
"""
|
||||
return ChatCompletionUserMessage(content=".", role="user")
|
||||
|
||||
|
||||
def _transform_system_message(
|
||||
supports_system_message: bool, messages: List[AllMessageValues]
|
||||
) -> Tuple[Optional[SystemInstructions], List[AllMessageValues]]:
|
||||
|
|
|
|||
|
|
@ -43,6 +43,7 @@ from litellm.types.llms.gemini import BidiGenerateContentServerMessage
|
|||
from litellm.types.llms.openai import (
|
||||
AllMessageValues,
|
||||
ChatCompletionResponseMessage,
|
||||
ChatCompletionThinkingBlock,
|
||||
ChatCompletionToolCallChunk,
|
||||
ChatCompletionToolCallFunctionChunk,
|
||||
ChatCompletionToolParamFunctionChunk,
|
||||
|
|
@ -792,7 +793,25 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
|
|||
content_str += _content_str
|
||||
|
||||
return content_str, reasoning_content_str
|
||||
|
||||
|
||||
def _extract_thinking_blocks_from_parts(
|
||||
self, parts: List[HttpxPartType]
|
||||
) -> List[ChatCompletionThinkingBlock]:
|
||||
"""Extract thinking blocks from parts if present"""
|
||||
thinking_blocks: List[ChatCompletionThinkingBlock] = []
|
||||
for part in parts:
|
||||
if "thoughtSignature" in part:
|
||||
part_copy = part.copy()
|
||||
part_copy.pop("thoughtSignature")
|
||||
thinking_blocks.append(
|
||||
ChatCompletionThinkingBlock(
|
||||
type="thinking",
|
||||
thinking=json.dumps(part_copy),
|
||||
signature=part["thoughtSignature"],
|
||||
)
|
||||
)
|
||||
return thinking_blocks
|
||||
|
||||
def _extract_image_response_from_parts(
|
||||
self, parts: List[HttpxPartType]
|
||||
) -> Optional[ImageURLObject]:
|
||||
|
|
@ -804,10 +823,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
|
|||
if mime_type.startswith("image/"):
|
||||
# Convert base64 data to data URI format
|
||||
data_uri = f"data:{mime_type};base64,{data}"
|
||||
return ImageURLObject(
|
||||
url=data_uri,
|
||||
detail="auto"
|
||||
)
|
||||
return ImageURLObject(url=data_uri, detail="auto")
|
||||
return None
|
||||
|
||||
def _extract_audio_response_from_parts(
|
||||
|
|
@ -1127,7 +1143,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
|
|||
elif web_search_queries:
|
||||
web_search_requests = len(grounding_metadata)
|
||||
return web_search_requests
|
||||
|
||||
|
||||
@staticmethod
|
||||
def _create_streaming_choice(
|
||||
chat_completion_message: ChatCompletionResponseMessage,
|
||||
|
|
@ -1151,9 +1167,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
|
|||
index=candidate.get("index", idx),
|
||||
delta=Delta(
|
||||
content=chat_completion_message.get("content"),
|
||||
reasoning_content=chat_completion_message.get(
|
||||
"reasoning_content"
|
||||
),
|
||||
reasoning_content=chat_completion_message.get("reasoning_content"),
|
||||
tool_calls=tools,
|
||||
image=image_response,
|
||||
function_call=functions,
|
||||
|
|
@ -1164,13 +1178,15 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
|
|||
return choice
|
||||
|
||||
@staticmethod
|
||||
def _extract_candidate_metadata(candidate: Candidates) -> Tuple[List[dict], List[dict], List, List]:
|
||||
def _extract_candidate_metadata(
|
||||
candidate: Candidates,
|
||||
) -> Tuple[List[dict], List[dict], List, List]:
|
||||
"""
|
||||
Extract metadata from a single candidate response.
|
||||
|
||||
|
||||
Returns:
|
||||
grounding_metadata: List[dict]
|
||||
url_context_metadata: List[dict]
|
||||
url_context_metadata: List[dict]
|
||||
safety_ratings: List
|
||||
citation_metadata: List
|
||||
"""
|
||||
|
|
@ -1178,7 +1194,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
|
|||
url_context_metadata: List[dict] = []
|
||||
safety_ratings: List = []
|
||||
citation_metadata: List = []
|
||||
|
||||
|
||||
if "groundingMetadata" in candidate:
|
||||
if isinstance(candidate["groundingMetadata"], list):
|
||||
grounding_metadata.extend(candidate["groundingMetadata"]) # type: ignore
|
||||
|
|
@ -1194,8 +1210,13 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
|
|||
if "urlContextMetadata" in candidate:
|
||||
# Add URL context metadata to grounding metadata
|
||||
url_context_metadata.append(cast(dict, candidate["urlContextMetadata"]))
|
||||
|
||||
return grounding_metadata, url_context_metadata, safety_ratings, citation_metadata
|
||||
|
||||
return (
|
||||
grounding_metadata,
|
||||
url_context_metadata,
|
||||
safety_ratings,
|
||||
citation_metadata,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _process_candidates(
|
||||
|
|
@ -1227,6 +1248,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
|
|||
tools: Optional[List[ChatCompletionToolCallChunk]] = []
|
||||
functions: Optional[ChatCompletionToolCallFunctionChunk] = None
|
||||
cumulative_tool_call_index: int = 0
|
||||
thinking_blocks: Optional[List[ChatCompletionThinkingBlock]] = None
|
||||
|
||||
for idx, candidate in enumerate(_candidates):
|
||||
if "content" not in candidate:
|
||||
|
|
@ -1239,7 +1261,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
|
|||
candidate_safety_ratings,
|
||||
candidate_citation_metadata,
|
||||
) = VertexGeminiConfig._extract_candidate_metadata(candidate)
|
||||
|
||||
|
||||
grounding_metadata.extend(candidate_grounding_metadata)
|
||||
url_context_metadata.extend(candidate_url_context_metadata)
|
||||
safety_ratings.extend(candidate_safety_ratings)
|
||||
|
|
@ -1264,6 +1286,12 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
|
|||
)
|
||||
)
|
||||
|
||||
thinking_blocks = (
|
||||
VertexGeminiConfig()._extract_thinking_blocks_from_parts(
|
||||
parts=candidate["content"]["parts"]
|
||||
)
|
||||
)
|
||||
|
||||
if audio_response is not None:
|
||||
cast(Dict[str, Any], chat_completion_message)[
|
||||
"audio"
|
||||
|
|
@ -1271,7 +1299,9 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
|
|||
chat_completion_message["content"] = None # OpenAI spec
|
||||
if image_response is not None:
|
||||
# Handle image response - combine with text content into structured format
|
||||
cast(Dict[str, Any], chat_completion_message)["image"] = image_response
|
||||
cast(Dict[str, Any], chat_completion_message)[
|
||||
"image"
|
||||
] = image_response
|
||||
if content is not None:
|
||||
chat_completion_message["content"] = content
|
||||
|
||||
|
|
@ -1298,15 +1328,18 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
|
|||
if functions is not None:
|
||||
chat_completion_message["function_call"] = functions
|
||||
|
||||
if thinking_blocks is not None:
|
||||
chat_completion_message["thinking_blocks"] = thinking_blocks # type: ignore
|
||||
|
||||
if isinstance(model_response, ModelResponseStream):
|
||||
choice = VertexGeminiConfig._create_streaming_choice(
|
||||
chat_completion_message=chat_completion_message,
|
||||
candidate=candidate,
|
||||
idx=idx,
|
||||
tools=tools,
|
||||
functions=functions,
|
||||
candidate=candidate,
|
||||
idx=idx,
|
||||
tools=tools,
|
||||
functions=functions,
|
||||
chat_completion_logprobs=chat_completion_logprobs,
|
||||
image_response=image_response
|
||||
image_response=image_response,
|
||||
)
|
||||
model_response.choices.append(choice)
|
||||
elif isinstance(model_response, ModelResponse):
|
||||
|
|
|
|||
54
litellm/llms/xai/cost_calculator.py
Normal file
54
litellm/llms/xai/cost_calculator.py
Normal file
|
|
@ -0,0 +1,54 @@
|
|||
"""
|
||||
Helper util for handling XAI-specific cost calculation
|
||||
- e.g.: reasoning tokens for grok models
|
||||
"""
|
||||
|
||||
from typing import Tuple, Union
|
||||
|
||||
from litellm.types.utils import Usage
|
||||
from litellm.utils import get_model_info
|
||||
|
||||
|
||||
def cost_per_token(model: str, usage: Usage) -> Tuple[float, float]:
|
||||
"""
|
||||
Calculates the cost per token for a given XAI model, prompt tokens, and completion tokens.
|
||||
|
||||
Input:
|
||||
- model: str, the model name without provider prefix
|
||||
- usage: LiteLLM Usage block, containing XAI-specific usage information
|
||||
|
||||
Returns:
|
||||
Tuple[float, float] - prompt_cost_in_usd, completion_cost_in_usd
|
||||
"""
|
||||
## GET MODEL INFO
|
||||
model_info = get_model_info(model=model, custom_llm_provider="xai")
|
||||
|
||||
def _safe_float_cast(
|
||||
value: Union[str, int, float, None, object], default: float = 0.0
|
||||
) -> float:
|
||||
"""Safely cast a value to float with proper type handling for mypy."""
|
||||
if value is None:
|
||||
return default
|
||||
try:
|
||||
return float(value) # type: ignore
|
||||
except (ValueError, TypeError):
|
||||
return default
|
||||
|
||||
## CALCULATE INPUT COST
|
||||
input_cost_per_token = _safe_float_cast(model_info.get("input_cost_per_token"))
|
||||
prompt_cost: float = (usage.prompt_tokens or 0) * input_cost_per_token
|
||||
|
||||
## CALCULATE OUTPUT COST
|
||||
output_cost_per_token = _safe_float_cast(model_info.get("output_cost_per_token"))
|
||||
|
||||
# For XAI models, completion is billed as (visible completion tokens + reasoning tokens)
|
||||
completion_tokens = int(getattr(usage, "completion_tokens", 0) or 0)
|
||||
reasoning_tokens = 0
|
||||
if hasattr(usage, "completion_tokens_details") and usage.completion_tokens_details:
|
||||
reasoning_tokens = int(
|
||||
getattr(usage.completion_tokens_details, "reasoning_tokens", 0) or 0
|
||||
)
|
||||
|
||||
completion_cost = (completion_tokens + reasoning_tokens) * output_cost_per_token
|
||||
|
||||
return prompt_cost, completion_cost
|
||||
|
|
@ -5817,16 +5817,6 @@
|
|||
"supports_response_schema": true,
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
"groq/llama3-8b-8192": {
|
||||
"max_tokens": 8192,
|
||||
"max_input_tokens": 8192,
|
||||
"max_output_tokens": 8192,
|
||||
"input_cost_per_token": 5e-08,
|
||||
"output_cost_per_token": 8e-08,
|
||||
"litellm_provider": "groq",
|
||||
"mode": "chat",
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
"groq/llama-3.2-1b-preview": {
|
||||
"max_tokens": 8192,
|
||||
"max_input_tokens": 8192,
|
||||
|
|
@ -5907,17 +5897,6 @@
|
|||
"supports_tool_choice": true,
|
||||
"deprecation_date": "2025-04-14"
|
||||
},
|
||||
"groq/llama3-70b-8192": {
|
||||
"max_tokens": 8192,
|
||||
"max_input_tokens": 8192,
|
||||
"max_output_tokens": 8192,
|
||||
"input_cost_per_token": 5.9e-07,
|
||||
"output_cost_per_token": 7.9e-07,
|
||||
"litellm_provider": "groq",
|
||||
"mode": "chat",
|
||||
"supports_response_schema": true,
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
"groq/llama-3.1-8b-instant": {
|
||||
"max_tokens": 8192,
|
||||
"max_input_tokens": 128000,
|
||||
|
|
@ -11991,6 +11970,108 @@
|
|||
"mode": "chat",
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
"openrouter/openai/gpt-4.1": {
|
||||
"max_tokens": 32768,
|
||||
"max_input_tokens": 1047576,
|
||||
"max_output_tokens": 32768,
|
||||
"input_cost_per_token": 2e-06,
|
||||
"output_cost_per_token": 8e-06,
|
||||
"cache_read_input_token_cost": 5e-07,
|
||||
"litellm_provider": "openrouter",
|
||||
"mode": "chat",
|
||||
"supports_function_calling": true,
|
||||
"supports_parallel_function_calling": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_vision": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_system_messages": true,
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
"openrouter/openai/gpt-4.1-2025-04-14": {
|
||||
"max_tokens": 32768,
|
||||
"max_input_tokens": 1047576,
|
||||
"max_output_tokens": 32768,
|
||||
"input_cost_per_token": 2e-06,
|
||||
"output_cost_per_token": 8e-06,
|
||||
"cache_read_input_token_cost": 5e-07,
|
||||
"litellm_provider": "openrouter",
|
||||
"mode": "chat",
|
||||
"supports_function_calling": true,
|
||||
"supports_parallel_function_calling": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_vision": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_system_messages": true,
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
"openrouter/openai/gpt-4.1-mini": {
|
||||
"max_tokens": 32768,
|
||||
"max_input_tokens": 1047576,
|
||||
"max_output_tokens": 32768,
|
||||
"input_cost_per_token": 4e-07,
|
||||
"output_cost_per_token": 1.6e-06,
|
||||
"cache_read_input_token_cost": 1e-07,
|
||||
"litellm_provider": "openrouter",
|
||||
"mode": "chat",
|
||||
"supports_function_calling": true,
|
||||
"supports_parallel_function_calling": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_vision": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_system_messages": true,
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
"openrouter/openai/gpt-4.1-mini-2025-04-14": {
|
||||
"max_tokens": 32768,
|
||||
"max_input_tokens": 1047576,
|
||||
"max_output_tokens": 32768,
|
||||
"input_cost_per_token": 4e-07,
|
||||
"output_cost_per_token": 1.6e-06,
|
||||
"cache_read_input_token_cost": 1e-07,
|
||||
"litellm_provider": "openrouter",
|
||||
"mode": "chat",
|
||||
"supports_function_calling": true,
|
||||
"supports_parallel_function_calling": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_vision": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_system_messages": true,
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
"openrouter/openai/gpt-4.1-nano": {
|
||||
"max_tokens": 32768,
|
||||
"max_input_tokens": 1047576,
|
||||
"max_output_tokens": 32768,
|
||||
"input_cost_per_token": 1e-07,
|
||||
"output_cost_per_token": 4e-07,
|
||||
"cache_read_input_token_cost": 2.5e-08,
|
||||
"litellm_provider": "openrouter",
|
||||
"mode": "chat",
|
||||
"supports_function_calling": true,
|
||||
"supports_parallel_function_calling": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_vision": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_system_messages": true,
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
"openrouter/openai/gpt-4.1-nano-2025-04-14": {
|
||||
"max_tokens": 32768,
|
||||
"max_input_tokens": 1047576,
|
||||
"max_output_tokens": 32768,
|
||||
"input_cost_per_token": 1e-07,
|
||||
"output_cost_per_token": 4e-07,
|
||||
"cache_read_input_token_cost": 2.5e-08,
|
||||
"litellm_provider": "openrouter",
|
||||
"mode": "chat",
|
||||
"supports_function_calling": true,
|
||||
"supports_parallel_function_calling": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_vision": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_system_messages": true,
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
"openrouter/openai/gpt-5-mini": {
|
||||
"max_tokens": 128000,
|
||||
"max_input_tokens": 400000,
|
||||
|
|
@ -14970,10 +15051,10 @@
|
|||
"output_cost_per_token": 6e-06,
|
||||
"max_input_tokens": 262000,
|
||||
"litellm_provider": "together_ai",
|
||||
"supports_function_calling": false,
|
||||
"supports_parallel_function_calling": false,
|
||||
"supports_function_calling": true,
|
||||
"supports_parallel_function_calling": true,
|
||||
"mode": "chat",
|
||||
"supports_tool_choice": false,
|
||||
"supports_tool_choice": true,
|
||||
"source": "https://www.together.ai/models/qwen3-235b-a22b-instruct-2507-fp8"
|
||||
},
|
||||
"together_ai/Qwen/Qwen3-Coder-480B-A35B-Instruct-FP8": {
|
||||
|
|
@ -14981,10 +15062,10 @@
|
|||
"output_cost_per_token": 2e-06,
|
||||
"max_input_tokens": 256000,
|
||||
"litellm_provider": "together_ai",
|
||||
"supports_function_calling": false,
|
||||
"supports_parallel_function_calling": false,
|
||||
"supports_function_calling": true,
|
||||
"supports_parallel_function_calling": true,
|
||||
"mode": "chat",
|
||||
"supports_tool_choice": false,
|
||||
"supports_tool_choice": true,
|
||||
"source": "https://www.together.ai/models/qwen3-coder-480b-a35b-instruct"
|
||||
},
|
||||
"together_ai/Qwen/Qwen3-235B-A22B-Thinking-2507": {
|
||||
|
|
@ -14992,10 +15073,10 @@
|
|||
"output_cost_per_token": 3e-06,
|
||||
"max_input_tokens": 256000,
|
||||
"litellm_provider": "together_ai",
|
||||
"supports_function_calling": false,
|
||||
"supports_parallel_function_calling": false,
|
||||
"supports_function_calling": true,
|
||||
"supports_parallel_function_calling": true,
|
||||
"mode": "chat",
|
||||
"supports_tool_choice": false,
|
||||
"supports_tool_choice": true,
|
||||
"source": "https://www.together.ai/models/qwen3-235b-a22b-thinking-2507"
|
||||
},
|
||||
"together_ai/Qwen/Qwen3-235B-A22B-fp8-tput": {
|
||||
|
|
@ -15038,10 +15119,10 @@
|
|||
"output_cost_per_token": 2.19e-06,
|
||||
"max_input_tokens": 128000,
|
||||
"litellm_provider": "together_ai",
|
||||
"supports_function_calling": false,
|
||||
"supports_parallel_function_calling": false,
|
||||
"supports_function_calling": true,
|
||||
"supports_parallel_function_calling": true,
|
||||
"mode": "chat",
|
||||
"supports_tool_choice": false,
|
||||
"supports_tool_choice": true,
|
||||
"source": "https://www.together.ai/models/deepseek-r1-0528-throughput"
|
||||
},
|
||||
"together_ai/mistralai/Mistral-Small-24B-Instruct-2501": {
|
||||
|
|
@ -15066,9 +15147,9 @@
|
|||
"output_cost_per_token": 6e-07,
|
||||
"max_input_tokens": 128000,
|
||||
"litellm_provider": "together_ai",
|
||||
"supports_function_calling": false,
|
||||
"supports_tool_choice": false,
|
||||
"supports_parallel_function_calling": false,
|
||||
"supports_function_calling": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_parallel_function_calling": true,
|
||||
"mode": "chat",
|
||||
"source": "https://www.together.ai/models/gpt-oss-120b"
|
||||
},
|
||||
|
|
@ -15077,9 +15158,9 @@
|
|||
"output_cost_per_token": 2e-07,
|
||||
"max_input_tokens": 128000,
|
||||
"litellm_provider": "together_ai",
|
||||
"supports_function_calling": false,
|
||||
"supports_tool_choice": false,
|
||||
"supports_parallel_function_calling": false,
|
||||
"supports_function_calling": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_parallel_function_calling": true,
|
||||
"mode": "chat",
|
||||
"source": "https://www.together.ai/models/gpt-oss-20b"
|
||||
},
|
||||
|
|
@ -15088,12 +15169,24 @@
|
|||
"output_cost_per_token": 1.1e-06,
|
||||
"max_input_tokens": 128000,
|
||||
"litellm_provider": "together_ai",
|
||||
"supports_function_calling": false,
|
||||
"supports_tool_choice": false,
|
||||
"supports_parallel_function_calling": false,
|
||||
"supports_function_calling": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_parallel_function_calling": true,
|
||||
"mode": "chat",
|
||||
"source": "https://www.together.ai/models/glm-4-5-air"
|
||||
},
|
||||
"together_ai/deepseek-ai/DeepSeek-V3.1": {
|
||||
"input_cost_per_token": 0.6e-06,
|
||||
"output_cost_per_token": 1.7e-06,
|
||||
"max_tokens": 128000,
|
||||
"litellm_provider": "together_ai",
|
||||
"supports_function_calling": true,
|
||||
"supports_parallel_function_calling": true,
|
||||
"supports_reasoning": true,
|
||||
"mode": "chat",
|
||||
"supports_tool_choice": true,
|
||||
"source": "https://www.together.ai/models/deepseek-v3-1"
|
||||
},
|
||||
"ollama/codegemma": {
|
||||
"max_tokens": 8192,
|
||||
"max_input_tokens": 8192,
|
||||
|
|
|
|||
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
|
|
@ -1 +1 @@
|
|||
(self.webpackChunk_N_E=self.webpackChunk_N_E||[]).push([[418],{11790:function(e,n,t){Promise.resolve().then(t.bind(t,52829))},52829:function(e,n,t){"use strict";t.r(n),t.d(n,{default:function(){return f}});var u=t(57437),s=t(2265),c=t(99376),r=t(72162);function f(){let e=(0,c.useSearchParams)().get("key"),[n,t]=(0,s.useState)(null);return(0,s.useEffect)(()=>{e&&t(e)},[e]),(0,u.jsx)(r.Z,{accessToken:n})}}},function(e){e.O(0,[50,487,154,162,971,117,744],function(){return e(e.s=11790)}),_N_E=e.O()}]);
|
||||
(self.webpackChunk_N_E=self.webpackChunk_N_E||[]).push([[418],{11790:function(e,n,t){Promise.resolve().then(t.bind(t,52829))},52829:function(e,n,t){"use strict";t.r(n),t.d(n,{default:function(){return f}});var u=t(57437),s=t(2265),c=t(99376),r=t(72162);function f(){let e=(0,c.useSearchParams)().get("key"),[n,t]=(0,s.useState)(null);return(0,s.useEffect)(()=>{e&&t(e)},[e]),(0,u.jsx)(r.Z,{accessToken:n})}}},function(e){e.O(0,[50,521,154,162,971,117,744],function(){return e(e.s=11790)}),_N_E=e.O()}]);
|
||||
|
|
@ -1 +1 @@
|
|||
(self.webpackChunk_N_E=self.webpackChunk_N_E||[]).push([[25],{58538:function(e,n,u){Promise.resolve().then(u.bind(u,22775))},22775:function(e,n,u){"use strict";u.r(n),u.d(n,{default:function(){return f}});var t=u(57437),s=u(2265),r=u(99376),c=u(36172);function f(){let e=(0,r.useSearchParams)().get("key"),[n,u]=(0,s.useState)(null);return(0,s.useEffect)(()=>{e&&u(e)},[e]),(0,t.jsx)(c.Z,{accessToken:n,publicPage:!0,premiumUser:!1,userRole:null})}}},function(e){e.O(0,[50,487,866,154,162,172,971,117,744],function(){return e(e.s=58538)}),_N_E=e.O()}]);
|
||||
(self.webpackChunk_N_E=self.webpackChunk_N_E||[]).push([[25],{58538:function(e,n,u){Promise.resolve().then(u.bind(u,22775))},22775:function(e,n,u){"use strict";u.r(n),u.d(n,{default:function(){return f}});var t=u(57437),s=u(2265),r=u(99376),c=u(36172);function f(){let e=(0,r.useSearchParams)().get("key"),[n,u]=(0,s.useState)(null);return(0,s.useEffect)(()=>{e&&u(e)},[e]),(0,t.jsx)(c.Z,{accessToken:n,publicPage:!0,premiumUser:!1,userRole:null})}}},function(e){e.O(0,[50,521,866,154,162,172,971,117,744],function(){return e(e.s=58538)}),_N_E=e.O()}]);
|
||||
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
|
|
@ -1,7 +1,7 @@
|
|||
2:I[19107,[],"ClientPageRoot"]
|
||||
3:I[75832,["665","static/chunks/3014691f-b7b79b78e27792f3.js","990","static/chunks/13b76428-ebdf3012af0e4489.js","50","static/chunks/50-d0da2dd7acce2eb9.js","487","static/chunks/487-79ed94231812dae7.js","866","static/chunks/866-9e1803a09e9ae8da.js","642","static/chunks/642-6fb8922b84a60dbe.js","154","static/chunks/154-fff436ed72b19a24.js","162","static/chunks/162-76acc071cdf524f9.js","172","static/chunks/172-0f7049c565983c4d.js","931","static/chunks/app/page-98cba01d87680536.js"],"default",1]
|
||||
3:I[75832,["665","static/chunks/3014691f-b7b79b78e27792f3.js","990","static/chunks/13b76428-ebdf3012af0e4489.js","50","static/chunks/50-fe160ecfa8bc4059.js","521","static/chunks/521-d97d355792d44830.js","866","static/chunks/866-3523e0e07cf314f6.js","220","static/chunks/220-1c8d82f7ce7658c4.js","154","static/chunks/154-fff436ed72b19a24.js","162","static/chunks/162-714ca0ed10a07f66.js","172","static/chunks/172-0f7049c565983c4d.js","931","static/chunks/app/page-8dc8d9524a1f3965.js"],"default",1]
|
||||
4:I[4707,[],""]
|
||||
5:I[36423,[],""]
|
||||
0:["0rzQKhrKTur56aOgaxVrq",[[["",{"children":["__PAGE__",{}]},"$undefined","$undefined",true],["",{"children":["__PAGE__",{},[["$L1",["$","$L2",null,{"props":{"params":{},"searchParams":{}},"Component":"$3"}],null],null],null]},[[[["$","link","0",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/css/31b7f215e119031e.css","precedence":"next","crossOrigin":"$undefined"}],["$","link","1",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/css/8cbc26d5aab3552a.css","precedence":"next","crossOrigin":"$undefined"}]],["$","html",null,{"lang":"en","children":["$","body",null,{"className":"__className_b0dd8a","children":["$","$L4",null,{"parallelRouterKey":"children","segmentPath":["children"],"error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L5",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":"404"}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],"notFoundStyles":[]}]}]}]],null],null],["$L6",null]]]]
|
||||
0:["0GF-OyXnYlAPMWfyPAZSs",[[["",{"children":["__PAGE__",{}]},"$undefined","$undefined",true],["",{"children":["__PAGE__",{},[["$L1",["$","$L2",null,{"props":{"params":{},"searchParams":{}},"Component":"$3"}],null],null],null]},[[[["$","link","0",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/css/31b7f215e119031e.css","precedence":"next","crossOrigin":"$undefined"}],["$","link","1",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/css/060d5ddee53e45ce.css","precedence":"next","crossOrigin":"$undefined"}]],["$","html",null,{"lang":"en","children":["$","body",null,{"className":"__className_b0dd8a","children":["$","$L4",null,{"parallelRouterKey":"children","segmentPath":["children"],"error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L5",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":"404"}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],"notFoundStyles":[]}]}]}]],null],null],["$L6",null]]]]
|
||||
6:[["$","meta","0",{"name":"viewport","content":"width=device-width, initial-scale=1"}],["$","meta","1",{"charSet":"utf-8"}],["$","title","2",{"children":"LiteLLM Dashboard"}],["$","meta","3",{"name":"description","content":"LiteLLM Proxy Admin UI"}],["$","link","4",{"rel":"icon","href":"/favicon.ico","type":"image/x-icon","sizes":"16x16"}],["$","link","5",{"rel":"icon","href":"./favicon.ico"}],["$","meta","6",{"name":"next-size-adjust"}]]
|
||||
1:null
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
2:I[19107,[],"ClientPageRoot"]
|
||||
3:I[52829,["50","static/chunks/50-d0da2dd7acce2eb9.js","487","static/chunks/487-79ed94231812dae7.js","154","static/chunks/154-fff436ed72b19a24.js","162","static/chunks/162-76acc071cdf524f9.js","418","static/chunks/app/model_hub/page-6daa1df7fb2c7a9d.js"],"default",1]
|
||||
3:I[52829,["50","static/chunks/50-fe160ecfa8bc4059.js","521","static/chunks/521-d97d355792d44830.js","154","static/chunks/154-fff436ed72b19a24.js","162","static/chunks/162-714ca0ed10a07f66.js","418","static/chunks/app/model_hub/page-d6e5fb7de2cedde9.js"],"default",1]
|
||||
4:I[4707,[],""]
|
||||
5:I[36423,[],""]
|
||||
0:["0rzQKhrKTur56aOgaxVrq",[[["",{"children":["model_hub",{"children":["__PAGE__",{}]}]},"$undefined","$undefined",true],["",{"children":["model_hub",{"children":["__PAGE__",{},[["$L1",["$","$L2",null,{"props":{"params":{},"searchParams":{}},"Component":"$3"}],null],null],null]},[null,["$","$L4",null,{"parallelRouterKey":"children","segmentPath":["children","model_hub","children"],"error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L5",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":"$undefined","notFoundStyles":"$undefined"}]],null]},[[[["$","link","0",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/css/31b7f215e119031e.css","precedence":"next","crossOrigin":"$undefined"}],["$","link","1",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/css/8cbc26d5aab3552a.css","precedence":"next","crossOrigin":"$undefined"}]],["$","html",null,{"lang":"en","children":["$","body",null,{"className":"__className_b0dd8a","children":["$","$L4",null,{"parallelRouterKey":"children","segmentPath":["children"],"error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L5",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":"404"}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],"notFoundStyles":[]}]}]}]],null],null],["$L6",null]]]]
|
||||
0:["0GF-OyXnYlAPMWfyPAZSs",[[["",{"children":["model_hub",{"children":["__PAGE__",{}]}]},"$undefined","$undefined",true],["",{"children":["model_hub",{"children":["__PAGE__",{},[["$L1",["$","$L2",null,{"props":{"params":{},"searchParams":{}},"Component":"$3"}],null],null],null]},[null,["$","$L4",null,{"parallelRouterKey":"children","segmentPath":["children","model_hub","children"],"error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L5",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":"$undefined","notFoundStyles":"$undefined"}]],null]},[[[["$","link","0",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/css/31b7f215e119031e.css","precedence":"next","crossOrigin":"$undefined"}],["$","link","1",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/css/060d5ddee53e45ce.css","precedence":"next","crossOrigin":"$undefined"}]],["$","html",null,{"lang":"en","children":["$","body",null,{"className":"__className_b0dd8a","children":["$","$L4",null,{"parallelRouterKey":"children","segmentPath":["children"],"error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L5",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":"404"}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],"notFoundStyles":[]}]}]}]],null],null],["$L6",null]]]]
|
||||
6:[["$","meta","0",{"name":"viewport","content":"width=device-width, initial-scale=1"}],["$","meta","1",{"charSet":"utf-8"}],["$","title","2",{"children":"LiteLLM Dashboard"}],["$","meta","3",{"name":"description","content":"LiteLLM Proxy Admin UI"}],["$","link","4",{"rel":"icon","href":"/favicon.ico","type":"image/x-icon","sizes":"16x16"}],["$","link","5",{"rel":"icon","href":"./favicon.ico"}],["$","meta","6",{"name":"next-size-adjust"}]]
|
||||
1:null
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
2:I[19107,[],"ClientPageRoot"]
|
||||
3:I[22775,["50","static/chunks/50-d0da2dd7acce2eb9.js","487","static/chunks/487-79ed94231812dae7.js","866","static/chunks/866-9e1803a09e9ae8da.js","154","static/chunks/154-fff436ed72b19a24.js","162","static/chunks/162-76acc071cdf524f9.js","172","static/chunks/172-0f7049c565983c4d.js","25","static/chunks/app/model_hub_table/page-aef41992dbd4614b.js"],"default",1]
|
||||
3:I[22775,["50","static/chunks/50-fe160ecfa8bc4059.js","521","static/chunks/521-d97d355792d44830.js","866","static/chunks/866-3523e0e07cf314f6.js","154","static/chunks/154-fff436ed72b19a24.js","162","static/chunks/162-714ca0ed10a07f66.js","172","static/chunks/172-0f7049c565983c4d.js","25","static/chunks/app/model_hub_table/page-e06e934de1021ee4.js"],"default",1]
|
||||
4:I[4707,[],""]
|
||||
5:I[36423,[],""]
|
||||
0:["0rzQKhrKTur56aOgaxVrq",[[["",{"children":["model_hub_table",{"children":["__PAGE__",{}]}]},"$undefined","$undefined",true],["",{"children":["model_hub_table",{"children":["__PAGE__",{},[["$L1",["$","$L2",null,{"props":{"params":{},"searchParams":{}},"Component":"$3"}],null],null],null]},[null,["$","$L4",null,{"parallelRouterKey":"children","segmentPath":["children","model_hub_table","children"],"error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L5",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":"$undefined","notFoundStyles":"$undefined"}]],null]},[[[["$","link","0",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/css/31b7f215e119031e.css","precedence":"next","crossOrigin":"$undefined"}],["$","link","1",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/css/8cbc26d5aab3552a.css","precedence":"next","crossOrigin":"$undefined"}]],["$","html",null,{"lang":"en","children":["$","body",null,{"className":"__className_b0dd8a","children":["$","$L4",null,{"parallelRouterKey":"children","segmentPath":["children"],"error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L5",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":"404"}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],"notFoundStyles":[]}]}]}]],null],null],["$L6",null]]]]
|
||||
0:["0GF-OyXnYlAPMWfyPAZSs",[[["",{"children":["model_hub_table",{"children":["__PAGE__",{}]}]},"$undefined","$undefined",true],["",{"children":["model_hub_table",{"children":["__PAGE__",{},[["$L1",["$","$L2",null,{"props":{"params":{},"searchParams":{}},"Component":"$3"}],null],null],null]},[null,["$","$L4",null,{"parallelRouterKey":"children","segmentPath":["children","model_hub_table","children"],"error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L5",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":"$undefined","notFoundStyles":"$undefined"}]],null]},[[[["$","link","0",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/css/31b7f215e119031e.css","precedence":"next","crossOrigin":"$undefined"}],["$","link","1",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/css/060d5ddee53e45ce.css","precedence":"next","crossOrigin":"$undefined"}]],["$","html",null,{"lang":"en","children":["$","body",null,{"className":"__className_b0dd8a","children":["$","$L4",null,{"parallelRouterKey":"children","segmentPath":["children"],"error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L5",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":"404"}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],"notFoundStyles":[]}]}]}]],null],null],["$L6",null]]]]
|
||||
6:[["$","meta","0",{"name":"viewport","content":"width=device-width, initial-scale=1"}],["$","meta","1",{"charSet":"utf-8"}],["$","title","2",{"children":"LiteLLM Dashboard"}],["$","meta","3",{"name":"description","content":"LiteLLM Proxy Admin UI"}],["$","link","4",{"rel":"icon","href":"/favicon.ico","type":"image/x-icon","sizes":"16x16"}],["$","link","5",{"rel":"icon","href":"./favicon.ico"}],["$","meta","6",{"name":"next-size-adjust"}]]
|
||||
1:null
|
||||
|
|
|
|||
File diff suppressed because one or more lines are too long
|
|
@ -1,7 +1,7 @@
|
|||
2:I[19107,[],"ClientPageRoot"]
|
||||
3:I[12011,["665","static/chunks/3014691f-b7b79b78e27792f3.js","50","static/chunks/50-d0da2dd7acce2eb9.js","154","static/chunks/154-fff436ed72b19a24.js","461","static/chunks/app/onboarding/page-3c5840c907b0a5c8.js"],"default",1]
|
||||
3:I[12011,["665","static/chunks/3014691f-b7b79b78e27792f3.js","50","static/chunks/50-fe160ecfa8bc4059.js","154","static/chunks/154-fff436ed72b19a24.js","461","static/chunks/app/onboarding/page-3c5840c907b0a5c8.js"],"default",1]
|
||||
4:I[4707,[],""]
|
||||
5:I[36423,[],""]
|
||||
0:["0rzQKhrKTur56aOgaxVrq",[[["",{"children":["onboarding",{"children":["__PAGE__",{}]}]},"$undefined","$undefined",true],["",{"children":["onboarding",{"children":["__PAGE__",{},[["$L1",["$","$L2",null,{"props":{"params":{},"searchParams":{}},"Component":"$3"}],null],null],null]},[null,["$","$L4",null,{"parallelRouterKey":"children","segmentPath":["children","onboarding","children"],"error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L5",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":"$undefined","notFoundStyles":"$undefined"}]],null]},[[[["$","link","0",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/css/31b7f215e119031e.css","precedence":"next","crossOrigin":"$undefined"}],["$","link","1",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/css/8cbc26d5aab3552a.css","precedence":"next","crossOrigin":"$undefined"}]],["$","html",null,{"lang":"en","children":["$","body",null,{"className":"__className_b0dd8a","children":["$","$L4",null,{"parallelRouterKey":"children","segmentPath":["children"],"error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L5",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":"404"}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],"notFoundStyles":[]}]}]}]],null],null],["$L6",null]]]]
|
||||
0:["0GF-OyXnYlAPMWfyPAZSs",[[["",{"children":["onboarding",{"children":["__PAGE__",{}]}]},"$undefined","$undefined",true],["",{"children":["onboarding",{"children":["__PAGE__",{},[["$L1",["$","$L2",null,{"props":{"params":{},"searchParams":{}},"Component":"$3"}],null],null],null]},[null,["$","$L4",null,{"parallelRouterKey":"children","segmentPath":["children","onboarding","children"],"error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L5",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":"$undefined","notFoundStyles":"$undefined"}]],null]},[[[["$","link","0",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/css/31b7f215e119031e.css","precedence":"next","crossOrigin":"$undefined"}],["$","link","1",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/css/060d5ddee53e45ce.css","precedence":"next","crossOrigin":"$undefined"}]],["$","html",null,{"lang":"en","children":["$","body",null,{"className":"__className_b0dd8a","children":["$","$L4",null,{"parallelRouterKey":"children","segmentPath":["children"],"error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L5",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":"404"}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],"notFoundStyles":[]}]}]}]],null],null],["$L6",null]]]]
|
||||
6:[["$","meta","0",{"name":"viewport","content":"width=device-width, initial-scale=1"}],["$","meta","1",{"charSet":"utf-8"}],["$","title","2",{"children":"LiteLLM Dashboard"}],["$","meta","3",{"name":"description","content":"LiteLLM Proxy Admin UI"}],["$","link","4",{"rel":"icon","href":"/favicon.ico","type":"image/x-icon","sizes":"16x16"}],["$","link","5",{"rel":"icon","href":"./favicon.ico"}],["$","meta","6",{"name":"next-size-adjust"}]]
|
||||
1:null
|
||||
|
|
|
|||
|
|
@ -1,30 +1,27 @@
|
|||
model_list:
|
||||
- model_name: fake-openai-endpoint
|
||||
litellm_params:
|
||||
model: openai/fake
|
||||
api_key: fake-key
|
||||
api_base: https://exampleopenaiendpoint-production.up.railway.app/
|
||||
- model_name: gpt-5-mini
|
||||
litellm_params:
|
||||
model: azure/gpt-5-mini
|
||||
api_base: os.environ/AZURE_GPT_5_MINI_API_BASE # runs os.getenv("AZURE_API_BASE")
|
||||
api_key: os.environ/AZURE_GPT_5_MINI_API_KEY # runs os.getenv("AZURE_API_KEY")
|
||||
stream_timeout: 60
|
||||
merge_reasoning_content_in_choices: true
|
||||
model_info:
|
||||
mode: chat
|
||||
- model_name: fake-openai-endpoint
|
||||
litellm_params:
|
||||
model: openai/fake
|
||||
api_key: fake-key
|
||||
api_base: https://exampleopenaiendpoint-production.up.railway.app/
|
||||
- model_name: gpt-5-mini
|
||||
litellm_params:
|
||||
model: azure/gpt-5-mini
|
||||
api_base: os.environ/AZURE_GPT_5_MINI_API_BASE # runs os.getenv("AZURE_API_BASE")
|
||||
api_key: os.environ/AZURE_GPT_5_MINI_API_KEY # runs os.getenv("AZURE_API_KEY")
|
||||
stream_timeout: 60
|
||||
merge_reasoning_content_in_choices: true
|
||||
model_info:
|
||||
mode: chat
|
||||
- model_name: ollama-deepseek-r1
|
||||
litellm_params:
|
||||
model: ollama/deepseek-r1:1.5b
|
||||
model_info:
|
||||
mode: chat
|
||||
|
||||
router_settings:
|
||||
model_group_alias: {"my-fake-gpt-4": "fake-openai-endpoint"}
|
||||
|
||||
|
||||
litellm_settings:
|
||||
callbacks: ["otel"]
|
||||
cache: true
|
||||
cache_params:
|
||||
type: redis
|
||||
ttl: 600
|
||||
supported_call_types: ["acompletion", "completion"]
|
||||
|
||||
model_group_settings:
|
||||
forward_client_headers_to_llm_api:
|
||||
- fake-openai-endpoint
|
||||
success_callback: ["braintrust"]
|
||||
|
|
|
|||
|
|
@ -2904,6 +2904,7 @@ class LitellmDataForBackendLLMCall(TypedDict, total=False):
|
|||
headers: dict
|
||||
organization: str
|
||||
timeout: Optional[float]
|
||||
stream_timeout: Optional[float]
|
||||
user: Optional[str]
|
||||
num_retries: Optional[int]
|
||||
|
||||
|
|
|
|||
|
|
@ -173,15 +173,24 @@ async def google_count_tokens(request: Request, model_name: str):
|
|||
"""
|
||||
from litellm.proxy.common_utils.http_parsing_utils import _read_request_body
|
||||
from litellm.proxy.proxy_server import token_counter as internal_token_counter
|
||||
from litellm.google_genai.adapters.transformation import GoogleGenAIAdapter
|
||||
|
||||
data = await _read_request_body(request=request)
|
||||
contents = data.get("contents", [])
|
||||
#Create TokenCountRequest for the internal endpoint
|
||||
from litellm.proxy._types import TokenCountRequest
|
||||
|
||||
# Translate contents to openai format messages using the adapter
|
||||
messages = (
|
||||
GoogleGenAIAdapter()
|
||||
.translate_generate_content_to_completion(model_name, contents)
|
||||
.get("messages", [])
|
||||
)
|
||||
|
||||
token_request = TokenCountRequest(
|
||||
model=model_name,
|
||||
contents=contents
|
||||
contents=contents,
|
||||
messages=messages, # compatibility when use openai-like endpoint
|
||||
)
|
||||
|
||||
# Call the internal token counter function with direct request flag set to False
|
||||
|
|
@ -192,11 +201,17 @@ async def google_count_tokens(request: Request, model_name: str):
|
|||
if token_response is not None:
|
||||
# cast the response to the well known format
|
||||
original_response: dict = token_response.original_response or {}
|
||||
return TokenCountDetailsResponse(
|
||||
totalTokens=original_response.get("totalTokens", 0),
|
||||
promptTokensDetails=original_response.get("promptTokensDetails", []),
|
||||
)
|
||||
|
||||
if original_response:
|
||||
return TokenCountDetailsResponse(
|
||||
totalTokens=original_response.get("totalTokens", 0),
|
||||
promptTokensDetails=original_response.get("promptTokensDetails", []),
|
||||
)
|
||||
else:
|
||||
return TokenCountDetailsResponse(
|
||||
totalTokens=token_response.total_tokens or 0,
|
||||
promptTokensDetails=[],
|
||||
)
|
||||
|
||||
#########################################################
|
||||
# Return the response in the well known format
|
||||
#########################################################
|
||||
|
|
|
|||
|
|
@ -271,6 +271,16 @@ class LiteLLMProxyRequestSetup:
|
|||
if timeout_header is not None:
|
||||
return float(timeout_header)
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def _get_stream_timeout_from_request(headers: dict) -> Optional[float]:
|
||||
"""
|
||||
Get the `stream_timeout` from the request headers.
|
||||
"""
|
||||
stream_timeout_header = headers.get("x-litellm-stream-timeout", None)
|
||||
if stream_timeout_header is not None:
|
||||
return float(stream_timeout_header)
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def _get_num_retries_from_request(headers: dict) -> Optional[int]:
|
||||
|
|
@ -439,6 +449,10 @@ class LiteLLMProxyRequestSetup:
|
|||
timeout = LiteLLMProxyRequestSetup._get_timeout_from_request(headers)
|
||||
if timeout is not None:
|
||||
data["timeout"] = timeout
|
||||
|
||||
stream_timeout = LiteLLMProxyRequestSetup._get_stream_timeout_from_request(headers)
|
||||
if stream_timeout is not None:
|
||||
data["stream_timeout"] = stream_timeout
|
||||
|
||||
num_retries = LiteLLMProxyRequestSetup._get_num_retries_from_request(headers)
|
||||
if num_retries is not None:
|
||||
|
|
|
|||
|
|
@ -12,6 +12,8 @@ import click
|
|||
import httpx
|
||||
from dotenv import load_dotenv
|
||||
|
||||
from litellm.constants import DEFAULT_NUM_WORKERS_LITELLM_PROXY
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from fastapi import FastAPI
|
||||
else:
|
||||
|
|
@ -308,8 +310,8 @@ class ProxyInitializationHelpers:
|
|||
@click.option("--port", default=4000, help="Port to bind the server to.", envvar="PORT")
|
||||
@click.option(
|
||||
"--num_workers",
|
||||
default=1,
|
||||
help="Number of uvicorn / gunicorn workers to spin up. By default, 1 uvicorn is used.",
|
||||
default=DEFAULT_NUM_WORKERS_LITELLM_PROXY,
|
||||
help="Number of uvicorn / gunicorn workers to spin up. By default, 4 uvicorn workers are used.",
|
||||
envvar="NUM_WORKERS",
|
||||
)
|
||||
@click.option("--api_base", default=None, help="API base URL.")
|
||||
|
|
|
|||
|
|
@ -1,4 +1,6 @@
|
|||
model_list:
|
||||
- model_name: xai/*
|
||||
- model_name: db-openai-endpoint
|
||||
litellm_params:
|
||||
model: xai/*
|
||||
model: openai/*
|
||||
api_base: https://exampleopenaiendpoint-production-0ee2.up.railway.app/
|
||||
mock_response: "hi"
|
||||
|
|
|
|||
|
|
@ -154,6 +154,7 @@ class UserAPIKeyLabelNames(Enum):
|
|||
|
||||
DEFINED_PROMETHEUS_METRICS = Literal[
|
||||
"litellm_llm_api_latency_metric",
|
||||
"litellm_llm_api_time_to_first_token_metric",
|
||||
"litellm_request_total_latency_metric",
|
||||
"litellm_overhead_latency_metric",
|
||||
"litellm_remaining_requests_metric",
|
||||
|
|
@ -162,6 +163,7 @@ DEFINED_PROMETHEUS_METRICS = Literal[
|
|||
"litellm_proxy_failed_requests_metric",
|
||||
"litellm_deployment_latency_per_output_token",
|
||||
"litellm_requests_metric",
|
||||
"litellm_spend_metric",
|
||||
"litellm_total_tokens_metric",
|
||||
"litellm_input_tokens_metric",
|
||||
"litellm_output_tokens_metric",
|
||||
|
|
@ -173,9 +175,11 @@ DEFINED_PROMETHEUS_METRICS = Literal[
|
|||
"litellm_remaining_api_key_budget_metric",
|
||||
"litellm_api_key_max_budget_metric",
|
||||
"litellm_api_key_budget_remaining_hours_metric",
|
||||
"litellm_deployment_state",
|
||||
"litellm_deployment_failure_responses",
|
||||
"litellm_deployment_total_requests",
|
||||
"litellm_deployment_success_responses",
|
||||
"litellm_deployment_cooled_down",
|
||||
"litellm_pod_lock_manager_size",
|
||||
"litellm_in_memory_daily_spend_update_queue_size",
|
||||
"litellm_redis_daily_spend_update_queue_size",
|
||||
|
|
@ -196,6 +200,14 @@ class PrometheusMetricLabels:
|
|||
UserAPIKeyLabelNames.USER.value,
|
||||
]
|
||||
|
||||
litellm_llm_api_time_to_first_token_metric = [
|
||||
UserAPIKeyLabelNames.v1_LITELLM_MODEL_NAME.value,
|
||||
UserAPIKeyLabelNames.API_KEY_HASH.value,
|
||||
UserAPIKeyLabelNames.API_KEY_ALIAS.value,
|
||||
UserAPIKeyLabelNames.TEAM.value,
|
||||
UserAPIKeyLabelNames.TEAM_ALIAS.value,
|
||||
]
|
||||
|
||||
litellm_request_total_latency_metric = [
|
||||
UserAPIKeyLabelNames.END_USER.value,
|
||||
UserAPIKeyLabelNames.API_KEY_HASH.value,
|
||||
|
|
@ -282,6 +294,16 @@ class PrometheusMetricLabels:
|
|||
UserAPIKeyLabelNames.USER_EMAIL.value,
|
||||
]
|
||||
|
||||
litellm_spend_metric = [
|
||||
UserAPIKeyLabelNames.END_USER.value,
|
||||
UserAPIKeyLabelNames.API_KEY_HASH.value,
|
||||
UserAPIKeyLabelNames.API_KEY_ALIAS.value,
|
||||
UserAPIKeyLabelNames.v1_LITELLM_MODEL_NAME.value,
|
||||
UserAPIKeyLabelNames.TEAM.value,
|
||||
UserAPIKeyLabelNames.TEAM_ALIAS.value,
|
||||
UserAPIKeyLabelNames.USER.value,
|
||||
]
|
||||
|
||||
litellm_input_tokens_metric = [
|
||||
UserAPIKeyLabelNames.END_USER.value,
|
||||
UserAPIKeyLabelNames.API_KEY_HASH.value,
|
||||
|
|
@ -315,6 +337,20 @@ class PrometheusMetricLabels:
|
|||
UserAPIKeyLabelNames.REQUESTED_MODEL.value,
|
||||
]
|
||||
|
||||
litellm_deployment_state = [
|
||||
UserAPIKeyLabelNames.v2_LITELLM_MODEL_NAME.value,
|
||||
UserAPIKeyLabelNames.MODEL_ID.value,
|
||||
UserAPIKeyLabelNames.API_BASE.value,
|
||||
UserAPIKeyLabelNames.API_PROVIDER.value,
|
||||
]
|
||||
|
||||
litellm_deployment_cooled_down = [
|
||||
UserAPIKeyLabelNames.v2_LITELLM_MODEL_NAME.value,
|
||||
UserAPIKeyLabelNames.MODEL_ID.value,
|
||||
UserAPIKeyLabelNames.API_BASE.value,
|
||||
UserAPIKeyLabelNames.API_PROVIDER.value,
|
||||
]
|
||||
|
||||
litellm_deployment_successful_fallbacks = [
|
||||
UserAPIKeyLabelNames.REQUESTED_MODEL.value,
|
||||
UserAPIKeyLabelNames.FALLBACK_MODEL.value,
|
||||
|
|
|
|||
|
|
@ -43,10 +43,14 @@ from openai.types.responses.response import (
|
|||
|
||||
# Handle OpenAI SDK version compatibility for Text type
|
||||
try:
|
||||
from openai.types.responses.response_create_params import Text as ResponseText
|
||||
from openai.types.responses.response_create_params import (
|
||||
Text as ResponseText, # type: ignore
|
||||
)
|
||||
except (ImportError, AttributeError):
|
||||
# Fall back to the concrete config type available in all SDK versions
|
||||
from openai.types.responses.response_text_config_param import ResponseTextConfigParam as ResponseText
|
||||
from openai.types.responses.response_text_config_param import (
|
||||
ResponseTextConfigParam as ResponseText,
|
||||
)
|
||||
|
||||
from openai.types.responses.response_create_params import (
|
||||
Reasoning,
|
||||
|
|
@ -1025,29 +1029,29 @@ class ResponseAPIUsage(BaseLiteLLMOpenAIResponseObject):
|
|||
class ResponsesAPIResponse(BaseLiteLLMOpenAIResponseObject):
|
||||
id: str
|
||||
created_at: int
|
||||
error: Optional[dict]
|
||||
incomplete_details: Optional[IncompleteDetails]
|
||||
instructions: Optional[str]
|
||||
metadata: Optional[Dict]
|
||||
model: Optional[str]
|
||||
object: Optional[str]
|
||||
error: Optional[dict] = None
|
||||
incomplete_details: Optional[IncompleteDetails] = None
|
||||
instructions: Optional[str] = None
|
||||
metadata: Optional[Dict] = None
|
||||
model: Optional[str] = None
|
||||
object: Optional[str] = None
|
||||
output: Union[
|
||||
List[Union[ResponseOutputItem, Dict]],
|
||||
List[Union[GenericResponseOutputItem, OutputFunctionToolCall]],
|
||||
]
|
||||
parallel_tool_calls: bool
|
||||
temperature: Optional[float]
|
||||
temperature: Optional[float] = None
|
||||
tool_choice: ToolChoice
|
||||
tools: Union[List[Tool], List[ResponseFunctionToolCall], List[Dict[str, Any]]]
|
||||
top_p: Optional[float]
|
||||
max_output_tokens: Optional[int]
|
||||
previous_response_id: Optional[str]
|
||||
reasoning: Optional[Reasoning]
|
||||
status: Optional[str]
|
||||
text: Optional[Union["ResponseText", Dict[str, Any]]]
|
||||
truncation: Optional[Literal["auto", "disabled"]]
|
||||
usage: Optional[ResponseAPIUsage]
|
||||
user: Optional[str]
|
||||
max_output_tokens: Optional[int] = None
|
||||
previous_response_id: Optional[str] = None
|
||||
reasoning: Optional[Reasoning] = None
|
||||
status: Optional[str] = None
|
||||
text: Optional[Union["ResponseText", Dict[str, Any]]] = None
|
||||
truncation: Optional[Literal["auto", "disabled"]] = None
|
||||
usage: Optional[ResponseAPIUsage] = None
|
||||
user: Optional[str] = None
|
||||
store: Optional[bool] = None
|
||||
# Define private attributes using PrivateAttr
|
||||
_hidden_params: dict = PrivateAttr(default_factory=dict)
|
||||
|
|
|
|||
|
|
@ -41,6 +41,7 @@ class PartType(TypedDict, total=False):
|
|||
function_call: FunctionCall
|
||||
function_response: FunctionResponse
|
||||
thought: bool
|
||||
thoughtSignature: str
|
||||
|
||||
|
||||
class HttpxFunctionCall(TypedDict):
|
||||
|
|
@ -72,6 +73,7 @@ class HttpxPartType(TypedDict, total=False):
|
|||
executableCode: HttpxExecutableCode
|
||||
codeExecutionResult: HttpxCodeExecutionResult
|
||||
thought: bool
|
||||
thoughtSignature: str
|
||||
|
||||
|
||||
class HttpxContentType(TypedDict, total=False):
|
||||
|
|
@ -245,10 +247,11 @@ class UsageMetadata(TypedDict, total=False):
|
|||
class TokenCountDetailsResponse(TypedDict):
|
||||
"""
|
||||
Response structure for token count details with modality breakdown.
|
||||
|
||||
|
||||
Example:
|
||||
{'totalTokens': 12, 'promptTokensDetails': [{'modality': 'TEXT', 'tokenCount': 12}]}
|
||||
"""
|
||||
|
||||
totalTokens: int
|
||||
promptTokensDetails: List[PromptTokensDetails]
|
||||
|
||||
|
|
|
|||
|
|
@ -5817,16 +5817,6 @@
|
|||
"supports_response_schema": true,
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
"groq/llama3-8b-8192": {
|
||||
"max_tokens": 8192,
|
||||
"max_input_tokens": 8192,
|
||||
"max_output_tokens": 8192,
|
||||
"input_cost_per_token": 5e-08,
|
||||
"output_cost_per_token": 8e-08,
|
||||
"litellm_provider": "groq",
|
||||
"mode": "chat",
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
"groq/llama-3.2-1b-preview": {
|
||||
"max_tokens": 8192,
|
||||
"max_input_tokens": 8192,
|
||||
|
|
@ -5907,17 +5897,6 @@
|
|||
"supports_tool_choice": true,
|
||||
"deprecation_date": "2025-04-14"
|
||||
},
|
||||
"groq/llama3-70b-8192": {
|
||||
"max_tokens": 8192,
|
||||
"max_input_tokens": 8192,
|
||||
"max_output_tokens": 8192,
|
||||
"input_cost_per_token": 5.9e-07,
|
||||
"output_cost_per_token": 7.9e-07,
|
||||
"litellm_provider": "groq",
|
||||
"mode": "chat",
|
||||
"supports_response_schema": true,
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
"groq/llama-3.1-8b-instant": {
|
||||
"max_tokens": 8192,
|
||||
"max_input_tokens": 128000,
|
||||
|
|
@ -11991,6 +11970,108 @@
|
|||
"mode": "chat",
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
"openrouter/openai/gpt-4.1": {
|
||||
"max_tokens": 32768,
|
||||
"max_input_tokens": 1047576,
|
||||
"max_output_tokens": 32768,
|
||||
"input_cost_per_token": 2e-06,
|
||||
"output_cost_per_token": 8e-06,
|
||||
"cache_read_input_token_cost": 5e-07,
|
||||
"litellm_provider": "openrouter",
|
||||
"mode": "chat",
|
||||
"supports_function_calling": true,
|
||||
"supports_parallel_function_calling": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_vision": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_system_messages": true,
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
"openrouter/openai/gpt-4.1-2025-04-14": {
|
||||
"max_tokens": 32768,
|
||||
"max_input_tokens": 1047576,
|
||||
"max_output_tokens": 32768,
|
||||
"input_cost_per_token": 2e-06,
|
||||
"output_cost_per_token": 8e-06,
|
||||
"cache_read_input_token_cost": 5e-07,
|
||||
"litellm_provider": "openrouter",
|
||||
"mode": "chat",
|
||||
"supports_function_calling": true,
|
||||
"supports_parallel_function_calling": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_vision": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_system_messages": true,
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
"openrouter/openai/gpt-4.1-mini": {
|
||||
"max_tokens": 32768,
|
||||
"max_input_tokens": 1047576,
|
||||
"max_output_tokens": 32768,
|
||||
"input_cost_per_token": 4e-07,
|
||||
"output_cost_per_token": 1.6e-06,
|
||||
"cache_read_input_token_cost": 1e-07,
|
||||
"litellm_provider": "openrouter",
|
||||
"mode": "chat",
|
||||
"supports_function_calling": true,
|
||||
"supports_parallel_function_calling": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_vision": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_system_messages": true,
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
"openrouter/openai/gpt-4.1-mini-2025-04-14": {
|
||||
"max_tokens": 32768,
|
||||
"max_input_tokens": 1047576,
|
||||
"max_output_tokens": 32768,
|
||||
"input_cost_per_token": 4e-07,
|
||||
"output_cost_per_token": 1.6e-06,
|
||||
"cache_read_input_token_cost": 1e-07,
|
||||
"litellm_provider": "openrouter",
|
||||
"mode": "chat",
|
||||
"supports_function_calling": true,
|
||||
"supports_parallel_function_calling": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_vision": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_system_messages": true,
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
"openrouter/openai/gpt-4.1-nano": {
|
||||
"max_tokens": 32768,
|
||||
"max_input_tokens": 1047576,
|
||||
"max_output_tokens": 32768,
|
||||
"input_cost_per_token": 1e-07,
|
||||
"output_cost_per_token": 4e-07,
|
||||
"cache_read_input_token_cost": 2.5e-08,
|
||||
"litellm_provider": "openrouter",
|
||||
"mode": "chat",
|
||||
"supports_function_calling": true,
|
||||
"supports_parallel_function_calling": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_vision": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_system_messages": true,
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
"openrouter/openai/gpt-4.1-nano-2025-04-14": {
|
||||
"max_tokens": 32768,
|
||||
"max_input_tokens": 1047576,
|
||||
"max_output_tokens": 32768,
|
||||
"input_cost_per_token": 1e-07,
|
||||
"output_cost_per_token": 4e-07,
|
||||
"cache_read_input_token_cost": 2.5e-08,
|
||||
"litellm_provider": "openrouter",
|
||||
"mode": "chat",
|
||||
"supports_function_calling": true,
|
||||
"supports_parallel_function_calling": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_vision": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_system_messages": true,
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
"openrouter/openai/gpt-5-mini": {
|
||||
"max_tokens": 128000,
|
||||
"max_input_tokens": 400000,
|
||||
|
|
@ -14970,10 +15051,10 @@
|
|||
"output_cost_per_token": 6e-06,
|
||||
"max_input_tokens": 262000,
|
||||
"litellm_provider": "together_ai",
|
||||
"supports_function_calling": false,
|
||||
"supports_parallel_function_calling": false,
|
||||
"supports_function_calling": true,
|
||||
"supports_parallel_function_calling": true,
|
||||
"mode": "chat",
|
||||
"supports_tool_choice": false,
|
||||
"supports_tool_choice": true,
|
||||
"source": "https://www.together.ai/models/qwen3-235b-a22b-instruct-2507-fp8"
|
||||
},
|
||||
"together_ai/Qwen/Qwen3-Coder-480B-A35B-Instruct-FP8": {
|
||||
|
|
@ -14981,10 +15062,10 @@
|
|||
"output_cost_per_token": 2e-06,
|
||||
"max_input_tokens": 256000,
|
||||
"litellm_provider": "together_ai",
|
||||
"supports_function_calling": false,
|
||||
"supports_parallel_function_calling": false,
|
||||
"supports_function_calling": true,
|
||||
"supports_parallel_function_calling": true,
|
||||
"mode": "chat",
|
||||
"supports_tool_choice": false,
|
||||
"supports_tool_choice": true,
|
||||
"source": "https://www.together.ai/models/qwen3-coder-480b-a35b-instruct"
|
||||
},
|
||||
"together_ai/Qwen/Qwen3-235B-A22B-Thinking-2507": {
|
||||
|
|
@ -14992,10 +15073,10 @@
|
|||
"output_cost_per_token": 3e-06,
|
||||
"max_input_tokens": 256000,
|
||||
"litellm_provider": "together_ai",
|
||||
"supports_function_calling": false,
|
||||
"supports_parallel_function_calling": false,
|
||||
"supports_function_calling": true,
|
||||
"supports_parallel_function_calling": true,
|
||||
"mode": "chat",
|
||||
"supports_tool_choice": false,
|
||||
"supports_tool_choice": true,
|
||||
"source": "https://www.together.ai/models/qwen3-235b-a22b-thinking-2507"
|
||||
},
|
||||
"together_ai/Qwen/Qwen3-235B-A22B-fp8-tput": {
|
||||
|
|
@ -15038,10 +15119,10 @@
|
|||
"output_cost_per_token": 2.19e-06,
|
||||
"max_input_tokens": 128000,
|
||||
"litellm_provider": "together_ai",
|
||||
"supports_function_calling": false,
|
||||
"supports_parallel_function_calling": false,
|
||||
"supports_function_calling": true,
|
||||
"supports_parallel_function_calling": true,
|
||||
"mode": "chat",
|
||||
"supports_tool_choice": false,
|
||||
"supports_tool_choice": true,
|
||||
"source": "https://www.together.ai/models/deepseek-r1-0528-throughput"
|
||||
},
|
||||
"together_ai/mistralai/Mistral-Small-24B-Instruct-2501": {
|
||||
|
|
@ -15066,9 +15147,9 @@
|
|||
"output_cost_per_token": 6e-07,
|
||||
"max_input_tokens": 128000,
|
||||
"litellm_provider": "together_ai",
|
||||
"supports_function_calling": false,
|
||||
"supports_tool_choice": false,
|
||||
"supports_parallel_function_calling": false,
|
||||
"supports_function_calling": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_parallel_function_calling": true,
|
||||
"mode": "chat",
|
||||
"source": "https://www.together.ai/models/gpt-oss-120b"
|
||||
},
|
||||
|
|
@ -15077,9 +15158,9 @@
|
|||
"output_cost_per_token": 2e-07,
|
||||
"max_input_tokens": 128000,
|
||||
"litellm_provider": "together_ai",
|
||||
"supports_function_calling": false,
|
||||
"supports_tool_choice": false,
|
||||
"supports_parallel_function_calling": false,
|
||||
"supports_function_calling": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_parallel_function_calling": true,
|
||||
"mode": "chat",
|
||||
"source": "https://www.together.ai/models/gpt-oss-20b"
|
||||
},
|
||||
|
|
@ -15088,12 +15169,24 @@
|
|||
"output_cost_per_token": 1.1e-06,
|
||||
"max_input_tokens": 128000,
|
||||
"litellm_provider": "together_ai",
|
||||
"supports_function_calling": false,
|
||||
"supports_tool_choice": false,
|
||||
"supports_parallel_function_calling": false,
|
||||
"supports_function_calling": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_parallel_function_calling": true,
|
||||
"mode": "chat",
|
||||
"source": "https://www.together.ai/models/glm-4-5-air"
|
||||
},
|
||||
"together_ai/deepseek-ai/DeepSeek-V3.1": {
|
||||
"input_cost_per_token": 0.6e-06,
|
||||
"output_cost_per_token": 1.7e-06,
|
||||
"max_tokens": 128000,
|
||||
"litellm_provider": "together_ai",
|
||||
"supports_function_calling": true,
|
||||
"supports_parallel_function_calling": true,
|
||||
"supports_reasoning": true,
|
||||
"mode": "chat",
|
||||
"supports_tool_choice": true,
|
||||
"source": "https://www.together.ai/models/deepseek-v3-1"
|
||||
},
|
||||
"ollama/codegemma": {
|
||||
"max_tokens": 8192,
|
||||
"max_input_tokens": 8192,
|
||||
|
|
|
|||
|
|
@ -25,7 +25,9 @@ IGNORE_FUNCTIONS = [
|
|||
"filter_value_from_dict", # max depth set.
|
||||
"normalize_json_schema_types", # max depth set.
|
||||
"_extract_fields_recursive", # max depth set.
|
||||
"_remove_json_schema_refs", # max depth set.
|
||||
"_remove_json_schema_refs", # max depth set.,
|
||||
"_convert_schema_types", # max depth set.,
|
||||
"_fix_enum_empty_strings", # max depth set.,
|
||||
]
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -1054,7 +1054,7 @@ def test_parse_content_for_reasoning(content, expected_reasoning, expected_conte
|
|||
("gemini/gemini-1.5-pro", True),
|
||||
("predibase/llama3-8b-instruct", True),
|
||||
("gpt-3.5-turbo", False),
|
||||
("groq/llama3-70b-8192", True),
|
||||
("groq/llama-3.3-70b-versatile", True),
|
||||
],
|
||||
)
|
||||
def test_supports_response_schema(model, expected_bool):
|
||||
|
|
|
|||
|
|
@ -141,6 +141,108 @@ class BaseLLMChatTest(ABC):
|
|||
# for OpenAI the content contains the JSON schema, so we need to assert that the content is not None
|
||||
assert response.choices[0].message.content is not None
|
||||
|
||||
|
||||
def test_tool_call_with_property_type_array(self):
|
||||
litellm._turn_on_debug()
|
||||
from litellm.utils import supports_function_calling
|
||||
os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True"
|
||||
litellm.model_cost = litellm.get_model_cost_map(url="")
|
||||
|
||||
base_completion_call_args = self.get_base_completion_call_args()
|
||||
if not supports_function_calling(base_completion_call_args["model"], None):
|
||||
print("Model does not support function calling")
|
||||
pytest.skip("Model does not support function calling")
|
||||
base_completion_call_args = self.get_base_completion_call_args()
|
||||
response = self.completion_function(
|
||||
**base_completion_call_args,
|
||||
messages = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": "Tell me if the shoe brand Air Jordan has more models than the shoe brand Nike."
|
||||
}
|
||||
],
|
||||
tools = [
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "shoe_get_id",
|
||||
"description": "Get information about a show by its ID or name",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"shoe_id": {
|
||||
"type": ["string", "number"],
|
||||
"description": "The shoe ID or name"
|
||||
}
|
||||
},
|
||||
"required": ["shoe_id"],
|
||||
"additionalProperties": False,
|
||||
"$schema": "http://json-schema.org/draft-07/schema#"
|
||||
}
|
||||
}
|
||||
},
|
||||
]
|
||||
)
|
||||
print(response)
|
||||
print(json.dumps(response, indent=4, default=str))
|
||||
|
||||
def test_tool_call_with_empty_enum_property(self):
|
||||
litellm._turn_on_debug()
|
||||
from litellm.utils import supports_function_calling
|
||||
os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True"
|
||||
litellm.model_cost = litellm.get_model_cost_map(url="")
|
||||
|
||||
base_completion_call_args = self.get_base_completion_call_args()
|
||||
if not supports_function_calling(base_completion_call_args["model"], None):
|
||||
print("Model does not support function calling")
|
||||
pytest.skip("Model does not support function calling")
|
||||
base_completion_call_args = self.get_base_completion_call_args()
|
||||
response = self.completion_function(
|
||||
**base_completion_call_args,
|
||||
messages = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": "Search for the latest iPhone models and tell me which storage options are available."
|
||||
}
|
||||
],
|
||||
tools = [
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "litellm_product_search",
|
||||
"description": "Search for product information and specifications.\n\nSupports filtering by category, brand, price range, and availability.\nCan retrieve detailed product specifications, pricing, and stock information.\nSupports different search modes and result formatting options.\n",
|
||||
"parameters": {
|
||||
"properties": {
|
||||
"search_mode": {
|
||||
"default": "",
|
||||
"description": "The search strategy to use for finding products.",
|
||||
"enum": [
|
||||
"",
|
||||
"product_search",
|
||||
"product_search_with_filters",
|
||||
"product_search_with_sorting",
|
||||
"product_search_with_pagination",
|
||||
"product_search_with_aggregation",
|
||||
],
|
||||
"title": "Search Mode",
|
||||
"type": "string"
|
||||
},
|
||||
},
|
||||
"required": [
|
||||
"search_mode"
|
||||
],
|
||||
"title": "product_search_arguments",
|
||||
"type": "object"
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
)
|
||||
print(response)
|
||||
print(json.dumps(response, indent=4, default=str))
|
||||
|
||||
|
||||
|
||||
def test_streaming(self):
|
||||
"""Check if litellm handles streaming correctly"""
|
||||
from litellm.types.utils import ModelResponseStream
|
||||
|
|
|
|||
|
|
@ -436,7 +436,10 @@ def test_gemini_with_empty_function_call_arguments():
|
|||
async def test_claude_tool_use_with_gemini():
|
||||
response = await litellm.anthropic.messages.acreate(
|
||||
messages=[
|
||||
{"role": "user", "content": "Hello, can you tell me the weather in Boston. Please respond with a tool call?"}
|
||||
{
|
||||
"role": "user",
|
||||
"content": "Hello, can you tell me the weather in Boston. Please respond with a tool call?",
|
||||
}
|
||||
],
|
||||
model="gemini/gemini-2.5-flash",
|
||||
stream=True,
|
||||
|
|
@ -578,11 +581,17 @@ def test_gemini_tool_use():
|
|||
assert stop_reason is not None
|
||||
assert stop_reason == "tool_calls"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_gemini_image_generation_async():
|
||||
litellm._turn_on_debug()
|
||||
response = await litellm.acompletion(
|
||||
messages=[{"role": "user", "content": "Generate an image of a banana wearing a costume that says LiteLLM"}],
|
||||
messages=[
|
||||
{
|
||||
"role": "user",
|
||||
"content": "Generate an image of a banana wearing a costume that says LiteLLM",
|
||||
}
|
||||
],
|
||||
model="gemini/gemini-2.5-flash-image-preview",
|
||||
)
|
||||
|
||||
|
|
@ -597,12 +606,16 @@ async def test_gemini_image_generation_async():
|
|||
assert IMAGE_URL["url"].startswith("data:image/png;base64,")
|
||||
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_gemini_image_generation_async_stream():
|
||||
#litellm._turn_on_debug()
|
||||
# litellm._turn_on_debug()
|
||||
response = await litellm.acompletion(
|
||||
messages=[{"role": "user", "content": "Generate an image of a banana wearing a costume that says LiteLLM"}],
|
||||
messages=[
|
||||
{
|
||||
"role": "user",
|
||||
"content": "Generate an image of a banana wearing a costume that says LiteLLM",
|
||||
}
|
||||
],
|
||||
model="gemini/gemini-2.5-flash-image-preview",
|
||||
stream=True,
|
||||
)
|
||||
|
|
@ -611,35 +624,144 @@ async def test_gemini_image_generation_async_stream():
|
|||
model_response_image = None
|
||||
async for chunk in response:
|
||||
print("CHUNK: ", chunk)
|
||||
if hasattr(chunk.choices[0].delta, "image") and chunk.choices[0].delta.image is not None:
|
||||
if (
|
||||
hasattr(chunk.choices[0].delta, "image")
|
||||
and chunk.choices[0].delta.image is not None
|
||||
):
|
||||
model_response_image = chunk.choices[0].delta.image
|
||||
print("MODEL_RESPONSE_IMAGE: ", model_response_image)
|
||||
assert model_response_image is not None
|
||||
assert model_response_image["url"].startswith("data:image/png;base64,")
|
||||
break
|
||||
|
||||
|
||||
#########################################################
|
||||
# Important: Validate we did get an image in the response
|
||||
#########################################################
|
||||
assert model_response_image is not None
|
||||
assert model_response_image["url"].startswith("data:image/png;base64,")
|
||||
|
||||
|
||||
|
||||
def test_system_message_with_no_user_message():
|
||||
"""
|
||||
Test that the system message is translated correctly for non-OpenAI providers.
|
||||
"""
|
||||
messages = [
|
||||
{
|
||||
"role": "system",
|
||||
"content": "Be a good bot!",
|
||||
},
|
||||
]
|
||||
"""
|
||||
Test that the system message is translated correctly for non-OpenAI providers.
|
||||
"""
|
||||
messages = [
|
||||
{
|
||||
"role": "system",
|
||||
"content": "Be a good bot!",
|
||||
},
|
||||
]
|
||||
|
||||
response = litellm.completion(
|
||||
model="gemini/gemini-2.5-flash",
|
||||
messages=messages,
|
||||
response = litellm.completion(
|
||||
model="gemini/gemini-2.5-flash",
|
||||
messages=messages,
|
||||
)
|
||||
assert response is not None
|
||||
|
||||
assert response.choices[0].message.content is not None
|
||||
|
||||
|
||||
def get_current_weather(location, unit="fahrenheit"):
|
||||
"""Get the current weather in a given location"""
|
||||
if "tokyo" in location.lower():
|
||||
return json.dumps({"location": "Tokyo", "temperature": "10", "unit": "celsius"})
|
||||
elif "san francisco" in location.lower():
|
||||
return json.dumps(
|
||||
{"location": "San Francisco", "temperature": "72", "unit": "fahrenheit"}
|
||||
)
|
||||
assert response is not None
|
||||
elif "paris" in location.lower():
|
||||
return json.dumps({"location": "Paris", "temperature": "22", "unit": "celsius"})
|
||||
else:
|
||||
return json.dumps({"location": location, "temperature": "unknown"})
|
||||
|
||||
assert response.choices[0].message.content is not None
|
||||
|
||||
def test_gemini_with_thinking():
|
||||
from litellm import completion
|
||||
|
||||
litellm._turn_on_debug()
|
||||
litellm.modify_params = True
|
||||
model = "gemini/gemini-2.5-flash"
|
||||
messages = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": "What's the weather like in San Francisco, Tokyo, and Paris? - give me 3 responses",
|
||||
}
|
||||
]
|
||||
|
||||
tools = [
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "get_current_weather",
|
||||
"description": "Get the current weather in a given location",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"location": {
|
||||
"type": "string",
|
||||
"description": "The city and state",
|
||||
},
|
||||
"unit": {
|
||||
"type": "string",
|
||||
"enum": ["celsius", "fahrenheit"],
|
||||
},
|
||||
},
|
||||
"required": ["location"],
|
||||
},
|
||||
},
|
||||
}
|
||||
]
|
||||
response = litellm.completion(
|
||||
model=model,
|
||||
messages=messages,
|
||||
tools=tools,
|
||||
tool_choice="auto", # auto is default, but we'll be explicit
|
||||
reasoning_effort="low",
|
||||
)
|
||||
print("Response\n", response)
|
||||
response_message = response.choices[0].message
|
||||
tool_calls = response_message.tool_calls
|
||||
|
||||
print("Expecting there to be 3 tool calls")
|
||||
assert len(tool_calls) > 0 # this has to call the function for SF, Tokyo and paris
|
||||
|
||||
# Step 2: check if the model wanted to call a function
|
||||
print(f"tool_calls: {tool_calls}")
|
||||
if tool_calls:
|
||||
# Step 3: call the function
|
||||
# Note: the JSON response may not always be valid; be sure to handle errors
|
||||
available_functions = {
|
||||
"get_current_weather": get_current_weather,
|
||||
} # only one function in this example, but you can have multiple
|
||||
messages.append(response_message) # extend conversation with assistant's reply
|
||||
print("Response message\n", response_message)
|
||||
# Step 4: send the info for each function call and function response to the model
|
||||
for tool_call in tool_calls:
|
||||
function_name = tool_call.function.name
|
||||
if function_name not in available_functions:
|
||||
# the model called a function that does not exist in available_functions - don't try calling anything
|
||||
return
|
||||
function_to_call = available_functions[function_name]
|
||||
function_args = json.loads(tool_call.function.arguments)
|
||||
function_response = function_to_call(
|
||||
location=function_args.get("location"),
|
||||
unit=function_args.get("unit"),
|
||||
)
|
||||
messages.append(
|
||||
{
|
||||
"tool_call_id": tool_call.id,
|
||||
"role": "tool",
|
||||
"name": function_name,
|
||||
"content": function_response,
|
||||
}
|
||||
) # extend conversation with function response
|
||||
print(f"messages: {messages}")
|
||||
second_response = litellm.completion(
|
||||
model=model,
|
||||
messages=messages,
|
||||
seed=22,
|
||||
reasoning_effort="low",
|
||||
tools=tools,
|
||||
drop_params=True,
|
||||
) # get a new response from the model where it can see the function response
|
||||
print("second response\n", second_response)
|
||||
|
|
|
|||
|
|
@ -565,7 +565,7 @@ def test_groq_response_cost_tracking(is_streaming):
|
|||
|
||||
response_cost = litellm.response_cost_calculator(
|
||||
response_object=response,
|
||||
model="groq/llama3-70b-8192",
|
||||
model="groq/llama-3.3-70b-versatile",
|
||||
custom_llm_provider="groq",
|
||||
call_type=CallTypes.acompletion.value,
|
||||
optional_params={},
|
||||
|
|
|
|||
|
|
@ -50,7 +50,7 @@ def get_current_weather(location, unit="fahrenheit"):
|
|||
"claude-3-haiku-20240307",
|
||||
"gemini/gemini-1.5-pro",
|
||||
"anthropic.claude-3-sonnet-20240229-v1:0",
|
||||
"groq/llama3-8b-8192",
|
||||
"groq/llama-3.1-8b-instant",
|
||||
"cohere_chat/command-r",
|
||||
],
|
||||
)
|
||||
|
|
|
|||
|
|
@ -31,7 +31,7 @@ async def test_get_available_deployments():
|
|||
},
|
||||
{
|
||||
"model_name": "gpt-3.5-turbo",
|
||||
"litellm_params": {"model": "groq/llama3-8b-8192"},
|
||||
"litellm_params": {"model": "groq/llama-3.1-8b-instant"},
|
||||
"model_info": {"id": "groq-llama"},
|
||||
},
|
||||
]
|
||||
|
|
@ -182,7 +182,7 @@ async def test_get_available_endpoints_tpm_rpm_check_async(ans_rpm):
|
|||
},
|
||||
{
|
||||
"model_name": "gpt-3.5-turbo",
|
||||
"litellm_params": {"model": "groq/llama3-8b-8192"},
|
||||
"litellm_params": {"model": "groq/llama-3.1-8b-instant"},
|
||||
"model_info": {"id": "5678", "rpm": non_ans_rpm},
|
||||
},
|
||||
]
|
||||
|
|
|
|||
|
|
@ -15,7 +15,7 @@ from litellm import completion, embedding
|
|||
|
||||
litellm.set_verbose = True
|
||||
|
||||
model_alias_map = {"good-model": "groq/llama3-8b-8192"}
|
||||
model_alias_map = {"good-model": "groq/llama-3.1-8b-instant"}
|
||||
|
||||
|
||||
def test_model_alias_map(caplog):
|
||||
|
|
@ -35,7 +35,7 @@ def test_model_alias_map(caplog):
|
|||
for log in captured_logs:
|
||||
assert "ERROR" not in log
|
||||
|
||||
assert "llama3-8b-8192" in response.model
|
||||
assert "llama-3.1-8b-instant" in response.model
|
||||
except litellm.ServiceUnavailableError:
|
||||
pass
|
||||
except Exception as e:
|
||||
|
|
|
|||
|
|
@ -119,7 +119,7 @@ async def test_router_provider_wildcard_routing():
|
|||
print("response 2 = ", response2)
|
||||
|
||||
response3 = await router.acompletion(
|
||||
model="groq/llama3-8b-8192",
|
||||
model="groq/llama-3.1-8b-instant",
|
||||
messages=[{"role": "user", "content": "hello"}],
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -44,7 +44,7 @@ async def test_batch_completion_multiple_models(mode):
|
|||
{
|
||||
"model_name": "groq-llama",
|
||||
"litellm_params": {
|
||||
"model": "groq/llama3-8b-8192",
|
||||
"model": "groq/llama-3.1-8b-instant",
|
||||
},
|
||||
},
|
||||
]
|
||||
|
|
@ -143,7 +143,7 @@ async def test_batch_completion_fastest_response_streaming():
|
|||
{
|
||||
"model_name": "groq-llama",
|
||||
"litellm_params": {
|
||||
"model": "groq/llama3-8b-8192",
|
||||
"model": "groq/llama-3.1-8b-instant",
|
||||
},
|
||||
},
|
||||
]
|
||||
|
|
@ -179,7 +179,7 @@ async def test_batch_completion_multiple_models_multiple_messages():
|
|||
{
|
||||
"model_name": "groq-llama",
|
||||
"litellm_params": {
|
||||
"model": "groq/llama3-8b-8192",
|
||||
"model": "groq/llama-3.1-8b-instant",
|
||||
},
|
||||
},
|
||||
]
|
||||
|
|
|
|||
|
|
@ -43,7 +43,7 @@ async def test_spend_calc_model_on_router_messages():
|
|||
{
|
||||
"model_name": "special-llama-model",
|
||||
"litellm_params": {
|
||||
"model": "groq/llama3-8b-8192",
|
||||
"model": "groq/llama-3.1-8b-instant",
|
||||
},
|
||||
}
|
||||
]
|
||||
|
|
@ -86,7 +86,7 @@ async def test_spend_calc_using_response():
|
|||
}
|
||||
],
|
||||
"created": "1677652288",
|
||||
"model": "groq/llama3-8b-8192",
|
||||
"model": "groq/llama-3.1-8b-instant",
|
||||
"object": "chat.completion",
|
||||
"system_fingerprint": "fp_873a560973",
|
||||
"usage": {
|
||||
|
|
|
|||
|
|
@ -11,7 +11,7 @@ from litellm.integrations.braintrust_logging import BraintrustLogger
|
|||
class TestBraintrustSpanName(unittest.TestCase):
|
||||
"""Test custom span_name functionality in Braintrust logging."""
|
||||
|
||||
@patch('litellm.integrations.braintrust_logging.HTTPHandler')
|
||||
@patch("litellm.integrations.braintrust_logging.HTTPHandler")
|
||||
def test_default_span_name(self, MockHTTPHandler):
|
||||
"""Test that default span name is 'Chat Completion' when not provided."""
|
||||
# Mock HTTP response
|
||||
|
|
@ -22,39 +22,43 @@ class TestBraintrustSpanName(unittest.TestCase):
|
|||
# Setup
|
||||
logger = BraintrustLogger(api_key="test-key")
|
||||
logger.default_project_id = "test-project-id"
|
||||
|
||||
|
||||
# Create a properly structured mock response
|
||||
response_obj = litellm.ModelResponse(
|
||||
id="test-id",
|
||||
object="chat.completion",
|
||||
created=1234567890,
|
||||
model="gpt-3.5-turbo",
|
||||
choices=[{
|
||||
"index": 0,
|
||||
"message": {"role": "assistant", "content": "test response"},
|
||||
"finish_reason": "stop"
|
||||
}],
|
||||
usage={"prompt_tokens": 10, "completion_tokens": 20, "total_tokens": 30}
|
||||
choices=[
|
||||
{
|
||||
"index": 0,
|
||||
"message": {"role": "assistant", "content": "test response"},
|
||||
"finish_reason": "stop",
|
||||
}
|
||||
],
|
||||
usage={"prompt_tokens": 10, "completion_tokens": 20, "total_tokens": 30},
|
||||
)
|
||||
|
||||
|
||||
kwargs = {
|
||||
"litellm_call_id": "test-call-id",
|
||||
"messages": [{"role": "user", "content": "test"}],
|
||||
"litellm_params": {"metadata": {}},
|
||||
"model": "gpt-3.5-turbo",
|
||||
"response_cost": 0.001
|
||||
"response_cost": 0.001,
|
||||
}
|
||||
|
||||
|
||||
# Execute
|
||||
logger.log_success_event(kwargs, response_obj, datetime.now(), datetime.now())
|
||||
|
||||
|
||||
# Verify
|
||||
call_args = mock_http_handler.post.call_args
|
||||
self.assertIsNotNone(call_args)
|
||||
json_data = call_args.kwargs['json']
|
||||
self.assertEqual(json_data['events'][0]['span_attributes']['name'], 'Chat Completion')
|
||||
json_data = call_args.kwargs["json"]
|
||||
self.assertEqual(
|
||||
json_data["events"][0]["span_attributes"]["name"], "Chat Completion"
|
||||
)
|
||||
|
||||
@patch('litellm.integrations.braintrust_logging.HTTPHandler')
|
||||
@patch("litellm.integrations.braintrust_logging.HTTPHandler")
|
||||
def test_custom_span_name(self, MockHTTPHandler):
|
||||
"""Test that custom span name is used when provided in metadata."""
|
||||
# Mock HTTP response
|
||||
|
|
@ -65,39 +69,43 @@ class TestBraintrustSpanName(unittest.TestCase):
|
|||
# Setup
|
||||
logger = BraintrustLogger(api_key="test-key")
|
||||
logger.default_project_id = "test-project-id"
|
||||
|
||||
|
||||
# Create a properly structured mock response
|
||||
response_obj = litellm.ModelResponse(
|
||||
id="test-id",
|
||||
object="chat.completion",
|
||||
created=1234567890,
|
||||
model="gpt-3.5-turbo",
|
||||
choices=[{
|
||||
"index": 0,
|
||||
"message": {"role": "assistant", "content": "test response"},
|
||||
"finish_reason": "stop"
|
||||
}],
|
||||
usage={"prompt_tokens": 10, "completion_tokens": 20, "total_tokens": 30}
|
||||
choices=[
|
||||
{
|
||||
"index": 0,
|
||||
"message": {"role": "assistant", "content": "test response"},
|
||||
"finish_reason": "stop",
|
||||
}
|
||||
],
|
||||
usage={"prompt_tokens": 10, "completion_tokens": 20, "total_tokens": 30},
|
||||
)
|
||||
|
||||
|
||||
kwargs = {
|
||||
"litellm_call_id": "test-call-id",
|
||||
"messages": [{"role": "user", "content": "test"}],
|
||||
"litellm_params": {"metadata": {"span_name": "Custom Operation"}},
|
||||
"model": "gpt-3.5-turbo",
|
||||
"response_cost": 0.001
|
||||
"response_cost": 0.001,
|
||||
}
|
||||
|
||||
|
||||
# Execute
|
||||
logger.log_success_event(kwargs, response_obj, datetime.now(), datetime.now())
|
||||
|
||||
|
||||
# Verify
|
||||
call_args = mock_http_handler.post.call_args
|
||||
self.assertIsNotNone(call_args)
|
||||
json_data = call_args.kwargs['json']
|
||||
self.assertEqual(json_data['events'][0]['span_attributes']['name'], 'Custom Operation')
|
||||
json_data = call_args.kwargs["json"]
|
||||
self.assertEqual(
|
||||
json_data["events"][0]["span_attributes"]["name"], "Custom Operation"
|
||||
)
|
||||
|
||||
@patch('litellm.integrations.braintrust_logging.HTTPHandler')
|
||||
@patch("litellm.integrations.braintrust_logging.HTTPHandler")
|
||||
def test_span_name_with_other_metadata(self, MockHTTPHandler):
|
||||
"""Test that span_name works alongside other metadata fields."""
|
||||
# Mock HTTP response
|
||||
|
|
@ -108,21 +116,23 @@ class TestBraintrustSpanName(unittest.TestCase):
|
|||
# Setup
|
||||
logger = BraintrustLogger(api_key="test-key")
|
||||
logger.default_project_id = "test-project-id"
|
||||
|
||||
|
||||
# Create a properly structured mock response
|
||||
response_obj = litellm.ModelResponse(
|
||||
id="test-id",
|
||||
object="chat.completion",
|
||||
created=1234567890,
|
||||
model="gpt-3.5-turbo",
|
||||
choices=[{
|
||||
"index": 0,
|
||||
"message": {"role": "assistant", "content": "test response"},
|
||||
"finish_reason": "stop"
|
||||
}],
|
||||
usage={"prompt_tokens": 10, "completion_tokens": 20, "total_tokens": 30}
|
||||
choices=[
|
||||
{
|
||||
"index": 0,
|
||||
"message": {"role": "assistant", "content": "test response"},
|
||||
"finish_reason": "stop",
|
||||
}
|
||||
],
|
||||
usage={"prompt_tokens": 10, "completion_tokens": 20, "total_tokens": 30},
|
||||
)
|
||||
|
||||
|
||||
kwargs = {
|
||||
"litellm_call_id": "test-call-id",
|
||||
"messages": [{"role": "user", "content": "test"}],
|
||||
|
|
@ -132,34 +142,40 @@ class TestBraintrustSpanName(unittest.TestCase):
|
|||
"project_id": "custom-project",
|
||||
"user_id": "user123",
|
||||
"session_id": "session456",
|
||||
"environment": "production"
|
||||
"environment": "production",
|
||||
}
|
||||
},
|
||||
"model": "gpt-3.5-turbo",
|
||||
"response_cost": 0.001
|
||||
"response_cost": 0.001,
|
||||
"standard_logging_object": {
|
||||
"user_id": "user123",
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
# Execute
|
||||
logger.log_success_event(kwargs, response_obj, datetime.now(), datetime.now())
|
||||
|
||||
|
||||
# Verify
|
||||
call_args = mock_http_handler.post.call_args
|
||||
self.assertIsNotNone(call_args)
|
||||
json_data = call_args.kwargs['json']
|
||||
|
||||
# Check span name
|
||||
self.assertEqual(json_data['events'][0]['span_attributes']['name'], 'Multi Metadata Test')
|
||||
|
||||
# Check that other metadata is preserved (except for filtered keys)
|
||||
event_metadata = json_data['events'][0]['metadata']
|
||||
self.assertEqual(event_metadata['user_id'], 'user123')
|
||||
self.assertEqual(event_metadata['session_id'], 'session456')
|
||||
self.assertEqual(event_metadata['environment'], 'production')
|
||||
|
||||
# Span name should be in span_attributes, not in metadata
|
||||
self.assertIn('span_name', event_metadata) # span_name is also kept in metadata
|
||||
json_data = call_args.kwargs["json"]
|
||||
|
||||
@patch('litellm.integrations.braintrust_logging.get_async_httpx_client')
|
||||
# Check span name
|
||||
self.assertEqual(
|
||||
json_data["events"][0]["span_attributes"]["name"], "Multi Metadata Test"
|
||||
)
|
||||
|
||||
# Check that other metadata is preserved (except for filtered keys)
|
||||
event_metadata = json_data["events"][0]["metadata"]
|
||||
print(event_metadata)
|
||||
self.assertEqual(event_metadata["user_id"], "user123")
|
||||
self.assertEqual(event_metadata["session_id"], "session456")
|
||||
self.assertEqual(event_metadata["environment"], "production")
|
||||
|
||||
# Span name should be in span_attributes, not in metadata
|
||||
self.assertIn("span_name", event_metadata) # span_name is also kept in metadata
|
||||
|
||||
@patch("litellm.integrations.braintrust_logging.get_async_httpx_client")
|
||||
async def test_async_custom_span_name(self, mock_get_http_handler):
|
||||
"""Test async logging with custom span name."""
|
||||
# Mock async HTTP response
|
||||
|
|
@ -170,38 +186,44 @@ class TestBraintrustSpanName(unittest.TestCase):
|
|||
# Setup
|
||||
logger = BraintrustLogger(api_key="test-key")
|
||||
logger.default_project_id = "test-project-id"
|
||||
|
||||
|
||||
# Create a properly structured mock response
|
||||
response_obj = litellm.ModelResponse(
|
||||
id="test-id",
|
||||
object="chat.completion",
|
||||
created=1234567890,
|
||||
model="gpt-3.5-turbo",
|
||||
choices=[{
|
||||
"index": 0,
|
||||
"message": {"role": "assistant", "content": "test response"},
|
||||
"finish_reason": "stop"
|
||||
}],
|
||||
usage={"prompt_tokens": 10, "completion_tokens": 20, "total_tokens": 30}
|
||||
choices=[
|
||||
{
|
||||
"index": 0,
|
||||
"message": {"role": "assistant", "content": "test response"},
|
||||
"finish_reason": "stop",
|
||||
}
|
||||
],
|
||||
usage={"prompt_tokens": 10, "completion_tokens": 20, "total_tokens": 30},
|
||||
)
|
||||
|
||||
|
||||
kwargs = {
|
||||
"litellm_call_id": "test-call-id",
|
||||
"messages": [{"role": "user", "content": "test"}],
|
||||
"litellm_params": {"metadata": {"span_name": "Async Custom Operation"}},
|
||||
"model": "gpt-3.5-turbo",
|
||||
"response_cost": 0.001
|
||||
"response_cost": 0.001,
|
||||
}
|
||||
|
||||
|
||||
# Execute
|
||||
await logger.async_log_success_event(kwargs, response_obj, datetime.now(), datetime.now())
|
||||
|
||||
await logger.async_log_success_event(
|
||||
kwargs, response_obj, datetime.now(), datetime.now()
|
||||
)
|
||||
|
||||
# Verify
|
||||
call_args = mock_http_handler.post.call_args
|
||||
self.assertIsNotNone(call_args)
|
||||
json_data = call_args.kwargs['json']
|
||||
self.assertEqual(json_data['events'][0]['span_attributes']['name'], 'Async Custom Operation')
|
||||
json_data = call_args.kwargs["json"]
|
||||
self.assertEqual(
|
||||
json_data["events"][0]["span_attributes"]["name"], "Async Custom Operation"
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
unittest.main()
|
||||
|
|
|
|||
41
tests/test_litellm/litellm_core_utils/test_image_handling.py
Normal file
41
tests/test_litellm/litellm_core_utils/test_image_handling.py
Normal file
|
|
@ -0,0 +1,41 @@
|
|||
import pytest
|
||||
from httpx import Request, Response
|
||||
|
||||
import litellm
|
||||
from litellm.litellm_core_utils.prompt_templates.image_handling import (
|
||||
convert_url_to_base64,
|
||||
)
|
||||
|
||||
|
||||
class DummyClient:
|
||||
def get(self, url, follow_redirects=True):
|
||||
return Response(status_code=404, request=Request("GET", url))
|
||||
|
||||
|
||||
def test_invalid_image_url_raises_bad_request(monkeypatch):
|
||||
monkeypatch.setattr(litellm, "module_level_client", DummyClient())
|
||||
with pytest.raises(litellm.ImageFetchError) as excinfo:
|
||||
convert_url_to_base64("https://invalid.example/image.png")
|
||||
assert "Unable to fetch image" in str(excinfo.value)
|
||||
|
||||
|
||||
def test_completion_with_invalid_image_url(monkeypatch):
|
||||
monkeypatch.setattr(litellm, "module_level_client", DummyClient())
|
||||
messages = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": "hi"},
|
||||
{
|
||||
"type": "image_url",
|
||||
"image_url": {"url": "https://invalid.example/image.png"},
|
||||
},
|
||||
],
|
||||
}
|
||||
]
|
||||
with pytest.raises(litellm.ImageFetchError) as excinfo:
|
||||
litellm.completion(
|
||||
model="gemini/gemini-pro", messages=messages, api_key="test"
|
||||
)
|
||||
assert excinfo.value.status_code == 400
|
||||
assert "Unable to fetch image" in str(excinfo.value)
|
||||
|
|
@ -159,6 +159,261 @@ class TestOllamaConfig:
|
|||
assert result.choices[0]["finish_reason"] == "stop"
|
||||
# No usage assertions here as we don't need to test them in every case
|
||||
|
||||
def test_transform_response_with_thinking_tags(self):
|
||||
"""Test that responses with <think>...</think> tags parse reasoning content correctly."""
|
||||
# Initialize config
|
||||
config = OllamaConfig()
|
||||
|
||||
# Create mock response with thinking tags
|
||||
raw_response = MagicMock()
|
||||
raw_response.json.return_value = {
|
||||
"response": "<think>I need to think about this problem step by step</think>Here is my answer",
|
||||
"prompt_eval_count": 15,
|
||||
"eval_count": 8,
|
||||
}
|
||||
|
||||
# Create properly structured model response object
|
||||
model_response = ModelResponse(
|
||||
id="test_id",
|
||||
choices=[{"message": Message(content="")}],
|
||||
)
|
||||
|
||||
# Create mock encoding
|
||||
mock_encoding = MagicMock()
|
||||
mock_encoding.encode.return_value = [1, 2, 3]
|
||||
|
||||
# Transform response
|
||||
result = config.transform_response(
|
||||
model="llama2",
|
||||
raw_response=raw_response,
|
||||
model_response=model_response,
|
||||
logging_obj=MagicMock(),
|
||||
request_data={},
|
||||
messages=[],
|
||||
optional_params={},
|
||||
litellm_params={},
|
||||
encoding=mock_encoding,
|
||||
)
|
||||
|
||||
# Verify reasoning content is extracted
|
||||
assert (
|
||||
result.choices[0]["message"].reasoning_content
|
||||
== "I need to think about this problem step by step"
|
||||
)
|
||||
assert result.choices[0]["message"].content == "Here is my answer"
|
||||
assert result.choices[0]["finish_reason"] == "stop"
|
||||
|
||||
def test_transform_response_with_thinking_tags_alternative(self):
|
||||
"""Test that responses with <thinking>...</thinking> tags parse reasoning content correctly."""
|
||||
# Initialize config
|
||||
config = OllamaConfig()
|
||||
|
||||
# Create mock response with thinking tags (alternative format)
|
||||
raw_response = MagicMock()
|
||||
raw_response.json.return_value = {
|
||||
"response": "<thinking>Let me analyze this carefully</thinking>The solution is X",
|
||||
}
|
||||
|
||||
# Create properly structured model response object
|
||||
model_response = ModelResponse(
|
||||
id="test_id",
|
||||
choices=[{"message": Message(content="")}],
|
||||
)
|
||||
|
||||
# Create mock encoding
|
||||
mock_encoding = MagicMock()
|
||||
mock_encoding.encode.return_value = [1, 2, 3]
|
||||
|
||||
# Transform response
|
||||
result = config.transform_response(
|
||||
model="llama2",
|
||||
raw_response=raw_response,
|
||||
model_response=model_response,
|
||||
logging_obj=MagicMock(),
|
||||
request_data={},
|
||||
messages=[],
|
||||
optional_params={},
|
||||
litellm_params={},
|
||||
encoding=mock_encoding,
|
||||
)
|
||||
|
||||
# Verify reasoning content is extracted
|
||||
assert (
|
||||
result.choices[0]["message"].reasoning_content
|
||||
== "Let me analyze this carefully"
|
||||
)
|
||||
assert result.choices[0]["message"].content == "The solution is X"
|
||||
assert result.choices[0]["finish_reason"] == "stop"
|
||||
|
||||
def test_transform_response_with_multiline_thinking_tags(self):
|
||||
"""Test that responses with multiline thinking content work correctly."""
|
||||
# Initialize config
|
||||
config = OllamaConfig()
|
||||
|
||||
# Create mock response with multiline thinking content
|
||||
raw_response = MagicMock()
|
||||
raw_response.json.return_value = {
|
||||
"response": "<think>\nThis is a complex problem.\nI need to break it down:\n1. First step\n2. Second step\n</think>Based on my analysis, the answer is Y",
|
||||
}
|
||||
|
||||
# Create properly structured model response object
|
||||
model_response = ModelResponse(
|
||||
id="test_id",
|
||||
choices=[{"message": Message(content="")}],
|
||||
)
|
||||
|
||||
# Create mock encoding
|
||||
mock_encoding = MagicMock()
|
||||
mock_encoding.encode.return_value = [1, 2, 3]
|
||||
|
||||
# Transform response
|
||||
result = config.transform_response(
|
||||
model="llama2",
|
||||
raw_response=raw_response,
|
||||
model_response=model_response,
|
||||
logging_obj=MagicMock(),
|
||||
request_data={},
|
||||
messages=[],
|
||||
optional_params={},
|
||||
litellm_params={},
|
||||
encoding=mock_encoding,
|
||||
)
|
||||
|
||||
# Verify multiline reasoning content is extracted
|
||||
expected_reasoning = "\nThis is a complex problem.\nI need to break it down:\n1. First step\n2. Second step\n"
|
||||
assert result.choices[0]["message"].reasoning_content == expected_reasoning
|
||||
assert (
|
||||
result.choices[0]["message"].content
|
||||
== "Based on my analysis, the answer is Y"
|
||||
)
|
||||
assert result.choices[0]["finish_reason"] == "stop"
|
||||
|
||||
def test_transform_response_thinking_only(self):
|
||||
"""Test response with only thinking content and no additional content."""
|
||||
# Initialize config
|
||||
config = OllamaConfig()
|
||||
|
||||
# Create mock response with only thinking content
|
||||
raw_response = MagicMock()
|
||||
raw_response.json.return_value = {
|
||||
"response": "<think>Just internal thoughts, no response</think>",
|
||||
}
|
||||
|
||||
# Create properly structured model response object
|
||||
model_response = ModelResponse(
|
||||
id="test_id",
|
||||
choices=[{"message": Message(content="")}],
|
||||
)
|
||||
|
||||
# Create mock encoding
|
||||
mock_encoding = MagicMock()
|
||||
mock_encoding.encode.return_value = [1, 2, 3]
|
||||
|
||||
# Transform response
|
||||
result = config.transform_response(
|
||||
model="llama2",
|
||||
raw_response=raw_response,
|
||||
model_response=model_response,
|
||||
logging_obj=MagicMock(),
|
||||
request_data={},
|
||||
messages=[],
|
||||
optional_params={},
|
||||
litellm_params={},
|
||||
encoding=mock_encoding,
|
||||
)
|
||||
|
||||
# Verify reasoning content is extracted and content is empty
|
||||
assert (
|
||||
result.choices[0]["message"].reasoning_content
|
||||
== "Just internal thoughts, no response"
|
||||
)
|
||||
assert result.choices[0]["message"].content == ""
|
||||
assert result.choices[0]["finish_reason"] == "stop"
|
||||
|
||||
def test_transform_response_json_mode_with_thinking_tags(self):
|
||||
"""Test JSON mode with thinking tags - should handle as text when JSON parsing fails."""
|
||||
# Initialize config
|
||||
config = OllamaConfig()
|
||||
|
||||
# Create mock response with thinking tags in JSON mode
|
||||
raw_response = MagicMock()
|
||||
raw_response.json.return_value = {
|
||||
"response": "<think>Planning my JSON response</think>This is not valid JSON",
|
||||
}
|
||||
|
||||
# Create properly structured model response object
|
||||
model_response = ModelResponse(
|
||||
id="test_id",
|
||||
choices=[{"message": Message(content="")}],
|
||||
)
|
||||
|
||||
# Create mock encoding
|
||||
mock_encoding = MagicMock()
|
||||
mock_encoding.encode.return_value = [1, 2, 3]
|
||||
|
||||
# Transform response
|
||||
result = config.transform_response(
|
||||
model="llama2",
|
||||
raw_response=raw_response,
|
||||
model_response=model_response,
|
||||
logging_obj=MagicMock(),
|
||||
request_data={"format": "json"},
|
||||
messages=[],
|
||||
optional_params={},
|
||||
litellm_params={},
|
||||
encoding=mock_encoding,
|
||||
)
|
||||
|
||||
# Verify reasoning content is extracted even in JSON mode when JSON parsing fails
|
||||
assert (
|
||||
result.choices[0]["message"].reasoning_content
|
||||
== "Planning my JSON response"
|
||||
)
|
||||
assert result.choices[0]["message"].content == "This is not valid JSON"
|
||||
assert result.choices[0]["finish_reason"] == "stop"
|
||||
|
||||
def test_transform_response_no_thinking_tags(self):
|
||||
"""Test that responses without thinking tags work normally."""
|
||||
# Initialize config
|
||||
config = OllamaConfig()
|
||||
|
||||
# Create mock response without thinking tags
|
||||
raw_response = MagicMock()
|
||||
raw_response.json.return_value = {
|
||||
"response": "Regular response without any thinking tags",
|
||||
}
|
||||
|
||||
# Create properly structured model response object
|
||||
model_response = ModelResponse(
|
||||
id="test_id",
|
||||
choices=[{"message": Message(content="")}],
|
||||
)
|
||||
|
||||
# Create mock encoding
|
||||
mock_encoding = MagicMock()
|
||||
mock_encoding.encode.return_value = [1, 2, 3]
|
||||
|
||||
# Transform response
|
||||
result = config.transform_response(
|
||||
model="llama2",
|
||||
raw_response=raw_response,
|
||||
model_response=model_response,
|
||||
logging_obj=MagicMock(),
|
||||
request_data={},
|
||||
messages=[],
|
||||
optional_params={},
|
||||
litellm_params={},
|
||||
encoding=mock_encoding,
|
||||
)
|
||||
|
||||
# Verify no reasoning content is extracted
|
||||
assert result.choices[0]["message"].reasoning_content is None
|
||||
assert (
|
||||
result.choices[0]["message"].content
|
||||
== "Regular response without any thinking tags"
|
||||
)
|
||||
assert result.choices[0]["finish_reason"] == "stop"
|
||||
|
||||
|
||||
class TestOllamaTextCompletionResponseIterator:
|
||||
def test_chunk_parser_with_thinking_field(self):
|
||||
|
|
@ -199,10 +454,11 @@ class TestOllamaTextCompletionResponseIterator:
|
|||
|
||||
result = iterator.chunk_parser(normal_chunk)
|
||||
|
||||
assert result["text"] == "Hello world"
|
||||
assert result["is_finished"] is False
|
||||
assert result["finish_reason"] == "stop"
|
||||
assert result["usage"] is None
|
||||
# Updated to handle ModelResponseStream return type
|
||||
assert isinstance(result, ModelResponseStream)
|
||||
assert result.choices and result.choices[0].delta is not None
|
||||
assert result.choices[0].delta.content == "Hello world"
|
||||
assert getattr(result.choices[0].delta, "reasoning_content", None) is None
|
||||
|
||||
def test_chunk_parser_done_chunk(self):
|
||||
"""Test that done chunks work correctly."""
|
||||
|
|
|
|||
|
|
@ -41,3 +41,14 @@ def test_gpt5_temperature_error(config: OpenAIConfig):
|
|||
model="gpt-5",
|
||||
drop_params=False,
|
||||
)
|
||||
|
||||
|
||||
def test_gpt5_unsupported_params_drop(config: OpenAIConfig):
|
||||
assert "top_p" not in config.get_supported_openai_params(model="gpt-5")
|
||||
params = config.map_openai_params(
|
||||
non_default_params={"top_p": 0.5},
|
||||
optional_params={},
|
||||
model="gpt-5",
|
||||
drop_params=True,
|
||||
)
|
||||
assert "top_p" not in params
|
||||
|
|
|
|||
|
|
@ -0,0 +1,75 @@
|
|||
from litellm.llms.vertex_ai.gemini.transformation import check_if_part_exists_in_parts
|
||||
|
||||
|
||||
def test_check_if_part_exists_in_parts():
|
||||
parts = [
|
||||
{"text": "Hello", "thought": True},
|
||||
{"text": "World", "thought": False},
|
||||
]
|
||||
part = {"text": "Hello", "thought": True}
|
||||
new_part = {"text": "Hello World", "thought": True}
|
||||
assert check_if_part_exists_in_parts(parts, part)
|
||||
assert not check_if_part_exists_in_parts(parts, new_part, ["thought"])
|
||||
assert check_if_part_exists_in_parts(parts, new_part, ["text"])
|
||||
|
||||
|
||||
def test_check_if_part_exists_in_parts_camel_case_snake_case():
|
||||
"""Test that function handles both camelCase and snake_case key variations"""
|
||||
# Test snake_case to camelCase matching
|
||||
parts_with_snake_case = [
|
||||
{
|
||||
"function_call": {
|
||||
"name": "get_current_weather",
|
||||
"args": {"location": "San Francisco, CA"},
|
||||
}
|
||||
},
|
||||
{"text": "Some other content"},
|
||||
]
|
||||
|
||||
part_with_camel_case = {
|
||||
"functionCall": {
|
||||
"name": "get_current_weather",
|
||||
"args": {"location": "San Francisco, CA"},
|
||||
}
|
||||
}
|
||||
|
||||
# Should find match between function_call and functionCall
|
||||
assert check_if_part_exists_in_parts(parts_with_snake_case, part_with_camel_case)
|
||||
|
||||
# Test camelCase to snake_case matching
|
||||
parts_with_camel_case = [
|
||||
{"functionCall": {"name": "calculate_sum", "args": {"a": 1, "b": 2}}}
|
||||
]
|
||||
|
||||
part_with_snake_case = {
|
||||
"function_call": {"name": "calculate_sum", "args": {"a": 1, "b": 2}}
|
||||
}
|
||||
|
||||
# Should find match between functionCall and function_call
|
||||
assert check_if_part_exists_in_parts(parts_with_camel_case, part_with_snake_case)
|
||||
|
||||
# Test no match when values differ
|
||||
part_with_different_values = {
|
||||
"function_call": {"name": "different_function", "args": {"x": 5}}
|
||||
}
|
||||
|
||||
assert not check_if_part_exists_in_parts(
|
||||
parts_with_snake_case, part_with_different_values
|
||||
)
|
||||
|
||||
# Test multiple keys with mixed casing
|
||||
parts_mixed = [
|
||||
{
|
||||
"function_call": {"name": "test"},
|
||||
"thoughtSignature": "reasoning",
|
||||
"text": "content",
|
||||
}
|
||||
]
|
||||
|
||||
part_mixed_casing = {
|
||||
"functionCall": {"name": "test"},
|
||||
"thought_signature": "reasoning",
|
||||
"text": "content",
|
||||
}
|
||||
|
||||
assert check_if_part_exists_in_parts(parts_mixed, part_mixed_casing)
|
||||
|
|
@ -677,3 +677,127 @@ def test_vertex_filter_format_uri():
|
|||
)
|
||||
|
||||
assert "uri" not in json.dumps(new_parameters)
|
||||
|
||||
def test_convert_schema_types_type_array_conversion():
|
||||
"""
|
||||
Test _convert_schema_types function handles type arrays and case conversion.
|
||||
|
||||
This test verifies the fix for the issue where type arrays like ["string", "number"]
|
||||
would raise an exception in Vertex AI schema validation.
|
||||
|
||||
Relevant issue: https://github.com/BerriAI/litellm/issues/14091
|
||||
"""
|
||||
from litellm.llms.vertex_ai.common_utils import _convert_schema_types
|
||||
|
||||
# Input: OpenAI-style schema with type array (the problematic case)
|
||||
input_schema = {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"studio": {
|
||||
"type": ["string", "number"],
|
||||
"description": "The studio ID or name"
|
||||
}
|
||||
},
|
||||
"required": ["studio"],
|
||||
"additionalProperties": False,
|
||||
"$schema": "http://json-schema.org/draft-07/schema#"
|
||||
}
|
||||
|
||||
# Expected output: Vertex AI compatible schema with anyOf and uppercase types
|
||||
expected_output = {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"studio": {
|
||||
"anyOf": [
|
||||
{"type": "string"},
|
||||
{"type": "number"}
|
||||
],
|
||||
"description": "The studio ID or name"
|
||||
}
|
||||
},
|
||||
"required": ["studio"],
|
||||
"additionalProperties": False,
|
||||
"$schema": "http://json-schema.org/draft-07/schema#"
|
||||
}
|
||||
|
||||
# Apply the transformation
|
||||
_convert_schema_types(input_schema)
|
||||
|
||||
# Verify the transformation
|
||||
assert input_schema == expected_output
|
||||
|
||||
# Verify specific transformations:
|
||||
# 1. Root level type converted to uppercase
|
||||
assert input_schema["type"] == "object"
|
||||
|
||||
# 2. Type array converted to anyOf format
|
||||
assert "anyOf" in input_schema["properties"]["studio"]
|
||||
assert "type" not in input_schema["properties"]["studio"]
|
||||
|
||||
# 3. Individual types in anyOf are uppercase
|
||||
anyof_types = input_schema["properties"]["studio"]["anyOf"]
|
||||
assert anyof_types[0]["type"] == "string"
|
||||
assert anyof_types[1]["type"] == "number"
|
||||
|
||||
# 4. Other properties preserved
|
||||
assert input_schema["properties"]["studio"]["description"] == "The studio ID or name"
|
||||
assert input_schema["required"] == ["studio"]
|
||||
|
||||
|
||||
def test_fix_enum_empty_strings():
|
||||
"""
|
||||
Test _fix_enum_empty_strings function replaces empty strings with None in enum arrays.
|
||||
|
||||
This test verifies the fix for the issue where Gemini rejects tool definitions
|
||||
with empty strings in enum values, causing API failures.
|
||||
|
||||
Relevant issue: Gemini does not accept empty strings in enum values
|
||||
"""
|
||||
from litellm.llms.vertex_ai.common_utils import _fix_enum_empty_strings
|
||||
|
||||
# Input: Schema with empty string in enum (the problematic case)
|
||||
input_schema = {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"user_agent_type": {
|
||||
"enum": ["", "desktop", "mobile", "tablet"],
|
||||
"type": "string",
|
||||
"description": "Device type for user agent"
|
||||
}
|
||||
},
|
||||
"required": ["user_agent_type"]
|
||||
}
|
||||
|
||||
# Expected output: Empty strings replaced with None
|
||||
expected_output = {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"user_agent_type": {
|
||||
"enum": [None, "desktop", "mobile", "tablet"],
|
||||
"type": "string",
|
||||
"description": "Device type for user agent"
|
||||
}
|
||||
},
|
||||
"required": ["user_agent_type"]
|
||||
}
|
||||
|
||||
# Apply the transformation
|
||||
_fix_enum_empty_strings(input_schema)
|
||||
|
||||
# Verify the transformation
|
||||
assert input_schema == expected_output
|
||||
|
||||
# Verify specific transformations:
|
||||
# 1. Empty string replaced with None
|
||||
enum_values = input_schema["properties"]["user_agent_type"]["enum"]
|
||||
assert "" not in enum_values
|
||||
assert None in enum_values
|
||||
|
||||
# 2. Other enum values preserved
|
||||
assert "desktop" in enum_values
|
||||
assert "mobile" in enum_values
|
||||
assert "tablet" in enum_values
|
||||
|
||||
# 3. Other properties preserved
|
||||
assert input_schema["properties"]["user_agent_type"]["type"] == "string"
|
||||
assert input_schema["properties"]["user_agent_type"]["description"] == "Device type for user agent"
|
||||
|
|
|
|||
189
tests/test_litellm/llms/xai/test_xai_cost_calculator.py
Normal file
189
tests/test_litellm/llms/xai/test_xai_cost_calculator.py
Normal file
|
|
@ -0,0 +1,189 @@
|
|||
"""
|
||||
Test suite for XAI cost calculation functionality.
|
||||
"""
|
||||
|
||||
import math
|
||||
import os
|
||||
import sys
|
||||
|
||||
import litellm
|
||||
from litellm.types.utils import (
|
||||
CompletionTokensDetailsWrapper,
|
||||
Usage,
|
||||
)
|
||||
|
||||
sys.path.insert(
|
||||
0, os.path.abspath("../../..")
|
||||
) # Adds the parent directory to the system path
|
||||
|
||||
from litellm.llms.xai.cost_calculator import cost_per_token
|
||||
|
||||
|
||||
class TestXAICostCalculator:
|
||||
"""Test suite for XAI cost calculation functionality."""
|
||||
|
||||
def setup_method(self):
|
||||
"""Set up test environment."""
|
||||
os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True"
|
||||
litellm.model_cost = litellm.get_model_cost_map(url="")
|
||||
|
||||
def test_basic_cost_calculation(self):
|
||||
"""Test basic cost calculation without reasoning tokens."""
|
||||
usage = Usage(prompt_tokens=12, completion_tokens=125, total_tokens=137)
|
||||
|
||||
prompt_cost, completion_cost = cost_per_token(model="grok-3-mini", usage=usage)
|
||||
|
||||
# Expected costs for grok-3-mini:
|
||||
# Input: 12 tokens * $3e-7 = $0.0000036
|
||||
# Output: 125 tokens * $5e-7 = $0.0000625
|
||||
expected_prompt_cost = 12 * 3e-7
|
||||
expected_completion_cost = 125 * 5e-7
|
||||
|
||||
assert math.isclose(prompt_cost, expected_prompt_cost, rel_tol=1e-10)
|
||||
assert math.isclose(completion_cost, expected_completion_cost, rel_tol=1e-10)
|
||||
|
||||
def test_reasoning_tokens_cost_calculation(self):
|
||||
"""Test cost calculation with reasoning tokens from completion_tokens_details."""
|
||||
usage = Usage(
|
||||
prompt_tokens=12,
|
||||
completion_tokens=125,
|
||||
total_tokens=1086,
|
||||
completion_tokens_details=CompletionTokensDetailsWrapper(
|
||||
accepted_prediction_tokens=0,
|
||||
audio_tokens=0,
|
||||
reasoning_tokens=949,
|
||||
rejected_prediction_tokens=0,
|
||||
text_tokens=None, # Not set, but doesn't matter for XAI billing
|
||||
),
|
||||
)
|
||||
|
||||
prompt_cost, completion_cost = cost_per_token(model="grok-3-mini", usage=usage)
|
||||
|
||||
# Expected costs for grok-3-mini:
|
||||
# Input: 12 tokens * $3e-7 = $0.0000036
|
||||
# Completion: (125 + 949) tokens * $5e-7 = $0.000537
|
||||
expected_prompt_cost = 12 * 3e-7
|
||||
expected_completion_cost = (125 + 949) * 5e-7
|
||||
|
||||
assert math.isclose(prompt_cost, expected_prompt_cost, rel_tol=1e-10)
|
||||
assert math.isclose(completion_cost, expected_completion_cost, rel_tol=1e-10)
|
||||
|
||||
def test_reasoning_and_text_tokens_cost_calculation(self):
|
||||
"""Test cost calculation with both reasoning and text tokens."""
|
||||
usage = Usage(
|
||||
prompt_tokens=12,
|
||||
completion_tokens=125,
|
||||
total_tokens=1086,
|
||||
completion_tokens_details=CompletionTokensDetailsWrapper(
|
||||
accepted_prediction_tokens=0,
|
||||
audio_tokens=0,
|
||||
reasoning_tokens=949,
|
||||
rejected_prediction_tokens=0,
|
||||
text_tokens=76, # Explicitly set (but ignored in XAI billing)
|
||||
),
|
||||
)
|
||||
|
||||
prompt_cost, completion_cost = cost_per_token(model="grok-3-mini", usage=usage)
|
||||
|
||||
# Expected costs for grok-3-mini:
|
||||
# Input: 12 tokens * $3e-7 = $0.0000036
|
||||
# Completion: (125 + 949) tokens * $5e-7 = $0.000537
|
||||
# Note: text_tokens field is ignored, only completion_tokens + reasoning_tokens matters
|
||||
expected_prompt_cost = 12 * 3e-7
|
||||
expected_completion_cost = (125 + 949) * 5e-7
|
||||
|
||||
assert math.isclose(prompt_cost, expected_prompt_cost, rel_tol=1e-10)
|
||||
assert math.isclose(completion_cost, expected_completion_cost, rel_tol=1e-10)
|
||||
|
||||
def test_grok_4_cost_calculation(self):
|
||||
"""Test cost calculation for grok-4 model."""
|
||||
usage = Usage(
|
||||
prompt_tokens=10,
|
||||
completion_tokens=200,
|
||||
total_tokens=210,
|
||||
completion_tokens_details=CompletionTokensDetailsWrapper(
|
||||
accepted_prediction_tokens=0,
|
||||
audio_tokens=0,
|
||||
reasoning_tokens=150,
|
||||
rejected_prediction_tokens=0,
|
||||
text_tokens=50, # Ignored in XAI billing
|
||||
),
|
||||
)
|
||||
|
||||
prompt_cost, completion_cost = cost_per_token(model="grok-4", usage=usage)
|
||||
|
||||
# Expected costs for grok-4:
|
||||
# Input: 10 tokens * $3e-6 = $0.00003
|
||||
# Completion: (200 + 150) tokens * $1.5e-5 = $0.00525
|
||||
expected_prompt_cost = 10 * 3e-6
|
||||
expected_completion_cost = (200 + 150) * 1.5e-5
|
||||
|
||||
assert math.isclose(prompt_cost, expected_prompt_cost, rel_tol=1e-10)
|
||||
assert math.isclose(completion_cost, expected_completion_cost, rel_tol=1e-10)
|
||||
|
||||
def test_grok_3_fast_beta_cost_calculation(self):
|
||||
"""Test cost calculation for grok-3-fast-beta model."""
|
||||
usage = Usage(
|
||||
prompt_tokens=20,
|
||||
completion_tokens=300,
|
||||
total_tokens=320,
|
||||
completion_tokens_details=CompletionTokensDetailsWrapper(
|
||||
accepted_prediction_tokens=0,
|
||||
audio_tokens=0,
|
||||
reasoning_tokens=200,
|
||||
rejected_prediction_tokens=0,
|
||||
text_tokens=100, # Ignored in XAI billing
|
||||
),
|
||||
)
|
||||
|
||||
prompt_cost, completion_cost = cost_per_token(
|
||||
model="grok-3-fast-beta", usage=usage
|
||||
)
|
||||
|
||||
# Expected costs for grok-3-fast-beta:
|
||||
# Input: 20 tokens * $5e-6 = $0.0001
|
||||
# Completion: (300 + 200) tokens * $2.5e-5 = $0.0125
|
||||
expected_prompt_cost = 20 * 5e-6
|
||||
expected_completion_cost = (300 + 200) * 2.5e-5
|
||||
|
||||
assert math.isclose(prompt_cost, expected_prompt_cost, rel_tol=1e-10)
|
||||
assert math.isclose(completion_cost, expected_completion_cost, rel_tol=1e-10)
|
||||
|
||||
def test_edge_case_no_completion_tokens_details(self):
|
||||
"""Test cost calculation when completion_tokens_details is not present."""
|
||||
usage = Usage(prompt_tokens=12, completion_tokens=125, total_tokens=137)
|
||||
|
||||
prompt_cost, completion_cost = cost_per_token(model="grok-3-mini", usage=usage)
|
||||
|
||||
# Should fall back to basic calculation
|
||||
expected_prompt_cost = 12 * 3e-7
|
||||
expected_completion_cost = 125 * 5e-7
|
||||
|
||||
assert math.isclose(prompt_cost, expected_prompt_cost, rel_tol=1e-10)
|
||||
assert math.isclose(completion_cost, expected_completion_cost, rel_tol=1e-10)
|
||||
|
||||
def test_edge_case_large_reasoning_tokens(self):
|
||||
"""Test cost calculation when reasoning_tokens is larger than completion_tokens."""
|
||||
usage = Usage(
|
||||
prompt_tokens=12,
|
||||
completion_tokens=50, # Less than reasoning_tokens
|
||||
total_tokens=62,
|
||||
completion_tokens_details=CompletionTokensDetailsWrapper(
|
||||
accepted_prediction_tokens=0,
|
||||
audio_tokens=0,
|
||||
reasoning_tokens=100, # More than completion_tokens
|
||||
rejected_prediction_tokens=0,
|
||||
text_tokens=None,
|
||||
),
|
||||
)
|
||||
|
||||
prompt_cost, completion_cost = cost_per_token(model="grok-3-mini", usage=usage)
|
||||
|
||||
# Expected costs:
|
||||
# Input: 12 tokens * $3e-7 = $0.0000036
|
||||
# Completion: (50 + 100) tokens * $5e-7 = $0.000075
|
||||
expected_prompt_cost = 12 * 3e-7
|
||||
expected_completion_cost = (50 + 100) * 5e-7
|
||||
|
||||
assert math.isclose(prompt_cost, expected_prompt_cost, rel_tol=1e-10)
|
||||
assert math.isclose(completion_cost, expected_completion_cost, rel_tol=1e-10)
|
||||
0
tests/test_litellm/proxy/google_endpoints/__init__.py
Normal file
0
tests/test_litellm/proxy/google_endpoints/__init__.py
Normal file
49
tests/test_litellm/proxy/google_endpoints/test_endpoints.py
Normal file
49
tests/test_litellm/proxy/google_endpoints/test_endpoints.py
Normal file
|
|
@ -0,0 +1,49 @@
|
|||
"""
|
||||
Test for google_endpoints/endpoints.py
|
||||
"""
|
||||
import pytest
|
||||
import sys, os
|
||||
from dotenv import load_dotenv
|
||||
|
||||
|
||||
from litellm.proxy.google_endpoints.endpoints import google_count_tokens
|
||||
from litellm.types.llms.vertex_ai import TokenCountDetailsResponse
|
||||
from starlette.requests import Request
|
||||
|
||||
load_dotenv()
|
||||
|
||||
sys.path.insert(
|
||||
0, os.path.abspath("../../../..")
|
||||
)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_proxy_gemini_to_openai_like_model_token_counting():
|
||||
"""
|
||||
Test the token counting endpoint for proxing gemini to openai-like models.
|
||||
"""
|
||||
response: TokenCountDetailsResponse = await google_count_tokens(
|
||||
request=Request(
|
||||
scope={
|
||||
"type": "http",
|
||||
"parsed_body": (
|
||||
[
|
||||
"contents"
|
||||
],
|
||||
{
|
||||
"contents": [
|
||||
{
|
||||
"parts": [
|
||||
{
|
||||
"text": "Hello, how are you?"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
)
|
||||
}
|
||||
),
|
||||
model_name="volcengine/foo",
|
||||
)
|
||||
|
||||
assert response.get("totalTokens") > 0
|
||||
|
|
@ -74,6 +74,96 @@ class TestProxyBaseLLMRequestProcessing:
|
|||
pytest.fail("litellm_call_id is not a valid UUID")
|
||||
assert data_passed["litellm_call_id"] == returned_data["litellm_call_id"]
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_stream_timeout_header_processing(self):
|
||||
"""
|
||||
Test that x-litellm-stream-timeout header gets processed and added to request data as stream_timeout.
|
||||
"""
|
||||
from litellm.proxy.litellm_pre_call_utils import LiteLLMProxyRequestSetup
|
||||
|
||||
# Test with stream timeout header
|
||||
headers_with_timeout = {"x-litellm-stream-timeout": "30.5"}
|
||||
result = LiteLLMProxyRequestSetup._get_stream_timeout_from_request(headers_with_timeout)
|
||||
assert result == 30.5
|
||||
|
||||
# Test without stream timeout header
|
||||
headers_without_timeout = {}
|
||||
result = LiteLLMProxyRequestSetup._get_stream_timeout_from_request(headers_without_timeout)
|
||||
assert result is None
|
||||
|
||||
# Test with invalid header value (should raise ValueError when converting to float)
|
||||
headers_with_invalid = {"x-litellm-stream-timeout": "invalid"}
|
||||
with pytest.raises(ValueError):
|
||||
LiteLLMProxyRequestSetup._get_stream_timeout_from_request(headers_with_invalid)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_add_litellm_data_to_request_with_stream_timeout_header(self):
|
||||
"""
|
||||
Test that x-litellm-stream-timeout header gets processed and added to request data
|
||||
when calling add_litellm_data_to_request.
|
||||
"""
|
||||
from litellm.integrations.opentelemetry import UserAPIKeyAuth
|
||||
from litellm.proxy.litellm_pre_call_utils import add_litellm_data_to_request
|
||||
|
||||
# Create test data with a basic completion request
|
||||
test_data = {
|
||||
"model": "gpt-3.5-turbo",
|
||||
"messages": [{"role": "user", "content": "Hello"}]
|
||||
}
|
||||
|
||||
# Mock request with stream timeout header
|
||||
mock_request = MagicMock(spec=Request)
|
||||
mock_request.headers = {"x-litellm-stream-timeout": "45.0"}
|
||||
mock_request.url.path = "/v1/chat/completions"
|
||||
mock_request.method = "POST"
|
||||
mock_request.query_params = {}
|
||||
mock_request.client = None
|
||||
|
||||
# Create a minimal mock with just the required attributes
|
||||
mock_user_api_key_dict = MagicMock()
|
||||
mock_user_api_key_dict.api_key = "test_api_key_hash"
|
||||
mock_user_api_key_dict.tpm_limit = None
|
||||
mock_user_api_key_dict.rpm_limit = None
|
||||
mock_user_api_key_dict.max_budget = None
|
||||
mock_user_api_key_dict.spend = 0
|
||||
mock_user_api_key_dict.allowed_model_region = None
|
||||
mock_user_api_key_dict.key_alias = None
|
||||
mock_user_api_key_dict.user_id = None
|
||||
mock_user_api_key_dict.team_id = None
|
||||
mock_user_api_key_dict.metadata = {} # Prevent enterprise feature check
|
||||
mock_user_api_key_dict.team_metadata = None
|
||||
mock_user_api_key_dict.org_id = None
|
||||
mock_user_api_key_dict.team_alias = None
|
||||
mock_user_api_key_dict.end_user_id = None
|
||||
mock_user_api_key_dict.user_email = None
|
||||
mock_user_api_key_dict.request_route = None
|
||||
mock_user_api_key_dict.team_max_budget = None
|
||||
mock_user_api_key_dict.team_spend = None
|
||||
mock_user_api_key_dict.model_max_budget = None
|
||||
mock_user_api_key_dict.parent_otel_span = None
|
||||
mock_user_api_key_dict.team_model_aliases = None
|
||||
|
||||
general_settings = {}
|
||||
mock_proxy_config = MagicMock()
|
||||
|
||||
# Call the actual function that processes headers and adds data
|
||||
result_data = await add_litellm_data_to_request(
|
||||
data=test_data,
|
||||
request=mock_request,
|
||||
general_settings=general_settings,
|
||||
user_api_key_dict=mock_user_api_key_dict,
|
||||
version=None,
|
||||
proxy_config=mock_proxy_config,
|
||||
)
|
||||
|
||||
# Verify that stream_timeout was extracted from header and added to request data
|
||||
assert "stream_timeout" in result_data
|
||||
assert result_data["stream_timeout"] == 45.0
|
||||
|
||||
# Verify that the original test data is preserved
|
||||
assert result_data["model"] == "gpt-3.5-turbo"
|
||||
assert result_data["messages"] == [{"role": "user", "content": "Hello"}]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
class TestCommonRequestProcessingHelpers:
|
||||
|
|
|
|||
|
|
@ -979,8 +979,8 @@ class TestProxyFunctionCalling:
|
|||
# Groq models (mixed support)
|
||||
("groq/gemma-7b-it", "litellm_proxy/groq/gemma-7b-it", True),
|
||||
(
|
||||
"groq/llama3-70b-8192",
|
||||
"litellm_proxy/groq/llama3-70b-8192",
|
||||
"groq/llama-3.3-70b-versatile",
|
||||
"litellm_proxy/groq/llama-3.3-70b-versatile",
|
||||
False,
|
||||
), # This model doesn't support function calling
|
||||
# Cohere models (generally don't support function calling)
|
||||
|
|
@ -1051,7 +1051,7 @@ class TestProxyFunctionCalling:
|
|||
("litellm_proxy/claude-prod", "anthropic/claude-3-sonnet-20240229", False),
|
||||
("litellm_proxy/claude-dev", "anthropic/claude-3-haiku-20240307", False),
|
||||
# Groq with custom names (cannot be resolved)
|
||||
("litellm_proxy/fast-llama", "groq/llama3-8b-8192", False),
|
||||
("litellm_proxy/fast-llama", "groq/llama-3.1-8b-instant", False),
|
||||
("litellm_proxy/groq-gemma", "groq/gemma-7b-it", False),
|
||||
# Cohere with custom names (cannot be resolved)
|
||||
("litellm_proxy/cohere-command", "cohere/command-r", False),
|
||||
|
|
|
|||
|
|
@ -550,7 +550,7 @@ async def test_proxy_all_models():
|
|||
async with aiohttp.ClientSession() as session:
|
||||
# call chat/completions with a model that the key was not created for + the model is not on the config.yaml
|
||||
await chat_completion(
|
||||
session=session, key=LITELLM_MASTER_KEY, model="groq/llama3-8b-8192"
|
||||
session=session, key=LITELLM_MASTER_KEY, model="groq/llama-3.1-8b-instant"
|
||||
)
|
||||
|
||||
await chat_completion(
|
||||
|
|
|
|||
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
Some files were not shown because too many files have changed in this diff Show more
Loading…
Add table
Reference in a new issue