diff --git a/deploy/charts/litellm-helm/templates/deployment.yaml b/deploy/charts/litellm-helm/templates/deployment.yaml index 6a5a6e87577..316323be99a 100644 --- a/deploy/charts/litellm-helm/templates/deployment.yaml +++ b/deploy/charts/litellm-helm/templates/deployment.yaml @@ -129,6 +129,10 @@ spec: args: - --config - /etc/litellm/config.yaml + {{ if .Values.numWorkers }} + - --num_workers + - {{ .Values.numWorkers | quote }} + {{- end }} ports: - name: http containerPort: {{ .Values.service.port }} @@ -208,3 +212,8 @@ spec: tolerations: {{- toYaml . | nindent 8 }} {{- end }} + terminationGracePeriodSeconds: {{ .Values.terminationGracePeriodSeconds | default 90 }} + {{- if .Values.topologySpreadConstraints }} + topologySpreadConstraints: + {{- toYaml .Values.topologySpreadConstraints | nindent 8 }} + {{- end }} \ No newline at end of file diff --git a/deploy/charts/litellm-helm/templates/servicemonitor.yaml b/deploy/charts/litellm-helm/templates/servicemonitor.yaml new file mode 100644 index 00000000000..743098deb3f --- /dev/null +++ b/deploy/charts/litellm-helm/templates/servicemonitor.yaml @@ -0,0 +1,39 @@ +{{- with .Values.serviceMonitor }} +{{- if and (eq .enabled true) }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ include "litellm.fullname" $ }} + labels: + {{- include "litellm.labels" $ | nindent 4 }} + {{- if .labels }} + {{- toYaml .labels | nindent 4 }} + {{- end }} + {{- if .annotations }} + annotations: + {{- toYaml .annotations | nindent 4 }} + {{- end }} +spec: + selector: + matchLabels: + {{- include "litellm.selectorLabels" $ | nindent 6 }} + namespaceSelector: + matchNames: + # if not set, use the release namespace + {{- if not .namespaceSelector.matchNames }} + - {{ $.Release.Namespace | quote }} + {{- else }} + {{- toYaml .namespaceSelector.matchNames | nindent 4 }} + {{- end }} + endpoints: + - port: http + path: /metrics/ + interval: {{ .interval }} + scrapeTimeout: {{ .scrapeTimeout }} + scheme: http + {{- if .relabelings }} + relabelings: +{{- toYaml .relabelings | nindent 4 }} + {{- end }} +{{- end }} +{{- end }} diff --git a/deploy/charts/litellm-helm/templates/tests/test-servicemonitor.yaml b/deploy/charts/litellm-helm/templates/tests/test-servicemonitor.yaml new file mode 100644 index 00000000000..c2a4f84ec21 --- /dev/null +++ b/deploy/charts/litellm-helm/templates/tests/test-servicemonitor.yaml @@ -0,0 +1,152 @@ +{{- if .Values.serviceMonitor.enabled }} +apiVersion: v1 +kind: Pod +metadata: + name: "{{ include "litellm.fullname" . }}-test-servicemonitor" + labels: + {{- include "litellm.labels" . | nindent 4 }} + annotations: + "helm.sh/hook": test +spec: + containers: + - name: test + image: bitnami/kubectl:latest + command: ['sh', '-c'] + args: + - | + set -e + echo "🔍 Testing ServiceMonitor configuration..." + + # Check if ServiceMonitor exists + if ! kubectl get servicemonitor {{ include "litellm.fullname" . }} -n {{ .Release.Namespace }} &>/dev/null; then + echo "❌ ServiceMonitor not found" + exit 1 + fi + echo "✅ ServiceMonitor exists" + + # Get ServiceMonitor YAML + SM=$(kubectl get servicemonitor {{ include "litellm.fullname" . }} -n {{ .Release.Namespace }} -o yaml) + + # Test endpoint configuration + ENDPOINT_PORT=$(echo "$SM" | grep -A 5 "endpoints:" | grep "port:" | awk '{print $2}') + if [ "$ENDPOINT_PORT" != "http" ]; then + echo "❌ Endpoint port mismatch. Expected: http, Got: $ENDPOINT_PORT" + exit 1 + fi + echo "✅ Endpoint port is correctly set to: $ENDPOINT_PORT" + + # Test endpoint path + ENDPOINT_PATH=$(echo "$SM" | grep -A 5 "endpoints:" | grep "path:" | awk '{print $2}') + if [ "$ENDPOINT_PATH" != "/metrics/" ]; then + echo "❌ Endpoint path mismatch. Expected: /metrics/, Got: $ENDPOINT_PATH" + exit 1 + fi + echo "✅ Endpoint path is correctly set to: $ENDPOINT_PATH" + + # Test interval + INTERVAL=$(echo "$SM" | grep "interval:" | awk '{print $2}') + if [ "$INTERVAL" != "{{ .Values.serviceMonitor.interval }}" ]; then + echo "❌ Interval mismatch. Expected: {{ .Values.serviceMonitor.interval }}, Got: $INTERVAL" + exit 1 + fi + echo "✅ Interval is correctly set to: $INTERVAL" + + # Test scrapeTimeout + TIMEOUT=$(echo "$SM" | grep "scrapeTimeout:" | awk '{print $2}') + if [ "$TIMEOUT" != "{{ .Values.serviceMonitor.scrapeTimeout }}" ]; then + echo "❌ ScrapeTimeout mismatch. Expected: {{ .Values.serviceMonitor.scrapeTimeout }}, Got: $TIMEOUT" + exit 1 + fi + echo "✅ ScrapeTimeout is correctly set to: $TIMEOUT" + + # Test scheme + SCHEME=$(echo "$SM" | grep "scheme:" | awk '{print $2}') + if [ "$SCHEME" != "http" ]; then + echo "❌ Scheme mismatch. Expected: http, Got: $SCHEME" + exit 1 + fi + echo "✅ Scheme is correctly set to: $SCHEME" + + {{- if .Values.serviceMonitor.labels }} + # Test custom labels + echo "🔍 Checking custom labels..." + {{- range $key, $value := .Values.serviceMonitor.labels }} + LABEL_VALUE=$(echo "$SM" | grep -A 20 "metadata:" | grep "{{ $key }}:" | awk '{print $2}') + if [ "$LABEL_VALUE" != "{{ $value }}" ]; then + echo "❌ Label {{ $key }} mismatch. Expected: {{ $value }}, Got: $LABEL_VALUE" + exit 1 + fi + echo "✅ Label {{ $key }} is correctly set to: {{ $value }}" + {{- end }} + {{- end }} + + {{- if .Values.serviceMonitor.annotations }} + # Test annotations + echo "🔍 Checking annotations..." + {{- range $key, $value := .Values.serviceMonitor.annotations }} + ANNOTATION_VALUE=$(echo "$SM" | grep -A 10 "annotations:" | grep "{{ $key }}:" | awk '{print $2}') + if [ "$ANNOTATION_VALUE" != "{{ $value }}" ]; then + echo "❌ Annotation {{ $key }} mismatch. Expected: {{ $value }}, Got: $ANNOTATION_VALUE" + exit 1 + fi + echo "✅ Annotation {{ $key }} is correctly set to: {{ $value }}" + {{- end }} + {{- end }} + + {{- if .Values.serviceMonitor.namespaceSelector.matchNames }} + # Test namespace selector + echo "🔍 Checking namespace selector..." + {{- range .Values.serviceMonitor.namespaceSelector.matchNames }} + if ! echo "$SM" | grep -A 5 "namespaceSelector:" | grep -q "{{ . }}"; then + echo "❌ Namespace {{ . }} not found in namespaceSelector" + exit 1 + fi + echo "✅ Namespace {{ . }} found in namespaceSelector" + {{- end }} + {{- else }} + # Test default namespace selector (should be release namespace) + if ! echo "$SM" | grep -A 5 "namespaceSelector:" | grep -q "{{ .Release.Namespace }}"; then + echo "❌ Release namespace {{ .Release.Namespace }} not found in namespaceSelector" + exit 1 + fi + echo "✅ Default namespace selector set to release namespace: {{ .Release.Namespace }}" + {{- end }} + + {{- if .Values.serviceMonitor.relabelings }} + # Test relabelings + echo "🔍 Checking relabelings configuration..." + if ! echo "$SM" | grep -q "relabelings:"; then + echo "❌ Relabelings section not found" + exit 1 + fi + echo "✅ Relabelings section exists" + {{- range .Values.serviceMonitor.relabelings }} + {{- if .targetLabel }} + if ! echo "$SM" | grep -A 50 "relabelings:" | grep -q "targetLabel: {{ .targetLabel }}"; then + echo "❌ Relabeling targetLabel {{ .targetLabel }} not found" + exit 1 + fi + echo "✅ Relabeling targetLabel {{ .targetLabel }} found" + {{- end }} + {{- if .action }} + if ! echo "$SM" | grep -A 50 "relabelings:" | grep -q "action: {{ .action }}"; then + echo "❌ Relabeling action {{ .action }} not found" + exit 1 + fi + echo "✅ Relabeling action {{ .action }} found" + {{- end }} + {{- end }} + {{- end }} + + # Test selector labels match the service + echo "🔍 Checking selector labels match service..." + SVC_LABELS=$(kubectl get svc {{ include "litellm.fullname" . }} -n {{ .Release.Namespace }} -o jsonpath='{.metadata.labels}') + echo "Service labels: $SVC_LABELS" + echo "✅ Selector labels validation passed" + + echo "" + echo "🎉 All ServiceMonitor tests passed successfully!" + serviceAccountName: {{ include "litellm.serviceAccountName" . }} + restartPolicy: Never +{{- end }} + diff --git a/deploy/charts/litellm-helm/values.yaml b/deploy/charts/litellm-helm/values.yaml index c1792497d29..acb8c9ca32f 100644 --- a/deploy/charts/litellm-helm/values.yaml +++ b/deploy/charts/litellm-helm/values.yaml @@ -3,6 +3,7 @@ # Declare variables to be passed into your templates. replicaCount: 1 +# numWorkers: 2 image: # Use "ghcr.io/berriai/litellm-database" for optimized image with database @@ -33,6 +34,15 @@ deploymentAnnotations: {} podAnnotations: {} podLabels: {} +terminationGracePeriodSeconds: 90 +topologySpreadConstraints: [] + # - maxSkew: 1 + # topologyKey: kubernetes.io/hostname + # whenUnsatisfiable: DoNotSchedule + # labelSelector: + # matchLabels: + # app: litellm + # At the time of writing, the litellm docker image requires write access to the # filesystem on startup so that prisma can install some dependencies. podSecurityContext: {} @@ -248,3 +258,19 @@ pdb: maxUnavailable: null # e.g. 1 or "20%" annotations: {} labels: {} + +serviceMonitor: + enabled: false + labels: {} + # test: test + annotations: {} + # kubernetes.io/test: test + interval: 15s + scrapeTimeout: 10s + relabelings: [] + # - targetLabel: __meta_kubernetes_pod_node_name + # replacement: $1 + # action: replace + namespaceSelector: + matchNames: [] + # - test-namespace \ No newline at end of file diff --git a/docker/Dockerfile.non_root b/docker/Dockerfile.non_root index 3fa0ab69e3b..2dcb7cb4787 100644 --- a/docker/Dockerfile.non_root +++ b/docker/Dockerfile.non_root @@ -20,27 +20,33 @@ COPY . . ENV LITELLM_NON_ROOT=true # Build Admin UI -RUN mkdir -p /tmp/litellm_ui && \ - npm install -g npm@latest && \ - npm cache clean --force && \ - cd ui/litellm-dashboard && \ - if [ -f "../../enterprise/enterprise_ui/enterprise_colors.json" ]; then \ - cp ../../enterprise/enterprise_ui/enterprise_colors.json ./ui_colors.json; \ - fi && \ - rm -f package-lock.json && \ - npm install --legacy-peer-deps && \ - npm run build && \ - cp -r ./out/* /tmp/litellm_ui/ && \ - cd /tmp/litellm_ui && \ +RUN mkdir -p /tmp/litellm_ui + +RUN npm install -g npm@latest && npm cache clean --force + +RUN cd /app/ui/litellm-dashboard && \ + if [ -f "/app/enterprise/enterprise_ui/enterprise_colors.json" ]; then \ + cp /app/enterprise/enterprise_ui/enterprise_colors.json ./ui_colors.json; \ + fi + +RUN cd /app/ui/litellm-dashboard && rm -f package-lock.json + +RUN cd /app/ui/litellm-dashboard && npm install --legacy-peer-deps + +RUN cd /app/ui/litellm-dashboard && npm run build + +RUN cp -r /app/ui/litellm-dashboard/out/* /tmp/litellm_ui/ + +RUN cd /tmp/litellm_ui && \ for html_file in *.html; do \ if [ "$html_file" != "index.html" ] && [ -f "$html_file" ]; then \ folder_name="${html_file%.html}" && \ mkdir -p "$folder_name" && \ mv "$html_file" "$folder_name/index.html"; \ fi; \ - done && \ - cd /app/ui/litellm-dashboard && \ - rm -rf ./out + done + +RUN cd /app/ui/litellm-dashboard && rm -rf ./out # Build package and wheel dependencies RUN rm -rf dist/* && python -m build && \ diff --git a/docs/my-website/blog/anthropic_opus_4_5_and_advanced_features/index.md b/docs/my-website/blog/anthropic_opus_4_5_and_advanced_features/index.md new file mode 100644 index 00000000000..b545e936186 --- /dev/null +++ b/docs/my-website/blog/anthropic_opus_4_5_and_advanced_features/index.md @@ -0,0 +1,1281 @@ +--- +slug: anthropic_advanced_features +title: "Day 0 Support: Claude 4.5 Opus (+Advanced Features)" +date: 2025-11-25T10:00:00 +authors: + - name: Sameer Kankute + title: SWE @ LiteLLM (LLM Translation) + url: https://www.linkedin.com/in/sameer-kankute/ + image_url: https://media.licdn.com/dms/image/v2/D4D03AQHB_loQYd5gjg/profile-displayphoto-shrink_800_800/profile-displayphoto-shrink_800_800/0/1719137160975?e=1765411200&v=beta&t=c8396f--_lH6Fb_pVvx_jGholPfcl0bvwmNynbNdnII + - name: Krrish Dholakia + title: "CEO, LiteLLM" + url: https://www.linkedin.com/in/krish-d/ + image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg + - name: Ishaan Jaff + title: "CTO, LiteLLM" + url: https://www.linkedin.com/in/reffajnaahsi/ + image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg +tags: [anthropic, claude, tool search, programmatic tool calling, effort, advanced features] +hide_table_of_contents: false +--- + +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +This guide covers Anthropic's latest model (Claude Opus 4.5) and its advanced features now available in LiteLLM: Tool Search, Programmatic Tool Calling, Tool Input Examples, and the Effort Parameter. + +--- + +| Feature | Supported Models | +|---------|-----------------| +| Tool Search | Claude Opus 4.5, Sonnet 4.5 | +| Programmatic Tool Calling | Claude Opus 4.5, Sonnet 4.5 | +| Input Examples | Claude Opus 4.5, Sonnet 4.5 | +| Effort Parameter | Claude Opus 4.5 only | + +Supported Providers: [Anthropic](../../docs/providers/anthropic), [Bedrock](../../docs/providers/bedrock), [Vertex AI](../../docs/providers/vertex_partner#vertex-ai---anthropic-claude). + +## Usage + + + + + +```python +import os +from litellm import completion + +# set env - [OPTIONAL] replace with your anthropic key +os.environ["ANTHROPIC_API_KEY"] = "your-api-key" + +messages = [{"role": "user", "content": "Hey! how's it going?"}] + +## OPENAI /chat/completions API format +response = completion(model="claude-opus-4-5-20251101", messages=messages) +print(response) + +``` + + + + +**1. Setup config.yaml** + +```yaml +model_list: + - model_name: claude-4 ### RECEIVED MODEL NAME ### + litellm_params: # all params accepted by litellm.completion() - https://docs.litellm.ai/docs/completion/input + model: claude-opus-4-5-20251101 ### MODEL NAME sent to `litellm.completion()` ### + api_key: "os.environ/ANTHROPIC_API_KEY" # does os.getenv("ANTHROPIC_API_KEY") +``` + +**2. Start the proxy** + +```bash +litellm --config /path/to/config.yaml +``` + +**3. Test it!** + + + +```bash +curl --location 'http://0.0.0.0:4000/chat/completions' \ +--header 'Content-Type: application/json' \ +--header 'Authorization: Bearer $LITELLM_KEY' \ +--data ' { + "model": "claude-4", + "messages": [ + { + "role": "user", + "content": "what llm are you" + } + ] + } +' +``` + + +```bash +curl --location 'http://0.0.0.0:4000/v1/messages' \ +--header 'Content-Type: application/json' \ +--header 'Authorization: Bearer $LITELLM_KEY' \ +--data ' { + "model": "claude-4", + "max_tokens": 1024, + "messages": [ + { + "role": "user", + "content": "what llm are you" + } + ] + } +' +``` + + + + + +## Usage - Bedrock + +:::info + +LiteLLM uses the boto3 library to authenticate with Bedrock. + +For more ways to authenticate with Bedrock, see the [Bedrock documentation](../../docs/providers/bedrock#authentication). + +::: + + + + + +```python +import os +from litellm import completion + +os.environ["AWS_ACCESS_KEY_ID"] = "" +os.environ["AWS_SECRET_ACCESS_KEY"] = "" +os.environ["AWS_REGION_NAME"] = "" + +## OPENAI /chat/completions API format +response = completion( + model="bedrock/us.anthropic.claude-opus-4-5-20251101-v1:0", + messages=[{ "content": "Hello, how are you?","role": "user"}] +) +``` + + + + +**1. Setup config.yaml** + +```yaml +model_list: + - model_name: claude-4 ### RECEIVED MODEL NAME ### + litellm_params: # all params accepted by litellm.completion() - https://docs.litellm.ai/docs/completion/input + model: bedrock/us.anthropic.claude-opus-4-5-20251101-v1:0 ### MODEL NAME sent to `litellm.completion()` ### + aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID + aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY + aws_region_name: os.environ/AWS_REGION_NAME +``` + +**2. Start the proxy** + +```bash +litellm --config /path/to/config.yaml +``` + +**3. Test it!** + + + +```bash +curl --location 'http://0.0.0.0:4000/chat/completions' \ +--header 'Content-Type: application/json' \ +--header 'Authorization: Bearer $LITELLM_KEY' \ +--data ' { + "model": "claude-4", + "messages": [ + { + "role": "user", + "content": "what llm are you" + } + ] + } +' +``` + + +```bash +curl --location 'http://0.0.0.0:4000/v1/messages' \ +--header 'Content-Type: application/json' \ +--header 'Authorization: Bearer $LITELLM_KEY' \ +--data ' { + "model": "claude-4", + "max_tokens": 1024, + "messages": [ + { + "role": "user", + "content": "what llm are you" + } + ] + } +' +``` + + +```bash +curl --location 'http://0.0.0.0:4000/bedrock/model/claude-4/invoke' \ +--header 'Content-Type: application/json' \ +--header 'Authorization: Bearer $LITELLM_KEY' \ +--data ' { + "max_tokens": 1024, + "messages": [{"role": "user", "content": "Hello, how are you?"}] + }' +``` + + +```bash +curl --location 'http://0.0.0.0:4000/bedrock/model/claude-4/converse' \ +--header 'Content-Type: application/json' \ +--header 'Authorization: Bearer $LITELLM_KEY' \ +--data ' { + "messages": [{"role": "user", "content": "Hello, how are you?"}] + }' +``` + + + + + + +## Usage - Vertex AI + + + + + +```python +from litellm import completion +import json + +## GET CREDENTIALS +## RUN ## +# !gcloud auth application-default login - run this to add vertex credentials to your env +## OR ## +file_path = 'path/to/vertex_ai_service_account.json' + +# Load the JSON file +with open(file_path, 'r') as file: + vertex_credentials = json.load(file) + +# Convert to JSON string +vertex_credentials_json = json.dumps(vertex_credentials) + +## COMPLETION CALL +response = completion( + model="vertex_ai/claude-opus-4-5@20251101", + messages=[{ "content": "Hello, how are you?","role": "user"}], + vertex_credentials=vertex_credentials_json, + vertex_project="your-project-id", + vertex_location="us-east5" +) +``` + + + + +**1. Setup config.yaml** + +```yaml +model_list: + - model_name: claude-4 ### RECEIVED MODEL NAME ### + litellm_params: + model: vertex_ai/claude-opus-4-5@20251101 + vertex_credentials: "/path/to/service_account.json" + vertex_project: "your-project-id" + vertex_location: "us-east5" +``` + +**2. Start the proxy** + +```bash +litellm --config /path/to/config.yaml +``` + +**3. Test it!** + + + +```bash +curl --location 'http://0.0.0.0:4000/chat/completions' \ +--header 'Content-Type: application/json' \ +--header 'Authorization: Bearer $LITELLM_KEY' \ +--data ' { + "model": "claude-4", + "messages": [ + { + "role": "user", + "content": "what llm are you" + } + ] + } +' +``` + + +```bash +curl --location 'http://0.0.0.0:4000/v1/messages' \ +--header 'Content-Type: application/json' \ +--header 'Authorization: Bearer $LITELLM_KEY' \ +--data ' { + "model": "claude-4", + "max_tokens": 1024, + "messages": [ + { + "role": "user", + "content": "what llm are you" + } + ] + } +' +``` + + + + + + + +## Tool Search {#tool-search} + +This lets Claude work with thousands of tools, by dynamically loading tools on-demand, instead of loading all tools into the context window upfront. + +### Usage Example + + + + +```python +import litellm +import os + +# Configure your API key +os.environ["ANTHROPIC_API_KEY"] = "your-api-key" + +# Define your tools with defer_loading +tools = [ + # Tool search tool (regex variant) + { + "type": "tool_search_tool_regex_20251119", + "name": "tool_search_tool_regex" + }, + # Deferred tools - loaded on-demand + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get the current weather in a given location. Returns temperature and conditions.", + "parameters": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA" + }, + "unit": { + "type": "string", + "enum": ["celsius", "fahrenheit"], + "description": "Temperature unit" + } + }, + "required": ["location"] + } + }, + "defer_loading": True # Load on-demand + }, + { + "type": "function", + "function": { + "name": "search_files", + "description": "Search through files in the workspace using keywords", + "parameters": { + "type": "object", + "properties": { + "query": {"type": "string"}, + "file_types": { + "type": "array", + "items": {"type": "string"} + } + }, + "required": ["query"] + } + }, + "defer_loading": True + }, + { + "type": "function", + "function": { + "name": "query_database", + "description": "Execute SQL queries against the database", + "parameters": { + "type": "object", + "properties": { + "sql": {"type": "string"} + }, + "required": ["sql"] + } + }, + "defer_loading": True + } +] + +# Make a request - Claude will search for and use relevant tools +response = litellm.completion( + model="anthropic/claude-opus-4-5-20251101", + messages=[{ + "role": "user", + "content": "What's the weather like in San Francisco?" + }], + tools=tools +) + +print("Claude's response:", response.choices[0].message.content) +print("Tool calls:", response.choices[0].message.tool_calls) + +# Check tool search usage +if hasattr(response.usage, 'server_tool_use'): + print(f"Tool searches performed: {response.usage.server_tool_use.tool_search_requests}") +``` + + + +1. Setup config.yaml + +```yaml +model_list: + - model_name: claude-4 + litellm_params: + model: anthropic/claude-opus-4-5-20251101 + api_key: os.environ/ANTHROPIC_API_KEY +``` + +2. Start the proxy + +```bash +litellm --config /path/to/config.yaml +``` + +3. Test it! + + +```bash +curl --location 'http://0.0.0.0:4000/chat/completions' \ +--header 'Content-Type: application/json' \ +--header 'Authorization: Bearer $LITELLM_KEY' \ +--data ' { + "model": "claude-4", + "messages": [{ + "role": "user", + "content": "What's the weather like in San Francisco?" + }], + "tools": [ + # Tool search tool (regex variant) + { + "type": "tool_search_tool_regex_20251119", + "name": "tool_search_tool_regex" + }, + # Deferred tools - loaded on-demand + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get the current weather in a given location. Returns temperature and conditions.", + "parameters": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA" + }, + "unit": { + "type": "string", + "enum": ["celsius", "fahrenheit"], + "description": "Temperature unit" + } + }, + "required": ["location"] + } + }, + "defer_loading": True # Load on-demand + }, + { + "type": "function", + "function": { + "name": "search_files", + "description": "Search through files in the workspace using keywords", + "parameters": { + "type": "object", + "properties": { + "query": {"type": "string"}, + "file_types": { + "type": "array", + "items": {"type": "string"} + } + }, + "required": ["query"] + } + }, + "defer_loading": True + }, + { + "type": "function", + "function": { + "name": "query_database", + "description": "Execute SQL queries against the database", + "parameters": { + "type": "object", + "properties": { + "sql": {"type": "string"} + }, + "required": ["sql"] + } + }, + "defer_loading": True + } + ] +} +' +``` + + + +### BM25 Variant (Natural Language Search) + +For natural language queries instead of regex patterns: + +```python +tools = [ + { + "type": "tool_search_tool_bm25_20251119", # Natural language variant + "name": "tool_search_tool_bm25" + }, + # ... your deferred tools +] +``` + +--- + +## Programmatic Tool Calling {#programmatic-tool-calling} + +Programmatic tool calling allows Claude to write code that calls your tools programmatically. [Learn more](https://platform.claude.com/docs/en/agents-and-tools/tool-use/programmatic-tool-calling) + + + + +```python +import litellm +import json + +# Define tools that can be called programmatically +tools = [ + # Code execution tool (required for programmatic calling) + { + "type": "code_execution_20250825", + "name": "code_execution" + }, + # Tool that can be called from code + { + "type": "function", + "function": { + "name": "query_database", + "description": "Execute a SQL query against the sales database. Returns a list of rows as JSON objects.", + "parameters": { + "type": "object", + "properties": { + "sql": { + "type": "string", + "description": "SQL query to execute" + } + }, + "required": ["sql"] + } + }, + "allowed_callers": ["code_execution_20250825"] # Enable programmatic calling + } +] + +# First request +response = litellm.completion( + model="anthropic/claude-sonnet-4-5-20250929", + messages=[{ + "role": "user", + "content": "Query sales data for West, East, and Central regions, then tell me which had the highest revenue" + }], + tools=tools +) + +print("Claude's response:", response.choices[0].message) + +# Handle tool calls +messages = [ + {"role": "user", "content": "Query sales data for West, East, and Central regions, then tell me which had the highest revenue"}, + {"role": "assistant", "content": response.choices[0].message.content, "tool_calls": response.choices[0].message.tool_calls} +] + +# Process each tool call +for tool_call in response.choices[0].message.tool_calls: + # Check if it's a programmatic call + if hasattr(tool_call, 'caller') and tool_call.caller: + print(f"Programmatic call to {tool_call.function.name}") + print(f"Called from: {tool_call.caller}") + + # Simulate tool execution + if tool_call.function.name == "query_database": + args = json.loads(tool_call.function.arguments) + # Simulate database query + result = json.dumps([ + {"region": "West", "revenue": 150000}, + {"region": "East", "revenue": 180000}, + {"region": "Central", "revenue": 120000} + ]) + + messages.append({ + "role": "user", + "content": [{ + "type": "tool_result", + "tool_use_id": tool_call.id, + "content": result + }] + }) + +# Get final response +final_response = litellm.completion( + model="anthropic/claude-sonnet-4-5-20250929", + messages=messages, + tools=tools +) + +print("\nFinal answer:", final_response.choices[0].message.content) +``` + + + + +1. Setup config.yaml + +```yaml +model_list: + - model_name: claude-4 + litellm_params: + model: anthropic/claude-opus-4-5-20251101 + api_key: os.environ/ANTHROPIC_API_KEY +``` + +2. Start the proxy + +```bash +litellm --config /path/to/config.yaml +``` + +3. Test it! + + +```bash +curl --location 'http://0.0.0.0:4000/chat/completions' \ +--header 'Content-Type: application/json' \ +--header 'Authorization: Bearer $LITELLM_KEY' \ +--data ' { + "model": "claude-4", + "messages": [{ + "role": "user", + "content": "Query sales data for West, East, and Central regions, then tell me which had the highest revenue" + }], + "tools": [ + # Code execution tool (required for programmatic calling) + { + "type": "code_execution_20250825", + "name": "code_execution" + }, + # Tool that can be called from code + { + "type": "function", + "function": { + "name": "query_database", + "description": "Execute a SQL query against the sales database. Returns a list of rows as JSON objects.", + "parameters": { + "type": "object", + "properties": { + "sql": { + "type": "string", + "description": "SQL query to execute" + } + }, + "required": ["sql"] + } + }, + "allowed_callers": ["code_execution_20250825"] # Enable programmatic calling + } + ] +} +' +``` + + + +--- + +## Tool Input Examples {#tool-input-examples} + +You can now provide Claude with examples of how to use your tools. [Learn more](https://platform.claude.com/docs/en/agents-and-tools/tool-use/tool-input-examples) + + + + + +```python +import litellm + +tools = [ + { + "type": "function", + "function": { + "name": "create_calendar_event", + "description": "Create a new calendar event with attendees and reminders", + "parameters": { + "type": "object", + "properties": { + "title": {"type": "string"}, + "start_time": { + "type": "string", + "description": "ISO 8601 format: YYYY-MM-DDTHH:MM:SS" + }, + "duration_minutes": {"type": "integer"}, + "attendees": { + "type": "array", + "items": { + "type": "object", + "properties": { + "email": {"type": "string"}, + "optional": {"type": "boolean"} + } + } + }, + "reminders": { + "type": "array", + "items": { + "type": "object", + "properties": { + "minutes_before": {"type": "integer"}, + "method": {"type": "string", "enum": ["email", "popup"]} + } + } + } + }, + "required": ["title", "start_time", "duration_minutes"] + } + }, + # Provide concrete examples + "input_examples": [ + { + "title": "Team Standup", + "start_time": "2025-01-15T09:00:00", + "duration_minutes": 30, + "attendees": [ + {"email": "alice@company.com", "optional": False}, + {"email": "bob@company.com", "optional": False} + ], + "reminders": [ + {"minutes_before": 15, "method": "popup"} + ] + }, + { + "title": "Lunch Break", + "start_time": "2025-01-15T12:00:00", + "duration_minutes": 60 + # Demonstrates optional fields can be omitted + } + ] + } +] + +response = litellm.completion( + model="anthropic/claude-sonnet-4-5-20250929", + messages=[{ + "role": "user", + "content": "Schedule a team meeting for tomorrow at 2pm for 45 minutes with john@company.com and sarah@company.com" + }], + tools=tools +) + +print("Tool call:", response.choices[0].message.tool_calls[0].function.arguments) +``` + + + + +1. Setup config.yaml + +```yaml +model_list: + - model_name: claude-4 + litellm_params: + model: anthropic/claude-opus-4-5-20251101 + api_key: os.environ/ANTHROPIC_API_KEY +``` + +2. Start the proxy + +```bash +litellm --config /path/to/config.yaml +``` + +3. Test it! + + +```bash +curl --location 'http://0.0.0.0:4000/chat/completions' \ +--header 'Content-Type: application/json' \ +--header 'Authorization: Bearer $LITELLM_KEY' \ +--data ' { + "model": "claude-4", + "messages": [{ + "role": "user", + "content": "Schedule a team meeting for tomorrow at 2pm for 45 minutes with john@company.com and sarah@company.com" + }], + "tools": [ + { + "type": "function", + "function": { + "name": "create_calendar_event", + "description": "Create a new calendar event with attendees and reminders", + "parameters": { + "type": "object", + "properties": { + "title": {"type": "string"}, + "start_time": { + "type": "string", + "description": "ISO 8601 format: YYYY-MM-DDTHH:MM:SS" + }, + "duration_minutes": {"type": "integer"}, + "attendees": { + "type": "array", + "items": { + "type": "object", + "properties": { + "email": {"type": "string"}, + "optional": {"type": "boolean"} + } + } + }, + "reminders": { + "type": "array", + "items": { + "type": "object", + "properties": { + "minutes_before": {"type": "integer"}, + "method": {"type": "string", "enum": ["email", "popup"]} + } + } + } + }, + "required": ["title", "start_time", "duration_minutes"] + } + }, + # Provide concrete examples + "input_examples": [ + { + "title": "Team Standup", + "start_time": "2025-01-15T09:00:00", + "duration_minutes": 30, + "attendees": [ + {"email": "alice@company.com", "optional": False}, + {"email": "bob@company.com", "optional": False} + ], + "reminders": [ + {"minutes_before": 15, "method": "popup"} + ] + }, + { + "title": "Lunch Break", + "start_time": "2025-01-15T12:00:00", + "duration_minutes": 60 + # Demonstrates optional fields can be omitted + } + ] + } +] +} +' +``` + + + +--- + +## Effort Parameter: Control Token Usage {#effort-parameter} + +Controls aspects like how much effort the model puts into its response, via `output_config={"effort": ..}`. + +:::info + +Soon, we will map OpenAI's `reasoning_effort` parameter to this. +::: + +Potential Values for `effort` parameter: `"high"`, `"medium"`, `"low"`. + +### Usage Example + + + + +```python +import litellm + +message = "Analyze the trade-offs between microservices and monolithic architectures" + +# High effort (default) - Maximum capability +response_high = litellm.completion( + model="anthropic/claude-opus-4-5-20251101", + messages=[{"role": "user", "content": message}], + output_config={"effort": "high"} +) + +print("High effort response:") +print(response_high.choices[0].message.content) +print(f"Tokens used: {response_high.usage.completion_tokens}\n") + +# Medium effort - Balanced approach +response_medium = litellm.completion( + model="anthropic/claude-opus-4-5-20251101", + messages=[{"role": "user", "content": message}], + output_config={"effort": "medium"} +) + +print("Medium effort response:") +print(response_medium.choices[0].message.content) +print(f"Tokens used: {response_medium.usage.completion_tokens}\n") + +# Low effort - Maximum efficiency +response_low = litellm.completion( + model="anthropic/claude-opus-4-5-20251101", + messages=[{"role": "user", "content": message}], + output_config={"effort": "low"} +) + +print("Low effort response:") +print(response_low.choices[0].message.content) +print(f"Tokens used: {response_low.usage.completion_tokens}\n") + +# Compare token usage +print("Token Comparison:") +print(f"High: {response_high.usage.completion_tokens} tokens") +print(f"Medium: {response_medium.usage.completion_tokens} tokens") +print(f"Low: {response_low.usage.completion_tokens} tokens") +``` + + + + +1. Setup config.yaml + +```yaml +model_list: + - model_name: claude-4 + litellm_params: + model: anthropic/claude-opus-4-5-20251101 + api_key: os.environ/ANTHROPIC_API_KEY +``` + +2. Start the proxy + +```bash +litellm --config /path/to/config.yaml +``` + +3. Test it! + +```bash +curl --location 'http://0.0.0.0:4000/chat/completions' \ +--header 'Content-Type: application/json' \ +--header 'Authorization: Bearer $LITELLM_KEY' \ +--data ' { + "model": "claude-4", + "messages": [{ + "role": "user", + "content": "Analyze the trade-offs between microservices and monolithic architectures" + }], + "output_config": { + "effort": "high" + } + } +' +``` + + + + +## Cost Tracking: Monitor Tool Search Usage {#cost-tracking} + +### Understanding Tool Search Costs + +Tool search operations are tracked separately in the usage object, allowing you to monitor and optimize costs. + +It is available in the `usage` object, under `server_tool_use.tool_search_requests`. + +Anthropic charges $0.0001 per tool search request. + +### Tracking Example + + + + +```python +import litellm + +tools = [ + { + "type": "tool_search_tool_regex_20251119", + "name": "tool_search_tool_regex" + }, + # ... 100 deferred tools +] + +response = litellm.completion( + model="anthropic/claude-sonnet-4-5-20250929", + messages=[{ + "role": "user", + "content": "Find and use the weather tool for San Francisco" + }], + tools=tools +) + +# Standard token usage +print("Token Usage:") +print(f" Input tokens: {response.usage.prompt_tokens}") +print(f" Output tokens: {response.usage.completion_tokens}") +print(f" Total tokens: {response.usage.total_tokens}") + +# Tool search specific usage +if hasattr(response.usage, 'server_tool_use') and response.usage.server_tool_use: + print(f"\nTool Search Usage:") + print(f" Search requests: {response.usage.server_tool_use.tool_search_requests}") + + # Calculate cost (example pricing) + input_cost = response.usage.prompt_tokens * 0.000003 # $3 per 1M tokens + output_cost = response.usage.completion_tokens * 0.000015 # $15 per 1M tokens + search_cost = response.usage.server_tool_use.tool_search_requests * 0.0001 # Example + + total_cost = input_cost + output_cost + search_cost + + print(f"\nCost Breakdown:") + print(f" Input tokens: ${input_cost:.6f}") + print(f" Output tokens: ${output_cost:.6f}") + print(f" Tool searches: ${search_cost:.6f}") + print(f" Total: ${total_cost:.6f}") +``` + + + + +1. Setup config.yaml + +```yaml +model_list: + - model_name: claude-4 + litellm_params: + model: anthropic/claude-opus-4-5-20251101 + api_key: os.environ/ANTHROPIC_API_KEY +``` + +2. Start the proxy + +```bash +litellm --config /path/to/config.yaml +``` + +3. Test it! + +```bash +curl --location 'http://0.0.0.0:4000/chat/completions' \ +--header 'Content-Type: application/json' \ +--header 'Authorization: Bearer $LITELLM_KEY' \ +--data ' { + "model": "claude-4", + "messages": [{ + "role": "user", + "content": "Find and use the weather tool for San Francisco" + }], + "tools": [ + { + "type": "tool_search_tool_regex_20251119", + "name": "tool_search_tool_regex" + }, + # ... 100 deferred tools + ] + } +' +``` + +Expected Response: + +```json +{ + ..., + "usage": { + ..., + "server_tool_use": { + "tool_search_requests": 1 + } + } +} +``` + + + + +### Cost Optimization Tips + +1. **Keep frequently used tools non-deferred** (3-5 tools) +2. **Use tool search for large catalogs** (10+ tools) +3. **Monitor search requests** to identify optimization opportunities +4. **Combine with effort parameter** for maximum efficiency + + +--- + +## Combining Features {#combining-features} + +### The Power of Integration + +These features work together seamlessly. Here's a real-world example combining all of them: + + + + +```python +import litellm +import json + +# Large tool catalog with search, programmatic calling, and examples +tools = [ + # Enable tool search + { + "type": "tool_search_tool_regex_20251119", + "name": "tool_search_tool_regex" + }, + # Enable programmatic calling + { + "type": "code_execution_20250825", + "name": "code_execution" + }, + # Database tool with all features + { + "type": "function", + "function": { + "name": "query_database", + "description": "Execute SQL queries against the analytics database. Returns JSON array of results.", + "parameters": { + "type": "object", + "properties": { + "sql": { + "type": "string", + "description": "SQL SELECT statement" + }, + "limit": { + "type": "integer", + "description": "Maximum rows to return" + } + }, + "required": ["sql"] + } + }, + "defer_loading": True, # Tool search + "allowed_callers": ["code_execution_20250825"], # Programmatic calling + "input_examples": [ # Input examples + { + "sql": "SELECT region, SUM(revenue) as total FROM sales GROUP BY region", + "limit": 100 + } + ] + }, + # ... 50 more tools with defer_loading +] + +# Make request with effort control +response = litellm.completion( + model="anthropic/claude-opus-4-5-20251101", + messages=[{ + "role": "user", + "content": "Analyze sales by region for the last quarter and identify top performers" + }], + tools=tools, + output_config={"effort": "medium"} # Balanced efficiency +) + +# Track comprehensive usage +print("Complete Usage Metrics:") +print(f" Input tokens: {response.usage.prompt_tokens}") +print(f" Output tokens: {response.usage.completion_tokens}") +print(f" Total tokens: {response.usage.total_tokens}") + +if hasattr(response.usage, 'server_tool_use') and response.usage.server_tool_use: + print(f" Tool searches: {response.usage.server_tool_use.tool_search_requests}") + +print(f"\nResponse: {response.choices[0].message.content}") +``` + + + + +1. Setup config.yaml + +```yaml +model_list: + - model_name: claude-4 + litellm_params: + model: anthropic/claude-opus-4-5-20251101 + api_key: os.environ/ANTHROPIC_API_KEY +``` + +2. Start the proxy + +```bash +litellm --config /path/to/config.yaml +``` + +3. Test it! + +```bash +curl --location 'http://0.0.0.0:4000/chat/completions' \ +--header 'Content-Type: application/json' \ +--header 'Authorization: Bearer $LITELLM_KEY' \ +--data ' { + "model": "claude-4", + "messages": [{ + "role": "user", + "content": "Analyze sales by region for the last quarter and identify top performers" + }], + "tools": [ + { + "type": "tool_search_tool_regex_20251119", + "name": "tool_search_tool_regex" + }, + # ... 100 deferred tools + ], + "output_config": { + "effort": "medium" + } + } +' +``` + +Expected Response: + +```json +{ + ..., + "usage": { + ..., + "server_tool_use": { + "tool_search_requests": 1 + } + } +} +``` + + + + +### Real-World Benefits + +This combination enables: + +1. **Massive scale** - Handle 1000+ tools efficiently +2. **Low latency** - Programmatic calling reduces round trips +3. **High accuracy** - Input examples ensure correct tool usage +4. **Cost control** - Effort parameter optimizes token spend +5. **Full visibility** - Track all usage metrics + diff --git a/docs/my-website/docs/completion/image_generation_chat.md b/docs/my-website/docs/completion/image_generation_chat.md index 5538b7f8ff3..83488ac7ce8 100644 --- a/docs/my-website/docs/completion/image_generation_chat.md +++ b/docs/my-website/docs/completion/image_generation_chat.md @@ -224,8 +224,8 @@ asyncio.run(generate_image()) | Provider | Model | |----------|--------| -| Google AI Studio | `gemini/gemini-2.0-flash-preview-image-generation`, `gemini/gemini-2.5-flash-image-preview` | -| Vertex AI | `vertex_ai/gemini-2.0-flash-preview-image-generation`, `vertex_ai/gemini-2.5-flash-image-preview` | +| Google AI Studio | `gemini/gemini-2.0-flash-preview-image-generation`, `gemini/gemini-2.5-flash-image-preview`, `gemini/gemini-3-pro-image-preview` | +| Vertex AI | `vertex_ai/gemini-2.0-flash-preview-image-generation`, `vertex_ai/gemini-2.5-flash-image-preview`, `vertex_ai/gemini-3-pro-image-preview` | ## Spec diff --git a/docs/my-website/docs/completion/knowledgebase.md b/docs/my-website/docs/completion/knowledgebase.md index 3040f7f1cc0..759f7912d87 100644 --- a/docs/my-website/docs/completion/knowledgebase.md +++ b/docs/my-website/docs/completion/knowledgebase.md @@ -20,6 +20,7 @@ LiteLLM integrates with vector stores, allowing your models to access your organ - [OpenAI Vector Stores](https://platform.openai.com/docs/api-reference/vector-stores/search) - [Azure Vector Stores](https://learn.microsoft.com/en-us/azure/ai-services/openai/how-to/file-search?tabs=python#vector-stores) (Cannot be directly queried. Only available for calling in Assistants messages. We will be adding Azure AI Search Vector Store API support soon.) - [Vertex AI RAG API](https://cloud.google.com/vertex-ai/generative-ai/docs/rag-overview) +- [Gemini File Search](https://ai.google.dev/gemini-api/docs/file-search) ## Quick Start diff --git a/docs/my-website/docs/mcp.md b/docs/my-website/docs/mcp.md index 887c5278144..a9f7e249133 100644 --- a/docs/my-website/docs/mcp.md +++ b/docs/my-website/docs/mcp.md @@ -248,6 +248,41 @@ mcp_servers: X-Custom-Header: "some-value" ``` +### MCP Walkthroughs + +- **Strands (STDIO)** – [watch tutorial](https://screen.studio/share/ruv4D73F) + +> Add it from the UI + +```json title="strands-mcp" showLineNumbers +{ + "mcpServers": { + "strands-agents": { + "command": "uvx", + "args": ["strands-agents-mcp-server"], + "env": { + "FASTMCP_LOG_LEVEL": "INFO" + }, + "disabled": false, + "autoApprove": ["search_docs", "fetch_doc"] + } + } +} +``` + +> config.yml + +```yaml title="config.yml – strands MCP" showLineNumbers +mcp_servers: + strands_mcp: + transport: "stdio" + command: "uvx" + args: ["strands-agents-mcp-server"] + env: + FASTMCP_LOG_LEVEL: "INFO" +``` + + ### MCP Aliases You can define aliases for your MCP servers in the `litellm_settings` section. This allows you to: diff --git a/docs/my-website/docs/observability/custom_callback.md b/docs/my-website/docs/observability/custom_callback.md index cfe97ca42c0..ae892621270 100644 --- a/docs/my-website/docs/observability/custom_callback.md +++ b/docs/my-website/docs/observability/custom_callback.md @@ -203,7 +203,11 @@ asyncio.run(test_chat_openai()) ## What's Available in kwargs? -The kwargs dictionary contains all the details about your API call: +The kwargs dictionary contains all the details about your API call. + +:::info +For the complete logging payload specification, see the [Standard Logging Payload Spec](https://docs.litellm.ai/docs/proxy/logging_spec). +::: ```python def custom_callback(kwargs, completion_response, start_time, end_time): diff --git a/docs/my-website/docs/providers/anthropic.md b/docs/my-website/docs/providers/anthropic.md index 45edd184d5d..24365f0cc47 100644 --- a/docs/my-website/docs/providers/anthropic.md +++ b/docs/my-website/docs/providers/anthropic.md @@ -18,11 +18,11 @@ LiteLLM supports all anthropic models. | Property | Details | |-------|-------| -| Description | Claude is a highly performant, trustworthy, and intelligent AI platform built by Anthropic. Claude excels at tasks involving language, reasoning, analysis, coding, and more. | -| Provider Route on LiteLLM | `anthropic/` (add this prefix to the model name, to route any requests to Anthropic - e.g. `anthropic/claude-3-5-sonnet-20240620`) | -| Provider Doc | [Anthropic ↗](https://docs.anthropic.com/en/docs/build-with-claude/overview) | -| API Endpoint for Provider | https://api.anthropic.com | -| Supported Endpoints | `/chat/completions` | +| Description | Claude is a highly performant, trustworthy, and intelligent AI platform built by Anthropic. Claude excels at tasks involving language, reasoning, analysis, coding, and more. Also available via Azure Foundry. | +| Provider Route on LiteLLM | `anthropic/` (add this prefix to the model name, to route any requests to Anthropic - e.g. `anthropic/claude-3-5-sonnet-20240620`). For Azure Foundry deployments, use `azure/claude-*` (see [Azure Anthropic documentation](../providers/azure/azure_anthropic)) | +| Provider Doc | [Anthropic ↗](https://docs.anthropic.com/en/docs/build-with-claude/overview), [Azure Foundry Claude ↗](https://learn.microsoft.com/en-us/azure/ai-services/foundry-models/claude) | +| API Endpoint for Provider | https://api.anthropic.com (or Azure Foundry endpoint: `https://.services.ai.azure.com/anthropic`) | +| Supported Endpoints | `/chat/completions`, `/v1/messages` (passthrough) | ## Supported OpenAI Parameters @@ -163,6 +163,22 @@ os.environ["ANTHROPIC_API_KEY"] = "your-api-key" # os.environ["LITELLM_ANTHROPIC_DISABLE_URL_SUFFIX"] = "true" # [OPTIONAL] Disable automatic URL suffix appending ``` +:::tip Azure Foundry Support + +Claude models are also available via Microsoft Azure Foundry. Use the `azure/` prefix instead of `anthropic/` and configure Azure authentication. See the [Azure Anthropic documentation](../providers/azure/azure_anthropic) for details. + +Example: +```python +response = completion( + model="azure/claude-sonnet-4-5", + api_base="https://.services.ai.azure.com/anthropic", + api_key="your-azure-api-key", + messages=[{"role": "user", "content": "Hello!"}] +) +``` + +::: + ### Custom API Base When using a custom API base for Anthropic (e.g., a proxy or custom endpoint), LiteLLM automatically appends the appropriate suffix (`/v1/messages` or `/v1/complete`) to your base URL. diff --git a/docs/my-website/docs/providers/anthropic_effort.md b/docs/my-website/docs/providers/anthropic_effort.md new file mode 100644 index 00000000000..0015162a95b --- /dev/null +++ b/docs/my-website/docs/providers/anthropic_effort.md @@ -0,0 +1,279 @@ +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +# Anthropic Effort Parameter + +Control how many tokens Claude uses when responding with the `effort` parameter, trading off between response thoroughness and token efficiency. + +## Overview + +The `effort` parameter allows you to control how eager Claude is about spending tokens when responding to requests. This gives you the ability to trade off between response thoroughness and token efficiency, all with a single model. + +**Note**: The effort parameter is currently in beta and only supported by Claude Opus 4.5. You must include the beta header `effort-2025-11-24` when using this feature (LiteLLM automatically adds this header when `output_config` with `effort` is detected). + +## How Effort Works + +By default, Claude uses maximum effort—spending as many tokens as needed for the best possible outcome. By lowering the effort level, you can instruct Claude to be more conservative with token usage, optimizing for speed and cost while accepting some reduction in capability. + +**Tip**: Setting `effort` to `"high"` produces exactly the same behavior as omitting the `effort` parameter entirely. + +The effort parameter affects **all tokens** in the response, including: +- Text responses and explanations +- Tool calls and function arguments +- Extended thinking (when enabled) + +This approach has two major advantages: +1. It doesn't require thinking to be enabled in order to use it. +2. It can affect all token spend including tool calls. For example, lower effort would mean Claude makes fewer tool calls. + +This gives a much greater degree of control over efficiency. + +## Effort Levels + +| Level | Description | Typical use case | +|-------|-------------|------------------| +| `high` | Maximum capability—Claude uses as many tokens as needed for the best possible outcome. Equivalent to not setting the parameter. | Complex reasoning, difficult coding problems, agentic tasks | +| `medium` | Balanced approach with moderate token savings. | Agentic tasks that require a balance of speed, cost, and performance | +| `low` | Most efficient—significant token savings with some capability reduction. | Simpler tasks that need the best speed and lowest costs, such as subagents | + +## Quick Start + +### Using LiteLLM SDK + + + + +```python +import litellm + +response = litellm.completion( + model="anthropic/claude-opus-4-5-20251101", + messages=[{ + "role": "user", + "content": "Analyze the trade-offs between microservices and monolithic architectures" + }], + output_config={ + "effort": "medium" + } +) + +print(response.choices[0].message.content) +``` + + + + +```typescript +import Anthropic from "@anthropic-ai/sdk"; + +const client = new Anthropic({ + apiKey: process.env.ANTHROPIC_API_KEY, +}); + +const response = await client.messages.create({ + model: "claude-opus-4-5-20251101", + max_tokens: 4096, + messages: [{ + role: "user", + content: "Analyze the trade-offs between microservices and monolithic architectures" + }], + output_config: { + effort: "medium" + } +}); + +console.log(response.content[0].text); +``` + + + + +### Using LiteLLM Proxy + +```bash +curl http://localhost:4000/v1/chat/completions \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer $LITELLM_API_KEY" \ + -d '{ + "model": "anthropic/claude-opus-4-5-20251101", + "messages": [{ + "role": "user", + "content": "Analyze the trade-offs between microservices and monolithic architectures" + }], + "output_config": { + "effort": "medium" + } + }' +``` + +### Direct Anthropic API Call + +```bash +curl https://api.anthropic.com/v1/messages \ + --header "x-api-key: $ANTHROPIC_API_KEY" \ + --header "anthropic-version: 2023-06-01" \ + --header "anthropic-beta: effort-2025-11-24" \ + --header "content-type: application/json" \ + --data '{ + "model": "claude-opus-4-5-20251101", + "max_tokens": 4096, + "messages": [{ + "role": "user", + "content": "Analyze the trade-offs between microservices and monolithic architectures" + }], + "output_config": { + "effort": "medium" + } + }' +``` + +## Model Compatibility + +The effort parameter is currently only supported by: +- **Claude Opus 4.5** (`claude-opus-4-5-20251101`) + +## When Should I Adjust the Effort Parameter? + +- Use **high effort** (the default) when you need Claude's best work—complex reasoning, nuanced analysis, difficult coding problems, or any task where quality is the top priority. + +- Use **medium effort** as a balanced option when you want solid performance without the full token expenditure of high effort. + +- Use **low effort** when you're optimizing for speed (because Claude answers with fewer tokens) or cost—for example, simple classification tasks, quick lookups, or high-volume use cases where marginal quality improvements don't justify additional latency or spend. + +## Effort with Tool Use + +When using tools, the effort parameter affects both the explanations around tool calls and the tool calls themselves. Lower effort levels tend to: +- Combine multiple operations into fewer tool calls +- Make fewer tool calls +- Proceed directly to action + +Example with tools: + +```python +import litellm + +response = litellm.completion( + model="anthropic/claude-opus-4-5-20251101", + messages=[{ + "role": "user", + "content": "Check the weather in multiple cities" + }], + tools=[{ + "type": "function", + "function": { + "name": "get_weather", + "description": "Get weather for a location", + "parameters": { + "type": "object", + "properties": { + "location": {"type": "string"} + }, + "required": ["location"] + } + } + }], + output_config={ + "effort": "low" # Will make fewer tool calls + } +) +``` + +## Effort with Extended Thinking + +The effort parameter works seamlessly with extended thinking. When both are enabled, effort controls the token budget across all response types: + +```python +import litellm + +response = litellm.completion( + model="anthropic/claude-opus-4-5-20251101", + messages=[{ + "role": "user", + "content": "Solve this complex problem" + }], + thinking={ + "type": "enabled", + "budget_tokens": 5000 + }, + output_config={ + "effort": "medium" # Affects both thinking and response tokens + } +) +``` + +## Best Practices + +1. **Start with the default (high)** for new tasks, then experiment with lower effort levels if you're looking to optimize costs. + +2. **Use medium effort for production agentic workflows** where you need a balance of quality and efficiency. + +3. **Reserve low effort for high-volume, simple tasks** like classification, routing, or data extraction where speed matters more than nuanced responses. + +4. **Monitor token usage** to understand the actual savings from different effort levels for your specific use cases. + +5. **Test with your specific prompts** as the impact of effort levels can vary based on task complexity. + +## Provider Support + +The effort parameter is supported across all Anthropic-compatible providers: + +- **Standard Anthropic**: ✅ Supported (Claude Opus 4.5) +- **Azure Anthropic**: ✅ Supported (Claude Opus 4.5) +- **Vertex AI Anthropic**: ✅ Supported (Claude Opus 4.5) + +LiteLLM automatically handles the beta header injection for all providers. + +## Usage and Pricing + +Token usage with different effort levels is tracked in the standard usage object. Lower effort levels result in fewer output tokens, which directly reduces costs: + +```python +response = litellm.completion( + model="anthropic/claude-opus-4-5-20251101", + messages=[{"role": "user", "content": "Analyze this"}], + output_config={"effort": "low"} +) + +print(f"Output tokens: {response.usage.completion_tokens}") +print(f"Total tokens: {response.usage.total_tokens}") +``` + +## Troubleshooting + +### Beta header not being added + +LiteLLM automatically adds the `effort-2025-11-24` beta header when `output_config` with `effort` is detected. If you're not seeing the header: + +1. Ensure you're using `output_config` with an `effort` field +2. Verify the model is Claude Opus 4.5 +3. Check that LiteLLM version supports this feature + +### Invalid effort value error + +Only three values are accepted: `"high"`, `"medium"`, `"low"`. Any other value will raise a validation error: + +```python +# ❌ This will raise an error +output_config={"effort": "very_low"} + +# ✅ Use one of the valid values +output_config={"effort": "low"} +``` + +### Model not supported + +Currently, only Claude Opus 4.5 supports the effort parameter. Using it with other models may result in the parameter being ignored or an error. + +## Related Features + +- [Extended Thinking](/docs/providers/anthropic_extended_thinking) - Control Claude's reasoning process +- [Tool Use](/docs/providers/anthropic_tools) - Enable Claude to use tools and functions +- [Programmatic Tool Calling](/docs/providers/anthropic_programmatic_tool_calling) - Let Claude write code that calls tools +- [Prompt Caching](/docs/providers/anthropic_prompt_caching) - Cache prompts to reduce costs + +## Additional Resources + +- [Anthropic Effort Documentation](https://docs.anthropic.com/en/docs/build-with-claude/effort) +- [LiteLLM Anthropic Provider Guide](/docs/providers/anthropic) +- [Cost Optimization Best Practices](/docs/guides/cost_optimization) + diff --git a/docs/my-website/docs/providers/anthropic_programmatic_tool_calling.md b/docs/my-website/docs/providers/anthropic_programmatic_tool_calling.md new file mode 100644 index 00000000000..6d3e15785e5 --- /dev/null +++ b/docs/my-website/docs/providers/anthropic_programmatic_tool_calling.md @@ -0,0 +1,430 @@ +# Anthropic Programmatic Tool Calling + +Programmatic tool calling allows Claude to write code that calls your tools programmatically within a code execution container, rather than requiring round trips through the model for each tool invocation. This reduces latency for multi-tool workflows and decreases token consumption by allowing Claude to filter or process data before it reaches the model's context window. + +:::info +Programmatic tool calling is currently in public beta. LiteLLM automatically adds the required `advanced-tool-use-2025-11-20` beta header when it detects tools with the `allowed_callers` field. + +This feature requires the code execution tool to be enabled. +::: + +## Model Compatibility + +Programmatic tool calling is available on the following models: + +| Model | Tool Version | +|-------|--------------| +| Claude Opus 4.5 (`claude-opus-4-5-20251101`) | `code_execution_20250825` | +| Claude Sonnet 4.5 (`claude-sonnet-4-5-20250929`) | `code_execution_20250825` | + +## Quick Start + +Here's a simple example where Claude programmatically queries a database multiple times and aggregates results: + +```python +import litellm + +response = litellm.completion( + model="anthropic/claude-sonnet-4-5-20250929", + messages=[ + { + "role": "user", + "content": "Query sales data for the West, East, and Central regions, then tell me which region had the highest revenue" + } + ], + tools=[ + { + "type": "code_execution_20250825", + "name": "code_execution" + }, + { + "type": "function", + "function": { + "name": "query_database", + "description": "Execute a SQL query against the sales database. Returns a list of rows as JSON objects.", + "parameters": { + "type": "object", + "properties": { + "sql": { + "type": "string", + "description": "SQL query to execute" + } + }, + "required": ["sql"] + } + }, + "allowed_callers": ["code_execution_20250825"] + } + ] +) + +print(response) +``` + +## How It Works + +When you configure a tool to be callable from code execution and Claude decides to use that tool: + +1. Claude writes Python code that invokes the tool as a function, potentially including multiple tool calls and pre/post-processing logic +2. Claude runs this code in a sandboxed container via code execution +3. When a tool function is called, code execution pauses and the API returns a `tool_use` block with a `caller` field +4. You provide the tool result, and code execution continues (intermediate results are not loaded into Claude's context window) +5. Once all code execution completes, Claude receives the final output and continues working on the task + +This approach is particularly useful for: + +- **Large data processing**: Filter or aggregate tool results before they reach Claude's context +- **Multi-step workflows**: Save tokens and latency by calling tools serially or in a loop without sampling Claude in-between tool calls +- **Conditional logic**: Make decisions based on intermediate tool results + +## The `allowed_callers` Field + +The `allowed_callers` field specifies which contexts can invoke a tool: + +```python +{ + "type": "function", + "function": { + "name": "query_database", + "description": "Execute a SQL query against the database", + "parameters": {...} + }, + "allowed_callers": ["code_execution_20250825"] +} +``` + +**Possible values:** + +- `["direct"]` - Only Claude can call this tool directly (default if omitted) +- `["code_execution_20250825"]` - Only callable from within code execution +- `["direct", "code_execution_20250825"]` - Callable both directly and from code execution + +:::tip +We recommend choosing either `["direct"]` or `["code_execution_20250825"]` for each tool rather than enabling both, as this provides clearer guidance to Claude for how best to use the tool. +::: + +## The `caller` Field in Responses + +Every tool use block includes a `caller` field indicating how it was invoked: + +**Direct invocation (traditional tool use):** + +```python +{ + "type": "tool_use", + "id": "toolu_abc123", + "name": "query_database", + "input": {"sql": ""}, + "caller": {"type": "direct"} +} +``` + +**Programmatic invocation:** + +```python +{ + "type": "tool_use", + "id": "toolu_xyz789", + "name": "query_database", + "input": {"sql": ""}, + "caller": { + "type": "code_execution_20250825", + "tool_id": "srvtoolu_abc123" + } +} +``` + +The `tool_id` references the code execution tool that made the programmatic call. + +## Container Lifecycle + +Programmatic tool calling uses code execution containers: + +- **Container creation**: A new container is created for each session unless you reuse an existing one +- **Expiration**: Containers expire after approximately 4.5 minutes of inactivity (subject to change) +- **Container ID**: Pass the `container` parameter to reuse an existing container +- **Reuse**: Pass the container ID to maintain state across requests + +```python +# First request - creates a new container +response1 = litellm.completion( + model="anthropic/claude-sonnet-4-5-20250929", + messages=[{"role": "user", "content": "Query the database"}], + tools=[...] +) + +# Get container ID from response (if available in response metadata) +container_id = response1.get("container", {}).get("id") + +# Second request - reuse the same container +response2 = litellm.completion( + model="anthropic/claude-sonnet-4-5-20250929", + messages=[...], + tools=[...], + container=container_id # Reuse container +) +``` + +:::warning +When a tool is called programmatically and the container is waiting for your tool result, you must respond before the container expires. Monitor the `expires_at` field. If the container expires, Claude may treat the tool call as timed out and retry it. +::: + +## Example Workflow + +### Step 1: Initial Request + +```python +import litellm + +response = litellm.completion( + model="anthropic/claude-sonnet-4-5-20250929", + messages=[{ + "role": "user", + "content": "Query customer purchase history from the last quarter and identify our top 5 customers by revenue" + }], + tools=[ + { + "type": "code_execution_20250825", + "name": "code_execution" + }, + { + "type": "function", + "function": { + "name": "query_database", + "description": "Execute a SQL query against the sales database. Returns a list of rows as JSON objects.", + "parameters": { + "type": "object", + "properties": { + "sql": {"type": "string", "description": "SQL query to execute"} + }, + "required": ["sql"] + } + }, + "allowed_callers": ["code_execution_20250825"] + } + ] +) +``` + +### Step 2: API Response with Tool Call + +Claude writes code that calls your tool. The response includes: + +```python +{ + "role": "assistant", + "content": [ + { + "type": "text", + "text": "I'll query the purchase history and analyze the results." + }, + { + "type": "server_tool_use", + "id": "srvtoolu_abc123", + "name": "code_execution", + "input": { + "code": "results = await query_database('')\ntop_customers = sorted(results, key=lambda x: x['revenue'], reverse=True)[:5]" + } + }, + { + "type": "tool_use", + "id": "toolu_def456", + "name": "query_database", + "input": {"sql": ""}, + "caller": { + "type": "code_execution_20250825", + "tool_id": "srvtoolu_abc123" + } + } + ], + "stop_reason": "tool_use" +} +``` + +### Step 3: Provide Tool Result + +```python +# Add assistant's response and tool result to conversation +messages = [ + {"role": "user", "content": "Query customer purchase history..."}, + { + "role": "assistant", + "content": response.choices[0].message.content, + "tool_calls": response.choices[0].message.tool_calls + }, + { + "role": "user", + "content": [ + { + "type": "tool_result", + "tool_use_id": "toolu_def456", + "content": '[{"customer_id": "C1", "revenue": 45000}, ...]' + } + ] + } +] + +# Continue the conversation +response2 = litellm.completion( + model="anthropic/claude-sonnet-4-5-20250929", + messages=messages, + tools=[...] +) +``` + +### Step 4: Final Response + +Once code execution completes, Claude provides the final response: + +```python +{ + "content": [ + { + "type": "code_execution_tool_result", + "tool_use_id": "srvtoolu_abc123", + "content": { + "type": "code_execution_result", + "stdout": "Top 5 customers by revenue:\n1. Customer C1: $45,000\n...", + "stderr": "", + "return_code": 0 + } + }, + { + "type": "text", + "text": "I've analyzed the purchase history from last quarter. Your top 5 customers generated $167,500 in total revenue..." + } + ], + "stop_reason": "end_turn" +} +``` + +## Advanced Patterns + +### Batch Processing with Loops + +Claude can write code that processes multiple items efficiently: + +```python +# Claude writes code like this: +regions = ["West", "East", "Central", "North", "South"] +results = {} +for region in regions: + data = await query_database(f"SELECT SUM(revenue) FROM sales WHERE region='{region}'") + results[region] = data[0]["total"] + +top_region = max(results.items(), key=lambda x: x[1]) +print(f"Top region: {top_region[0]} with ${top_region[1]:,}") +``` + +This pattern: +- Reduces model round-trips from N (one per region) to 1 +- Processes large result sets programmatically before returning to Claude +- Saves tokens by only returning aggregated conclusions + +### Early Termination + +Claude can stop processing as soon as success criteria are met: + +```python +endpoints = ["us-east", "eu-west", "apac"] +for endpoint in endpoints: + status = await check_health(endpoint) + if status == "healthy": + print(f"Found healthy endpoint: {endpoint}") + break # Stop early +``` + +### Data Filtering + +```python +logs = await fetch_logs(server_id) +errors = [log for log in logs if "ERROR" in log] +print(f"Found {len(errors)} errors") +for error in errors[-10:]: # Only return last 10 errors + print(error) +``` + +## Best Practices + +### Tool Design + +- **Provide detailed output descriptions**: Since Claude deserializes tool results in code, clearly document the format (JSON structure, field types, etc.) +- **Return structured data**: JSON or other easily parseable formats work best for programmatic processing +- **Keep responses concise**: Return only necessary data to minimize processing overhead + +### When to Use Programmatic Calling + +**Good use cases:** + +- Processing large datasets where you only need aggregates or summaries +- Multi-step workflows with 3+ dependent tool calls +- Operations requiring filtering, sorting, or transformation of tool results +- Tasks where intermediate data shouldn't influence Claude's reasoning +- Parallel operations across many items (e.g., checking 50 endpoints) + +**Less ideal use cases:** + +- Single tool calls with simple responses +- Tools that need immediate user feedback +- Very fast operations where code execution overhead would outweigh the benefit + +## Token Efficiency + +Programmatic tool calling can significantly reduce token consumption: + +- **Tool results from programmatic calls are not added to Claude's context** - only the final code output is +- **Intermediate processing happens in code** - filtering, aggregation, etc. don't consume model tokens +- **Multiple tool calls in one code execution** - reduces overhead compared to separate model turns + +For example, calling 10 tools directly uses ~10x the tokens of calling them programmatically and returning a summary. + +## Provider Support + +LiteLLM supports programmatic tool calling across all Anthropic-compatible providers: + +- **Standard Anthropic API** (`anthropic/claude-sonnet-4-5-20250929`) +- **Azure Anthropic** (`azure/claude-sonnet-4-5-20250929`) +- **Vertex AI Anthropic** (`vertex_ai/claude-sonnet-4-5-20250929`) + +The beta header is automatically added when LiteLLM detects tools with `allowed_callers` field. + +## Limitations + +### Feature Incompatibilities + +- **Structured outputs**: Tools with `strict: true` are not supported with programmatic calling +- **Tool choice**: You cannot force programmatic calling of a specific tool via `tool_choice` +- **Parallel tool use**: `disable_parallel_tool_use: true` is not supported with programmatic calling + +### Tool Restrictions + +The following tools cannot currently be called programmatically: + +- Web search +- Web fetch +- Tools provided by an MCP connector + +## Troubleshooting + +### Common Issues + +**"Tool not allowed" error** + +- Verify your tool definition includes `"allowed_callers": ["code_execution_20250825"]` +- Check that you're using a compatible model (Claude Sonnet 4.5 or Opus 4.5) + +**Container expiration** + +- Ensure you respond to tool calls within the container's lifetime (~4.5 minutes) +- Consider implementing faster tool execution + +**Beta header not added** + +- LiteLLM automatically adds the beta header when it detects `allowed_callers` +- If you're manually setting headers, ensure you include `advanced-tool-use-2025-11-20` + +## Related Features + +- [Anthropic Tool Search](./anthropic_tool_search.md) - Dynamically discover and load tools on-demand +- [Anthropic Provider](./anthropic.md) - General Anthropic provider documentation + diff --git a/docs/my-website/docs/providers/anthropic_tool_input_examples.md b/docs/my-website/docs/providers/anthropic_tool_input_examples.md new file mode 100644 index 00000000000..d0b7cc1762c --- /dev/null +++ b/docs/my-website/docs/providers/anthropic_tool_input_examples.md @@ -0,0 +1,438 @@ +# Anthropic Tool Input Examples + +Provide concrete examples of valid tool inputs to help Claude understand how to use your tools more effectively. This is particularly useful for complex tools with nested objects, optional parameters, or format-sensitive inputs. + +:::info +Tool input examples is a beta feature. LiteLLM automatically adds the required `advanced-tool-use-2025-11-20` beta header when it detects tools with the `input_examples` field. +::: + +## When to Use Input Examples + +Input examples are most helpful for: + +- **Complex nested objects**: Tools with deeply nested parameter structures +- **Optional parameters**: Showing when optional parameters should be included +- **Format-sensitive inputs**: Demonstrating expected formats (dates, addresses, etc.) +- **Enum values**: Illustrating valid enum choices in context +- **Edge cases**: Showing how to handle special cases + +:::tip +**Prioritize descriptions first!** Clear, detailed tool descriptions are more important than examples. Use `input_examples` as a supplement for complex tools where descriptions alone may not be sufficient. +::: + +## Quick Start + +Add an `input_examples` field to your tool definition with an array of example input objects: + +```python +import litellm + +response = litellm.completion( + model="anthropic/claude-sonnet-4-5-20250929", + messages=[ + {"role": "user", "content": "What's the weather like in San Francisco?"} + ], + tools=[ + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get the current weather in a given location", + "parameters": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA" + }, + "unit": { + "type": "string", + "enum": ["celsius", "fahrenheit"], + "description": "The unit of temperature" + } + }, + "required": ["location"] + } + }, + "input_examples": [ + { + "location": "San Francisco, CA", + "unit": "fahrenheit" + }, + { + "location": "Tokyo, Japan", + "unit": "celsius" + }, + { + "location": "New York, NY" # 'unit' is optional + } + ] + } + ] +) + +print(response) +``` + +## How It Works + +When you provide `input_examples`: + +1. **LiteLLM detects** the `input_examples` field in your tool definition +2. **Beta header added automatically**: The `advanced-tool-use-2025-11-20` header is injected +3. **Examples included in prompt**: Anthropic includes the examples alongside your tool schema +4. **Claude learns patterns**: The model uses examples to understand proper tool usage +5. **Better tool calls**: Claude makes more accurate tool calls with correct parameter formats + +## Example Formats + +### Simple Tool with Examples + +```python +{ + "type": "function", + "function": { + "name": "send_email", + "description": "Send an email to a recipient", + "parameters": { + "type": "object", + "properties": { + "to": {"type": "string", "description": "Email address"}, + "subject": {"type": "string"}, + "body": {"type": "string"} + }, + "required": ["to", "subject", "body"] + } + }, + "input_examples": [ + { + "to": "user@example.com", + "subject": "Meeting Reminder", + "body": "Don't forget our meeting tomorrow at 2 PM." + }, + { + "to": "team@company.com", + "subject": "Weekly Update", + "body": "Here's this week's progress report..." + } + ] +} +``` + +### Complex Nested Objects + +```python +{ + "type": "function", + "function": { + "name": "create_calendar_event", + "description": "Create a new calendar event", + "parameters": { + "type": "object", + "properties": { + "title": {"type": "string"}, + "start": { + "type": "object", + "properties": { + "date": {"type": "string"}, + "time": {"type": "string"} + } + }, + "attendees": { + "type": "array", + "items": { + "type": "object", + "properties": { + "email": {"type": "string"}, + "optional": {"type": "boolean"} + } + } + } + }, + "required": ["title", "start"] + } + }, + "input_examples": [ + { + "title": "Team Standup", + "start": { + "date": "2025-01-15", + "time": "09:00" + }, + "attendees": [ + {"email": "alice@example.com", "optional": False}, + {"email": "bob@example.com", "optional": True} + ] + }, + { + "title": "Lunch Break", + "start": { + "date": "2025-01-15", + "time": "12:00" + } + # No attendees - showing optional field + } + ] +} +``` + +### Format-Sensitive Parameters + +```python +{ + "type": "function", + "function": { + "name": "search_flights", + "description": "Search for available flights", + "parameters": { + "type": "object", + "properties": { + "origin": {"type": "string", "description": "Airport code"}, + "destination": {"type": "string", "description": "Airport code"}, + "date": {"type": "string", "description": "Date in YYYY-MM-DD format"}, + "passengers": {"type": "integer"} + }, + "required": ["origin", "destination", "date"] + } + }, + "input_examples": [ + { + "origin": "SFO", + "destination": "JFK", + "date": "2025-03-15", + "passengers": 2 + }, + { + "origin": "LAX", + "destination": "ORD", + "date": "2025-04-20", + "passengers": 1 + } + ] +} +``` + +## Requirements and Limitations + +### Schema Validation + +- Each example **must be valid** according to the tool's `input_schema` +- Invalid examples will return a **400 error** from Anthropic +- Validation happens server-side (LiteLLM passes examples through) + +### Server-Side Tools Not Supported + +Input examples are **only supported for user-defined tools**. The following server-side tools do NOT support `input_examples`: + +- `web_search` (web search tool) +- `code_execution` (code execution tool) +- `computer_use` (computer use tool) +- `bash_tool` (bash execution tool) +- `text_editor` (text editor tool) + +### Token Costs + +Examples add to your prompt tokens: + +- **Simple examples**: ~20-50 tokens per example +- **Complex nested objects**: ~100-200 tokens per example +- **Trade-off**: Higher token cost for better tool call accuracy + +### Model Compatibility + +Input examples work with all Claude models that support the `advanced-tool-use-2025-11-20` beta header: + +- Claude Opus 4.5 (`claude-opus-4-5-20251101`) +- Claude Sonnet 4.5 (`claude-sonnet-4-5-20250929`) +- Claude Opus 4.1 (`claude-opus-4-1-20250805`) + +:::note +On Google Cloud's Vertex AI and Amazon Bedrock, only Claude Opus 4.5 supports tool input examples. +::: + +## Best Practices + +### 1. Show Diverse Examples + +Include examples that demonstrate different use cases: + +```python +"input_examples": [ + {"location": "San Francisco, CA", "unit": "fahrenheit"}, # US city + {"location": "Tokyo, Japan", "unit": "celsius"}, # International + {"location": "New York, NY"} # Optional param omitted +] +``` + +### 2. Demonstrate Optional Parameters + +Show when optional parameters should and shouldn't be included: + +```python +"input_examples": [ + { + "query": "machine learning", + "filters": {"year": 2024, "category": "research"} # With optional filters + }, + { + "query": "artificial intelligence" # Without optional filters + } +] +``` + +### 3. Illustrate Format Requirements + +Make format expectations clear through examples: + +```python +"input_examples": [ + { + "phone": "+1-555-123-4567", # Shows expected phone format + "date": "2025-01-15", # Shows date format (YYYY-MM-DD) + "time": "14:30" # Shows time format (HH:MM) + } +] +``` + +### 4. Keep Examples Realistic + +Use realistic, production-like examples rather than placeholder data: + +```python +# ✅ Good - realistic examples +"input_examples": [ + {"email": "alice@company.com", "role": "admin"}, + {"email": "bob@company.com", "role": "user"} +] + +# ❌ Bad - placeholder examples +"input_examples": [ + {"email": "test@test.com", "role": "role1"}, + {"email": "example@example.com", "role": "role2"} +] +``` + +### 5. Limit Example Count + +Provide 2-5 examples per tool: + +- **Too few** (1): May not show enough variation +- **Just right** (2-5): Demonstrates patterns without bloating tokens +- **Too many** (10+): Wastes tokens, diminishing returns + +## Integration with Other Features + +Input examples work seamlessly with other Anthropic tool features: + +### With Tool Search + +```python +{ + "type": "function", + "function": { + "name": "query_database", + "description": "Execute a SQL query", + "parameters": {...} + }, + "defer_loading": True, # Tool search + "input_examples": [ # Input examples + {"sql": "SELECT * FROM users WHERE id = 1"} + ] +} +``` + +### With Programmatic Tool Calling + +```python +{ + "type": "function", + "function": { + "name": "fetch_data", + "description": "Fetch data from API", + "parameters": {...} + }, + "allowed_callers": ["code_execution_20250825"], # Programmatic calling + "input_examples": [ # Input examples + {"endpoint": "/api/users", "method": "GET"} + ] +} +``` + +### All Features Combined + +```python +{ + "type": "function", + "function": { + "name": "advanced_tool", + "description": "A complex tool", + "parameters": {...} + }, + "defer_loading": True, # Tool search + "allowed_callers": ["code_execution_20250825"], # Programmatic calling + "input_examples": [ # Input examples + {"param1": "value1", "param2": "value2"} + ] +} +``` + +## Provider Support + +LiteLLM supports input examples across all Anthropic-compatible providers: + +- **Standard Anthropic API** (`anthropic/claude-sonnet-4-5-20250929`) +- **Azure Anthropic** (`azure/claude-sonnet-4-5-20250929`) +- **Vertex AI Anthropic** (`vertex_ai/claude-sonnet-4-5-20250929`) + +The beta header is automatically added when LiteLLM detects tools with `input_examples` field. + +## Troubleshooting + +### "Invalid request" error with examples + +**Problem**: Receiving 400 error when using input examples + +**Solution**: Ensure each example is valid according to your `input_schema`: + +```python +# Check that: +# 1. All required fields are present in examples +# 2. Field types match the schema +# 3. Enum values are valid +# 4. Nested objects follow the schema structure +``` + +### Examples not improving tool calls + +**Problem**: Adding examples doesn't seem to help + +**Solution**: +1. **Check descriptions first**: Ensure tool descriptions are detailed and clear +2. **Review example quality**: Make sure examples are realistic and diverse +3. **Verify schema**: Confirm examples actually match your schema +4. **Add more variation**: Include examples showing different use cases + +### Token usage too high + +**Problem**: Input examples consuming too many tokens + +**Solution**: +1. **Reduce example count**: Use 2-3 examples instead of 5+ +2. **Simplify examples**: Remove unnecessary fields from examples +3. **Consider descriptions**: If descriptions are clear, examples may not be needed + +## When NOT to Use Input Examples + +Skip input examples if: + +- **Tool is simple**: Single parameter tools with clear descriptions +- **Schema is self-explanatory**: Well-structured schema with good descriptions +- **Token budget is tight**: Examples add 20-200 tokens each +- **Server-side tools**: web_search, code_execution, etc. don't support examples + +## Related Features + +- [Anthropic Tool Search](./anthropic_tool_search.md) - Dynamically discover and load tools on-demand +- [Anthropic Programmatic Tool Calling](./anthropic_programmatic_tool_calling.md) - Call tools from code execution +- [Anthropic Provider](./anthropic.md) - General Anthropic provider documentation + diff --git a/docs/my-website/docs/providers/anthropic_tool_search.md b/docs/my-website/docs/providers/anthropic_tool_search.md new file mode 100644 index 00000000000..7b9e7cfaa72 --- /dev/null +++ b/docs/my-website/docs/providers/anthropic_tool_search.md @@ -0,0 +1,397 @@ +# Anthropic Tool Search + +Tool search enables Claude to dynamically discover and load tools on-demand from large tool catalogs (10,000+ tools). Instead of loading all tool definitions into the context window upfront, Claude searches your tool catalog and loads only the tools it needs. + +## Benefits + +- **Context efficiency**: Avoid consuming massive portions of your context window with tool definitions +- **Better tool selection**: Claude's tool selection accuracy degrades with more than 30-50 tools. Tool search maintains accuracy even with thousands of tools +- **On-demand loading**: Tools are only loaded when Claude needs them + +## Supported Models + +Tool search is available on: +- Claude Opus 4.5 +- Claude Sonnet 4.5 + +## Supported Platforms + +- Anthropic API (direct) +- Azure Anthropic (Microsoft Foundry) +- Google Cloud Vertex AI +- Amazon Bedrock (invoke API only, not converse API) + +## Tool Search Variants + +LiteLLM supports both tool search variants: + +### 1. Regex Tool Search (`tool_search_tool_regex_20251119`) + +Claude constructs regex patterns to search for tools. + +### 2. BM25 Tool Search (`tool_search_tool_bm25_20251119`) + +Claude uses natural language queries to search for tools using the BM25 algorithm. + +## Quick Start + +### Basic Example with Regex Tool Search + +```python +import litellm + +response = litellm.completion( + model="anthropic/claude-sonnet-4-5-20250929", + messages=[ + {"role": "user", "content": "What is the weather in San Francisco?"} + ], + tools=[ + # Tool search tool (regex variant) + { + "type": "tool_search_tool_regex_20251119", + "name": "tool_search_tool_regex" + }, + # Deferred tool - will be loaded on-demand + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get the weather at a specific location", + "parameters": { + "type": "object", + "properties": { + "location": {"type": "string"}, + "unit": { + "type": "string", + "enum": ["celsius", "fahrenheit"] + } + }, + "required": ["location"] + } + }, + "defer_loading": True # Mark for deferred loading + }, + # Another deferred tool + { + "type": "function", + "function": { + "name": "search_files", + "description": "Search through files in the workspace", + "parameters": { + "type": "object", + "properties": { + "query": {"type": "string"}, + "file_types": { + "type": "array", + "items": {"type": "string"} + } + }, + "required": ["query"] + } + }, + "defer_loading": True + } + ] +) + +print(response.choices[0].message.content) +``` + +### BM25 Tool Search Example + +```python +import litellm + +response = litellm.completion( + model="anthropic/claude-sonnet-4-5-20250929", + messages=[ + {"role": "user", "content": "Search for Python files containing 'authentication'"} + ], + tools=[ + # Tool search tool (BM25 variant) + { + "type": "tool_search_tool_bm25_20251119", + "name": "tool_search_tool_bm25" + }, + # Deferred tools... + { + "type": "function", + "function": { + "name": "search_codebase", + "description": "Search through codebase files by content and filename", + "parameters": { + "type": "object", + "properties": { + "query": {"type": "string"}, + "file_pattern": {"type": "string"} + }, + "required": ["query"] + } + }, + "defer_loading": True + } + ] +) +``` + +## Using with Azure Anthropic + +```python +import litellm + +response = litellm.completion( + model="azure_anthropic/claude-sonnet-4-5", + api_base="https://.services.ai.azure.com/anthropic", + api_key="your-azure-api-key", + messages=[ + {"role": "user", "content": "What's the weather like?"} + ], + tools=[ + { + "type": "tool_search_tool_regex_20251119", + "name": "tool_search_tool_regex" + }, + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get current weather", + "parameters": { + "type": "object", + "properties": { + "location": {"type": "string"} + }, + "required": ["location"] + } + }, + "defer_loading": True + } + ] +) +``` + +## Using with Vertex AI + +```python +import litellm + +response = litellm.completion( + model="vertex_ai/claude-sonnet-4-5", + vertex_project="your-project-id", + vertex_location="us-central1", + messages=[ + {"role": "user", "content": "Search my documents"} + ], + tools=[ + { + "type": "tool_search_tool_bm25_20251119", + "name": "tool_search_tool_bm25" + }, + # Your deferred tools... + ] +) +``` + +## Streaming Support + +Tool search works with streaming: + +```python +import litellm + +response = litellm.completion( + model="anthropic/claude-sonnet-4-5-20250929", + messages=[ + {"role": "user", "content": "Get the weather"} + ], + tools=[ + { + "type": "tool_search_tool_regex_20251119", + "name": "tool_search_tool_regex" + }, + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get weather information", + "parameters": { + "type": "object", + "properties": { + "location": {"type": "string"} + }, + "required": ["location"] + } + }, + "defer_loading": True + } + ], + stream=True +) + +for chunk in response: + if chunk.choices[0].delta.content: + print(chunk.choices[0].delta.content, end="") +``` + +## LiteLLM Proxy + +Tool search works automatically through the LiteLLM proxy: + +### Proxy Config + +```yaml +model_list: + - model_name: claude-sonnet + litellm_params: + model: anthropic/claude-sonnet-4-5-20250929 + api_key: os.environ/ANTHROPIC_API_KEY +``` + +### Client Request + +```python +import openai + +client = openai.OpenAI( + api_key="your-litellm-proxy-key", + base_url="http://0.0.0.0:4000" +) + +response = client.chat.completions.create( + model="claude-sonnet", + messages=[ + {"role": "user", "content": "What's the weather?"} + ], + tools=[ + { + "type": "tool_search_tool_regex_20251119", + "name": "tool_search_tool_regex" + }, + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get weather information", + "parameters": { + "type": "object", + "properties": { + "location": {"type": "string"} + }, + "required": ["location"] + } + }, + "defer_loading": True + } + ] +) +``` + +## Important Notes + +### Beta Header + +LiteLLM automatically adds the `advanced-tool-use-2025-11-20` beta header when tool search tools are detected. You don't need to manually specify it. + +### Deferred Loading + +- Tools with `defer_loading: true` are only loaded when Claude discovers them via search +- At least one tool must be non-deferred (the tool search tool itself) +- Keep your 3-5 most frequently used tools as non-deferred for optimal performance + +### Tool Descriptions + +Write clear, descriptive tool names and descriptions that match how users describe tasks. The search algorithm uses: +- Tool names +- Tool descriptions +- Argument names +- Argument descriptions + +### Usage Tracking + +Tool search requests are tracked in the usage object: + +```python +response = litellm.completion( + model="anthropic/claude-sonnet-4-5-20250929", + messages=[{"role": "user", "content": "Search for tools"}], + tools=[...] +) + +# Check tool search usage +if response.usage.server_tool_use: + print(f"Tool search requests: {response.usage.server_tool_use.tool_search_requests}") +``` + +## Error Handling + +### All Tools Deferred + +```python +# ❌ This will fail - at least one tool must be non-deferred +tools = [ + { + "type": "function", + "function": {...}, + "defer_loading": True + } +] + +# ✅ Correct - tool search tool is non-deferred +tools = [ + { + "type": "tool_search_tool_regex_20251119", + "name": "tool_search_tool_regex" + }, + { + "type": "function", + "function": {...}, + "defer_loading": True + } +] +``` + +### Missing Tool Definition + +If Claude references a tool that isn't in your deferred tools list, you'll get an error. Make sure all tools that might be discovered are included in the tools parameter with `defer_loading: true`. + +## Best Practices + +1. **Keep frequently used tools non-deferred**: Your 3-5 most common tools should not have `defer_loading: true` + +2. **Use semantic descriptions**: Tool descriptions should use natural language that matches user queries + +3. **Choose the right variant**: + - Use **regex** for exact pattern matching (faster) + - Use **BM25** for natural language semantic search + +4. **Monitor usage**: Track `tool_search_requests` in the usage object to understand search patterns + +5. **Optimize tool catalog**: Remove unused tools and consolidate similar functionality + +## When to Use Tool Search + +**Good use cases:** +- 10+ tools available in your system +- Tool definitions consuming >10K tokens +- Experiencing tool selection accuracy issues +- Building systems with multiple tool categories +- Tool library growing over time + +**When traditional tool calling is better:** +- Less than 10 tools total +- All tools are frequently used +- Very small tool definitions (\<100 tokens total) + +## Limitations + +- Not compatible with tool use examples +- Requires Claude Opus 4.5 or Sonnet 4.5 +- On Bedrock, only available via invoke API (not converse API) +- Maximum 10,000 tools in catalog +- Returns 3-5 most relevant tools per search + +## Additional Resources + +- [Anthropic Tool Search Documentation](https://docs.anthropic.com/en/docs/build-with-claude/tool-use/tool-search) +- [LiteLLM Tool Calling Guide](https://docs.litellm.ai/docs/completion/function_call) + diff --git a/docs/my-website/docs/providers/azure/azure.md b/docs/my-website/docs/providers/azure/azure.md index 0ff5b2a5a77..0b9fd29e680 100644 --- a/docs/my-website/docs/providers/azure/azure.md +++ b/docs/my-website/docs/providers/azure/azure.md @@ -9,10 +9,10 @@ import TabItem from '@theme/TabItem'; | Property | Details | |-------|-------| -| Description | Azure OpenAI Service provides REST API access to OpenAI's powerful language models including o1, o1-mini, GPT-5, GPT-4o, GPT-4o mini, GPT-4 Turbo with Vision, GPT-4, GPT-3.5-Turbo, and Embeddings model series | -| Provider Route on LiteLLM | `azure/`, [`azure/o_series/`](#o-series-models), [`azure/gpt5_series/`](#gpt-5-models) | -| Supported Operations | [`/chat/completions`](#azure-openai-chat-completion-models), [`/responses`](./azure_responses), [`/completions`](#azure-instruct-models), [`/embeddings`](./azure_embedding), [`/audio/speech`](azure_speech), [`/audio/transcriptions`](../audio_transcription), `/fine_tuning`, [`/batches`](#azure-batches-api), `/files`, [`/images`](../image_generation#azure-openai-image-generation-models) | -| Link to Provider Doc | [Azure OpenAI ↗](https://learn.microsoft.com/en-us/azure/ai-services/openai/overview) +| Description | Azure OpenAI Service provides REST API access to OpenAI's powerful language models including o1, o1-mini, GPT-5, GPT-4o, GPT-4o mini, GPT-4 Turbo with Vision, GPT-4, GPT-3.5-Turbo, and Embeddings model series. Also supports Claude models via Azure Foundry. | +| Provider Route on LiteLLM | `azure/`, [`azure/o_series/`](#o-series-models), [`azure/gpt5_series/`](#gpt-5-models), [`azure/claude-*`](./azure_anthropic) (Claude models via Azure Foundry) | +| Supported Operations | [`/chat/completions`](#azure-openai-chat-completion-models), [`/responses`](./azure_responses), [`/completions`](#azure-instruct-models), [`/embeddings`](./azure_embedding), [`/audio/speech`](azure_speech), [`/audio/transcriptions`](../audio_transcription), `/fine_tuning`, [`/batches`](#azure-batches-api), `/files`, [`/images`](../image_generation#azure-openai-image-generation-models), [`/anthropic/v1/messages`](./azure_anthropic) | +| Link to Provider Doc | [Azure OpenAI ↗](https://learn.microsoft.com/en-us/azure/ai-services/openai/overview), [Azure Foundry Claude ↗](https://learn.microsoft.com/en-us/azure/ai-services/foundry-models/claude) ## API Keys, Params api_key, api_base, api_version etc can be passed directly to `litellm.completion` - see here or set as `litellm.api_key` params see here @@ -27,6 +27,12 @@ os.environ["AZURE_AD_TOKEN"] = "" os.environ["AZURE_API_TYPE"] = "" ``` +:::info Azure Foundry Claude Models + +Azure also supports Claude models via Azure Foundry. Use `azure/claude-*` model names (e.g., `azure/claude-sonnet-4-5`) with Azure authentication. See the [Azure Anthropic documentation](./azure_anthropic) for details. + +::: + ## **Usage - LiteLLM Python SDK** Open In Colab diff --git a/docs/my-website/docs/providers/azure/azure_anthropic.md b/docs/my-website/docs/providers/azure/azure_anthropic.md new file mode 100644 index 00000000000..771912646b5 --- /dev/null +++ b/docs/my-website/docs/providers/azure/azure_anthropic.md @@ -0,0 +1,378 @@ +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +# Azure Anthropic (Claude via Azure Foundry) + +LiteLLM supports Claude models deployed via Microsoft Azure Foundry, including Claude Sonnet 4.5, Claude Haiku 4.5, and Claude Opus 4.1. + +## Available Models + +Azure Foundry supports the following Claude models: + +- `claude-sonnet-4-5` - Anthropic's most capable model for building real-world agents and handling complex, long-horizon tasks +- `claude-haiku-4-5` - Near-frontier performance with the right speed and cost for high-volume use cases +- `claude-opus-4-1` - Industry leader for coding, delivering sustained performance on long-running tasks + +| Property | Details | +|-------|-------| +| Description | Claude models deployed via Microsoft Azure Foundry. Uses the same API as Anthropic's Messages API but with Azure authentication. | +| Provider Route on LiteLLM | `azure/` (add this prefix to Claude model names - e.g. `azure/claude-sonnet-4-5`) | +| Provider Doc | [Azure Foundry Claude Models ↗](https://learn.microsoft.com/en-us/azure/ai-services/foundry-models/claude) | +| API Endpoint | `https://.services.ai.azure.com/anthropic/v1/messages` | +| Supported Endpoints | `/chat/completions`, `/anthropic/v1/messages`| + +## Key Features + +- **Extended thinking**: Enhanced reasoning capabilities for complex tasks +- **Image and text input**: Strong vision capabilities for analyzing charts, graphs, technical diagrams, and reports +- **Code generation**: Advanced thinking with code generation, analysis, and debugging (Claude Sonnet 4.5 and Claude Opus 4.1) +- **Same API as Anthropic**: All request/response transformations are identical to the main Anthropic provider + +## Authentication + +Azure Anthropic supports two authentication methods: + +1. **API Key**: Use the `api-key` header +2. **Azure AD Token**: Use `Authorization: Bearer ` header (Microsoft Entra ID) + +## API Keys and Configuration + +```python +import os + +# Option 1: API Key authentication +os.environ["AZURE_API_KEY"] = "your-azure-api-key" +os.environ["AZURE_API_BASE"] = "https://.services.ai.azure.com/anthropic" + +# Option 2: Azure AD Token authentication +os.environ["AZURE_AD_TOKEN"] = "your-azure-ad-token" +os.environ["AZURE_API_BASE"] = "https://.services.ai.azure.com/anthropic" + +# Optional: Azure AD Token Provider (for automatic token refresh) +os.environ["AZURE_TENANT_ID"] = "your-tenant-id" +os.environ["AZURE_CLIENT_ID"] = "your-client-id" +os.environ["AZURE_CLIENT_SECRET"] = "your-client-secret" +os.environ["AZURE_SCOPE"] = "https://cognitiveservices.azure.com/.default" +``` + +## Usage - LiteLLM Python SDK + +### Basic Completion + +```python +from litellm import completion + +# Set environment variables +os.environ["AZURE_API_KEY"] = "your-azure-api-key" +os.environ["AZURE_API_BASE"] = "https://.services.ai.azure.com/anthropic" + +# Make a completion request +response = completion( + model="azure/claude-sonnet-4-5", + messages=[ + {"role": "user", "content": "What are 3 things to visit in Seattle?"} + ], + max_tokens=1000, + temperature=0.7, +) + +print(response) +``` + +### Completion with API Key Parameter + +```python +import litellm + +response = litellm.completion( + model="azure/claude-sonnet-4-5", + api_base="https://.services.ai.azure.com/anthropic", + api_key="your-azure-api-key", + messages=[ + {"role": "user", "content": "Hello!"} + ], + max_tokens=1000, +) +``` + +### Completion with Azure AD Token + +```python +import litellm + +response = litellm.completion( + model="azure/claude-sonnet-4-5", + api_base="https://.services.ai.azure.com/anthropic", + azure_ad_token="your-azure-ad-token", + messages=[ + {"role": "user", "content": "Hello!"} + ], + max_tokens=1000, +) +``` + +### Streaming + +```python +from litellm import completion + +response = completion( + model="azure/claude-sonnet-4-5", + messages=[ + {"role": "user", "content": "Write a short story"} + ], + stream=True, + max_tokens=1000, +) + +for chunk in response: + if chunk.choices[0].delta.content: + print(chunk.choices[0].delta.content, end="", flush=True) +``` + +### Tool Calling + +```python +from litellm import completion + +response = completion( + model="azure/claude-sonnet-4-5", + messages=[ + {"role": "user", "content": "What's the weather in Seattle?"} + ], + tools=[ + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get the current weather in a given location", + "parameters": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA" + } + }, + "required": ["location"] + } + } + } + ], + tool_choice="auto", + max_tokens=1000, +) + +print(response) +``` + +## Usage - LiteLLM Proxy Server + +### 1. Save key in your environment + +```bash +export AZURE_API_KEY="your-azure-api-key" +export AZURE_API_BASE="https://.services.ai.azure.com/anthropic" +``` + +### 2. Configure the proxy + +```yaml +model_list: + - model_name: claude-sonnet-4-5 + litellm_params: + model: azure/claude-sonnet-4-5 + api_base: https://.services.ai.azure.com/anthropic + api_key: os.environ/AZURE_API_KEY +``` + +### 3. Test it + + + + +```bash +curl --location 'http://0.0.0.0:4000/chat/completions' \ +--header 'Content-Type: application/json' \ +--data '{ + "model": "claude-sonnet-4-5", + "messages": [ + { + "role": "user", + "content": "Hello!" + } + ], + "max_tokens": 1000 +}' +``` + + + + +```python +from openai import OpenAI + +client = OpenAI( + api_key="anything", + base_url="http://0.0.0.0:4000" +) + +response = client.chat.completions.create( + model="claude-sonnet-4-5", + messages=[ + {"role": "user", "content": "Hello!"} + ], + max_tokens=1000 +) + +print(response) +``` + + + + +## Messages API + +Azure Anthropic also supports the native Anthropic Messages API. The endpoint structure is the same as Anthropic's `/v1/messages` API. + +### Using Anthropic SDK + +```python +from anthropic import Anthropic + +client = Anthropic( + api_key="your-azure-api-key", + base_url="https://.services.ai.azure.com/anthropic" +) + +response = client.messages.create( + model="claude-sonnet-4-5", + max_tokens=1000, + messages=[ + {"role": "user", "content": "Hello, world"} + ] +) + +print(response) +``` + +### Using LiteLLM Proxy + +```bash +curl --request POST \ + --url http://0.0.0.0:4000/anthropic/v1/messages \ + --header 'accept: application/json' \ + --header 'content-type: application/json' \ + --header "Authorization: bearer sk-anything" \ + --data '{ + "model": "claude-sonnet-4-5", + "max_tokens": 1024, + "messages": [ + {"role": "user", "content": "Hello, world"} + ] +}' +``` + +## Supported OpenAI Parameters + +Azure Anthropic supports the same parameters as the main Anthropic provider: + +``` +"stream", +"stop", +"temperature", +"top_p", +"max_tokens", +"max_completion_tokens", +"tools", +"tool_choice", +"extra_headers", +"parallel_tool_calls", +"response_format", +"user", +"thinking", +"reasoning_effort" +``` + +:::info + +Azure Anthropic API requires `max_tokens` to be passed. LiteLLM automatically passes `max_tokens=4096` when no `max_tokens` are provided. + +::: + +## Differences from Standard Anthropic Provider + +The only difference between Azure Anthropic and the standard Anthropic provider is authentication: + +- **Standard Anthropic**: Uses `x-api-key` header +- **Azure Anthropic**: Uses `api-key` header or `Authorization: Bearer ` for Azure AD authentication + +All other request/response transformations, tool calling, streaming, and feature support are identical. + +## API Base URL Format + +The API base URL should follow this format: + +``` +https://.services.ai.azure.com/anthropic +``` + +LiteLLM will automatically append `/v1/messages` if not already present in the URL. + +## Example: Full Configuration + +```python +import os +from litellm import completion + +# Configure Azure Anthropic +os.environ["AZURE_API_KEY"] = "your-azure-api-key" +os.environ["AZURE_API_BASE"] = "https://my-resource.services.ai.azure.com/anthropic" + +# Make a request +response = completion( + model="azure/claude-sonnet-4-5", + messages=[ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": "Explain quantum computing in simple terms."} + ], + max_tokens=1000, + temperature=0.7, + stream=False, +) + +print(response.choices[0].message.content) +``` + +## Troubleshooting + +### Missing API Base Error + +If you see an error about missing API base, ensure you've set: + +```python +os.environ["AZURE_API_BASE"] = "https://.services.ai.azure.com/anthropic" +``` + +Or pass it directly: + +```python +response = completion( + model="azure/claude-sonnet-4-5", + api_base="https://.services.ai.azure.com/anthropic", + # ... +) +``` + +### Authentication Errors + +- **API Key**: Ensure `AZURE_API_KEY` is set or passed as `api_key` parameter +- **Azure AD Token**: Ensure `AZURE_AD_TOKEN` is set or passed as `azure_ad_token` parameter +- **Token Provider**: For automatic token refresh, configure `AZURE_TENANT_ID`, `AZURE_CLIENT_ID`, and `AZURE_CLIENT_SECRET` + +## Related Documentation + +- [Anthropic Provider Documentation](./anthropic.md) - For standard Anthropic API usage +- [Azure OpenAI Documentation](./azure.md) - For Azure OpenAI models +- [Azure Authentication Guide](../secret_managers/azure_key_vault.md) - For Azure AD token setup + diff --git a/docs/my-website/docs/providers/bedrock.md b/docs/my-website/docs/providers/bedrock.md index f0b89615a0d..9e22f67527e 100644 --- a/docs/my-website/docs/providers/bedrock.md +++ b/docs/my-website/docs/providers/bedrock.md @@ -7,7 +7,7 @@ ALL Bedrock models (Anthropic, Meta, Deepseek, Mistral, Amazon, etc.) are Suppor | Property | Details | |-------|-------| | Description | Amazon Bedrock is a fully managed service that offers a choice of high-performing foundation models (FMs). | -| Provider Route on LiteLLM | `bedrock/`, [`bedrock/converse/`](#set-converse--invoke-route), [`bedrock/invoke/`](#set-invoke-route), [`bedrock/converse_like/`](#calling-via-internal-proxy), [`bedrock/llama/`](#deepseek-not-r1), [`bedrock/deepseek_r1/`](#deepseek-r1), [`bedrock/qwen3/`](#qwen3-imported-models) | +| Provider Route on LiteLLM | `bedrock/`, [`bedrock/converse/`](#set-converse--invoke-route), [`bedrock/invoke/`](#set-invoke-route), [`bedrock/converse_like/`](#calling-via-internal-proxy), [`bedrock/llama/`](#deepseek-not-r1), [`bedrock/deepseek_r1/`](#deepseek-r1), [`bedrock/qwen3/`](#qwen3-imported-models), [`bedrock/openai/`](./bedrock_imported.md#openai-compatible-imported-models-qwen-25-vl-etc) | | Provider Doc | [Amazon Bedrock ↗](https://docs.aws.amazon.com/bedrock/latest/userguide/what-is-bedrock.html) | | Supported OpenAI Endpoints | `/chat/completions`, `/completions`, `/embeddings`, `/images/generations` | | Rerank Endpoint | `/rerank` | @@ -1598,206 +1598,6 @@ curl -X POST 'http://0.0.0.0:4000/chat/completions' \ -## Bedrock Imported Models (Deepseek, Deepseek R1) - -### Deepseek R1 - -This is a separate route, as the chat template is different. - -| Property | Details | -|----------|---------| -| Provider Route | `bedrock/deepseek_r1/{model_arn}` | -| Provider Documentation | [Bedrock Imported Models](https://docs.aws.amazon.com/bedrock/latest/userguide/model-customization-import-model.html), [Deepseek Bedrock Imported Model](https://aws.amazon.com/blogs/machine-learning/deploy-deepseek-r1-distilled-llama-models-with-amazon-bedrock-custom-model-import/) | - - - - -```python -from litellm import completion -import os - -response = completion( - model="bedrock/deepseek_r1/arn:aws:bedrock:us-east-1:086734376398:imported-model/r4c4kewx2s0n", # bedrock/deepseek_r1/{your-model-arn} - messages=[{"role": "user", "content": "Tell me a joke"}], -) -``` - - - - - - -**1. Add to config** - -```yaml -model_list: - - model_name: DeepSeek-R1-Distill-Llama-70B - litellm_params: - model: bedrock/deepseek_r1/arn:aws:bedrock:us-east-1:086734376398:imported-model/r4c4kewx2s0n - -``` - -**2. Start proxy** - -```bash -litellm --config /path/to/config.yaml - -# RUNNING at http://0.0.0.0:4000 -``` - -**3. Test it!** - -```bash -curl --location 'http://0.0.0.0:4000/chat/completions' \ - --header 'Authorization: Bearer sk-1234' \ - --header 'Content-Type: application/json' \ - --data '{ - "model": "DeepSeek-R1-Distill-Llama-70B", # 👈 the 'model_name' in config - "messages": [ - { - "role": "user", - "content": "what llm are you" - } - ], - }' -``` - - - - - -### Deepseek (not R1) - -| Property | Details | -|----------|---------| -| Provider Route | `bedrock/llama/{model_arn}` | -| Provider Documentation | [Bedrock Imported Models](https://docs.aws.amazon.com/bedrock/latest/userguide/model-customization-import-model.html), [Deepseek Bedrock Imported Model](https://aws.amazon.com/blogs/machine-learning/deploy-deepseek-r1-distilled-llama-models-with-amazon-bedrock-custom-model-import/) | - - - -Use this route to call Bedrock Imported Models that follow the `llama` Invoke Request / Response spec - - - - - -```python -from litellm import completion -import os - -response = completion( - model="bedrock/llama/arn:aws:bedrock:us-east-1:086734376398:imported-model/r4c4kewx2s0n", # bedrock/llama/{your-model-arn} - messages=[{"role": "user", "content": "Tell me a joke"}], -) -``` - - - - - - -**1. Add to config** - -```yaml -model_list: - - model_name: DeepSeek-R1-Distill-Llama-70B - litellm_params: - model: bedrock/llama/arn:aws:bedrock:us-east-1:086734376398:imported-model/r4c4kewx2s0n - -``` - -**2. Start proxy** - -```bash -litellm --config /path/to/config.yaml - -# RUNNING at http://0.0.0.0:4000 -``` - -**3. Test it!** - -```bash -curl --location 'http://0.0.0.0:4000/chat/completions' \ - --header 'Authorization: Bearer sk-1234' \ - --header 'Content-Type: application/json' \ - --data '{ - "model": "DeepSeek-R1-Distill-Llama-70B", # 👈 the 'model_name' in config - "messages": [ - { - "role": "user", - "content": "what llm are you" - } - ], - }' -``` - - - - -### Qwen3 Imported Models - -| Property | Details | -|----------|---------| -| Provider Route | `bedrock/qwen3/{model_arn}` | -| Provider Documentation | [Bedrock Imported Models](https://docs.aws.amazon.com/bedrock/latest/userguide/model-customization-import-model.html), [Qwen3 Models](https://aws.amazon.com/about-aws/whats-new/2025/09/qwen3-models-fully-managed-amazon-bedrock/) | - - - - -```python -from litellm import completion -import os - -response = completion( - model="bedrock/qwen3/arn:aws:bedrock:us-east-1:086734376398:imported-model/your-qwen3-model", # bedrock/qwen3/{your-model-arn} - messages=[{"role": "user", "content": "Tell me a joke"}], - max_tokens=100, - temperature=0.7 -) -``` - - - - - -**1. Add to config** - -```yaml -model_list: - - model_name: Qwen3-32B - litellm_params: - model: bedrock/qwen3/arn:aws:bedrock:us-east-1:086734376398:imported-model/your-qwen3-model - -``` - -**2. Start proxy** - -```bash -litellm --config /path/to/config.yaml - -# RUNNING at http://0.0.0.0:4000 -``` - -**3. Test it!** - -```bash -curl --location 'http://0.0.0.0:4000/chat/completions' \ - --header 'Authorization: Bearer sk-1234' \ - --header 'Content-Type: application/json' \ - --data '{ - "model": "Qwen3-32B", # 👈 the 'model_name' in config - "messages": [ - { - "role": "user", - "content": "what llm are you" - } - ], - }' -``` - - - - ### OpenAI GPT OSS | Property | Details | diff --git a/docs/my-website/docs/providers/bedrock_imported.md b/docs/my-website/docs/providers/bedrock_imported.md new file mode 100644 index 00000000000..8b0dd721c3c --- /dev/null +++ b/docs/my-website/docs/providers/bedrock_imported.md @@ -0,0 +1,369 @@ +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +# Bedrock Imported Models + +Bedrock Imported Models (Deepseek, Deepseek R1, Qwen, OpenAI-compatible models) + +### Deepseek R1 + +This is a separate route, as the chat template is different. + +| Property | Details | +|----------|---------| +| Provider Route | `bedrock/deepseek_r1/{model_arn}` | +| Provider Documentation | [Bedrock Imported Models](https://docs.aws.amazon.com/bedrock/latest/userguide/model-customization-import-model.html), [Deepseek Bedrock Imported Model](https://aws.amazon.com/blogs/machine-learning/deploy-deepseek-r1-distilled-llama-models-with-amazon-bedrock-custom-model-import/) | + + + + +```python +from litellm import completion +import os + +response = completion( + model="bedrock/deepseek_r1/arn:aws:bedrock:us-east-1:086734376398:imported-model/r4c4kewx2s0n", # bedrock/deepseek_r1/{your-model-arn} + messages=[{"role": "user", "content": "Tell me a joke"}], +) +``` + + + + + + +**1. Add to config** + +```yaml +model_list: + - model_name: DeepSeek-R1-Distill-Llama-70B + litellm_params: + model: bedrock/deepseek_r1/arn:aws:bedrock:us-east-1:086734376398:imported-model/r4c4kewx2s0n + +``` + +**2. Start proxy** + +```bash +litellm --config /path/to/config.yaml + +# RUNNING at http://0.0.0.0:4000 +``` + +**3. Test it!** + +```bash +curl --location 'http://0.0.0.0:4000/chat/completions' \ + --header 'Authorization: Bearer sk-1234' \ + --header 'Content-Type: application/json' \ + --data '{ + "model": "DeepSeek-R1-Distill-Llama-70B", # 👈 the 'model_name' in config + "messages": [ + { + "role": "user", + "content": "what llm are you" + } + ], + }' +``` + + + + + +### Deepseek (not R1) + +| Property | Details | +|----------|---------| +| Provider Route | `bedrock/llama/{model_arn}` | +| Provider Documentation | [Bedrock Imported Models](https://docs.aws.amazon.com/bedrock/latest/userguide/model-customization-import-model.html), [Deepseek Bedrock Imported Model](https://aws.amazon.com/blogs/machine-learning/deploy-deepseek-r1-distilled-llama-models-with-amazon-bedrock-custom-model-import/) | + + + +Use this route to call Bedrock Imported Models that follow the `llama` Invoke Request / Response spec + + + + + +```python +from litellm import completion +import os + +response = completion( + model="bedrock/llama/arn:aws:bedrock:us-east-1:086734376398:imported-model/r4c4kewx2s0n", # bedrock/llama/{your-model-arn} + messages=[{"role": "user", "content": "Tell me a joke"}], +) +``` + + + + + + +**1. Add to config** + +```yaml +model_list: + - model_name: DeepSeek-R1-Distill-Llama-70B + litellm_params: + model: bedrock/llama/arn:aws:bedrock:us-east-1:086734376398:imported-model/r4c4kewx2s0n + +``` + +**2. Start proxy** + +```bash +litellm --config /path/to/config.yaml + +# RUNNING at http://0.0.0.0:4000 +``` + +**3. Test it!** + +```bash +curl --location 'http://0.0.0.0:4000/chat/completions' \ + --header 'Authorization: Bearer sk-1234' \ + --header 'Content-Type: application/json' \ + --data '{ + "model": "DeepSeek-R1-Distill-Llama-70B", # 👈 the 'model_name' in config + "messages": [ + { + "role": "user", + "content": "what llm are you" + } + ], + }' +``` + + + + +### Qwen3 Imported Models + +| Property | Details | +|----------|---------| +| Provider Route | `bedrock/qwen3/{model_arn}` | +| Provider Documentation | [Bedrock Imported Models](https://docs.aws.amazon.com/bedrock/latest/userguide/model-customization-import-model.html), [Qwen3 Models](https://aws.amazon.com/about-aws/whats-new/2025/09/qwen3-models-fully-managed-amazon-bedrock/) | + + + + +```python +from litellm import completion +import os + +response = completion( + model="bedrock/qwen3/arn:aws:bedrock:us-east-1:086734376398:imported-model/your-qwen3-model", # bedrock/qwen3/{your-model-arn} + messages=[{"role": "user", "content": "Tell me a joke"}], + max_tokens=100, + temperature=0.7 +) +``` + + + + + +**1. Add to config** + +```yaml +model_list: + - model_name: Qwen3-32B + litellm_params: + model: bedrock/qwen3/arn:aws:bedrock:us-east-1:086734376398:imported-model/your-qwen3-model + +``` + +**2. Start proxy** + +```bash +litellm --config /path/to/config.yaml + +# RUNNING at http://0.0.0.0:4000 +``` + +**3. Test it!** + +```bash +curl --location 'http://0.0.0.0:4000/chat/completions' \ + --header 'Authorization: Bearer sk-1234' \ + --header 'Content-Type: application/json' \ + --data '{ + "model": "Qwen3-32B", # 👈 the 'model_name' in config + "messages": [ + { + "role": "user", + "content": "what llm are you" + } + ], + }' +``` + + + + +### OpenAI-Compatible Imported Models (Qwen 2.5 VL, etc.) + +Use this route for Bedrock imported models that follow the **OpenAI Chat Completions API spec**. This includes models like Qwen 2.5 VL that accept OpenAI-formatted messages with support for vision (images), tool calling, and other OpenAI features. + +| Property | Details | +|----------|---------| +| Provider Route | `bedrock/openai/{model_arn}` | +| Provider Documentation | [Bedrock Imported Models](https://docs.aws.amazon.com/bedrock/latest/userguide/model-customization-import-model.html) | +| Supported Features | Vision (images), tool calling, streaming, system messages | + +#### LiteLLMSDK Usage + +**Basic Usage** + +```python +from litellm import completion + +response = completion( + model="bedrock/openai/arn:aws:bedrock:us-east-1:046319184608:imported-model/0m2lasirsp6z", # bedrock/openai/{your-model-arn} + messages=[{"role": "user", "content": "Tell me a joke"}], + max_tokens=300, + temperature=0.5 +) +``` + +**With Vision (Images)** + +```python +import base64 +from litellm import completion + +# Load and encode image +with open("image.jpg", "rb") as f: + image_base64 = base64.b64encode(f.read()).decode("utf-8") + +response = completion( + model="bedrock/openai/arn:aws:bedrock:us-east-1:046319184608:imported-model/0m2lasirsp6z", + messages=[ + { + "role": "system", + "content": "You are a helpful assistant that can analyze images." + }, + { + "role": "user", + "content": [ + {"type": "text", "text": "What's in this image?"}, + { + "type": "image_url", + "image_url": {"url": f"data:image/jpeg;base64,{image_base64}"} + } + ] + } + ], + max_tokens=300, + temperature=0.5 +) +``` + +**Comparing Multiple Images** + +```python +import base64 +from litellm import completion + +# Load images +with open("image1.jpg", "rb") as f: + image1_base64 = base64.b64encode(f.read()).decode("utf-8") +with open("image2.jpg", "rb") as f: + image2_base64 = base64.b64encode(f.read()).decode("utf-8") + +response = completion( + model="bedrock/openai/arn:aws:bedrock:us-east-1:046319184608:imported-model/0m2lasirsp6z", + messages=[ + { + "role": "system", + "content": "You are a helpful assistant that can analyze images." + }, + { + "role": "user", + "content": [ + {"type": "text", "text": "Spot the difference between these two images?"}, + { + "type": "image_url", + "image_url": {"url": f"data:image/jpeg;base64,{image1_base64}"} + }, + { + "type": "image_url", + "image_url": {"url": f"data:image/jpeg;base64,{image2_base64}"} + } + ] + } + ], + max_tokens=300, + temperature=0.5 +) +``` + +#### LiteLLM Proxy Usage (AI Gateway) + +**1. Add to config** + +```yaml +model_list: + - model_name: qwen-25vl-72b + litellm_params: + model: bedrock/openai/arn:aws:bedrock:us-east-1:046319184608:imported-model/0m2lasirsp6z +``` + +**2. Start proxy** + +```bash +litellm --config /path/to/config.yaml + +# RUNNING at http://0.0.0.0:4000 +``` + +**3. Test it!** + +Basic text request: + +```bash +curl --location 'http://0.0.0.0:4000/chat/completions' \ + --header 'Authorization: Bearer sk-1234' \ + --header 'Content-Type: application/json' \ + --data '{ + "model": "qwen-25vl-72b", + "messages": [ + { + "role": "user", + "content": "what llm are you" + } + ], + "max_tokens": 300 + }' +``` + +With vision (image): + +```bash +curl --location 'http://0.0.0.0:4000/chat/completions' \ + --header 'Authorization: Bearer sk-1234' \ + --header 'Content-Type: application/json' \ + --data '{ + "model": "qwen-25vl-72b", + "messages": [ + { + "role": "system", + "content": "You are a helpful assistant that can analyze images." + }, + { + "role": "user", + "content": [ + {"type": "text", "text": "What is in this image?"}, + { + "type": "image_url", + "image_url": {"url": "data:image/jpeg;base64,/9j/4AAQSkZ..."} + } + ] + } + ], + "max_tokens": 300, + "temperature": 0.5 + }' +``` \ No newline at end of file diff --git a/docs/my-website/docs/providers/elevenlabs.md b/docs/my-website/docs/providers/elevenlabs.md index e80ea534f55..5cf62f51203 100644 --- a/docs/my-website/docs/providers/elevenlabs.md +++ b/docs/my-website/docs/providers/elevenlabs.md @@ -7,10 +7,10 @@ ElevenLabs provides high-quality AI voice technology, including speech-to-text c | Property | Details | |----------|---------| -| Description | ElevenLabs offers advanced AI voice technology with speech-to-text transcription capabilities that support multiple languages and speaker diarization. | +| Description | ElevenLabs offers advanced AI voice technology with speech-to-text transcription and text-to-speech capabilities that support multiple languages and speaker diarization. | | Provider Route on LiteLLM | `elevenlabs/` | | Provider Doc | [ElevenLabs API ↗](https://elevenlabs.io/docs/api-reference) | -| Supported Endpoints | `/audio/transcriptions` | +| Supported Endpoints | `/audio/transcriptions`, `/audio/speech` | ## Quick Start @@ -228,4 +228,241 @@ ElevenLabs returns transcription responses in OpenAI-compatible format: 1. **Invalid API Key**: Ensure `ELEVENLABS_API_KEY` is set correctly +--- + +## Text-to-Speech (TTS) + +ElevenLabs provides high-quality text-to-speech capabilities through their TTS API, supporting multiple voices, languages, and audio formats. + +### Overview + +| Property | Details | +|----------|---------| +| Description | Convert text to natural-sounding speech using ElevenLabs' advanced TTS models | +| Provider Route on LiteLLM | `elevenlabs/` | +| Supported Operations | `/audio/speech` | +| Link to Provider Doc | [ElevenLabs TTS API ↗](https://elevenlabs.io/docs/api-reference/text-to-speech) | + +### Quick Start + +#### LiteLLM Python SDK + +```python showLineNumbers title="ElevenLabs Text-to-Speech with SDK" +import litellm +import os + +os.environ["ELEVENLABS_API_KEY"] = "your-elevenlabs-api-key" + +# Basic usage with voice mapping +audio = litellm.speech( + model="elevenlabs/eleven_multilingual_v2", + input="Testing ElevenLabs speech from LiteLLM.", + voice="alloy", # Maps to ElevenLabs voice ID automatically +) + +# Save audio to file +with open("test_output.mp3", "wb") as f: + f.write(audio.read()) +``` + +#### Advanced Usage: Overriding Parameters and ElevenLabs-Specific Features + +```python showLineNumbers title="Advanced TTS with custom parameters" +import litellm +import os + +os.environ["ELEVENLABS_API_KEY"] = "your-elevenlabs-api-key" + +# Example showing parameter overriding and ElevenLabs-specific parameters +audio = litellm.speech( + model="elevenlabs/eleven_multilingual_v2", + input="Testing ElevenLabs speech from LiteLLM.", + voice="alloy", # Can use mapped voice name or raw ElevenLabs voice_id + response_format="pcm", # Maps to ElevenLabs output_format + speed=1.1, # Maps to voice_settings.speed + # ElevenLabs-specific parameters - passed directly to API + pronunciation_dictionary_locators=[ + {"pronunciation_dictionary_id": "dict_123", "version_id": "v1"} + ], + model_id="eleven_multilingual_v2", # Override model if needed +) + +# Save audio to file +with open("test_output.mp3", "wb") as f: + f.write(audio.read()) +``` + +### Voice Mapping + +LiteLLM automatically maps common OpenAI voice names to ElevenLabs voice IDs: + +| OpenAI Voice | ElevenLabs Voice ID | Description | +|--------------|---------------------|-------------| +| `alloy` | `21m00Tcm4TlvDq8ikWAM` | Rachel - Neutral and balanced | +| `amber` | `5Q0t7uMcjvnagumLfvZi` | Paul - Warm and friendly | +| `ash` | `AZnzlk1XvdvUeBnXmlld` | Domi - Energetic | +| `august` | `D38z5RcWu1voky8WS1ja` | Fin - Professional | +| `blue` | `2EiwWnXFnvU5JabPnv8n` | Clyde - Deep and authoritative | +| `coral` | `9BWtsMINqrJLrRacOk9x` | Aria - Expressive | +| `lily` | `EXAVITQu4vr4xnSDxMaL` | Sarah - Friendly | +| `onyx` | `29vD33N1CtxCmqQRPOHJ` | Drew - Strong | +| `sage` | `CwhRBWXzGAHq8TQ4Fs17` | Roger - Calm | +| `verse` | `CYw3kZ02Hs0563khs1Fj` | Dave - Conversational | + +**Using Custom Voice IDs**: You can also pass any ElevenLabs voice ID directly. If the voice name is not in the mapping, LiteLLM will use it as-is: + +```python showLineNumbers title="Using custom ElevenLabs voice ID" +audio = litellm.speech( + model="elevenlabs/eleven_multilingual_v2", + input="Testing with a custom voice.", + voice="21m00Tcm4TlvDq8ikWAM", # Direct ElevenLabs voice ID +) +``` + +### Response Format Mapping + +LiteLLM maps OpenAI response formats to ElevenLabs output formats: + +| OpenAI Format | ElevenLabs Format | +|---------------|-------------------| +| `mp3` | `mp3_44100_128` | +| `pcm` | `pcm_44100` | +| `opus` | `opus_48000_128` | + +You can also pass ElevenLabs-specific output formats directly using the `output_format` parameter. + +### Supported Parameters + +```python showLineNumbers title="All Supported Parameters" +audio = litellm.speech( + model="elevenlabs/eleven_multilingual_v2", # Required + input="Text to convert to speech", # Required + voice="alloy", # Required: Voice selection (mapped or raw ID) + response_format="mp3", # Optional: Audio format (mp3, pcm, opus) + speed=1.0, # Optional: Speech speed (maps to voice_settings.speed) + # ElevenLabs-specific parameters (passed directly): + model_id="eleven_multilingual_v2", # Optional: Override model + voice_settings={ # Optional: Voice customization + "stability": 0.5, + "similarity_boost": 0.75, + "speed": 1.0 + }, + pronunciation_dictionary_locators=[ # Optional: Custom pronunciation + {"pronunciation_dictionary_id": "dict_123", "version_id": "v1"} + ], +) +``` + +### LiteLLM Proxy + +#### 1. Configure your proxy + +```yaml showLineNumbers title="ElevenLabs TTS configuration in config.yaml" +model_list: + - model_name: elevenlabs-tts + litellm_params: + model: elevenlabs/eleven_multilingual_v2 + api_key: os.environ/ELEVENLABS_API_KEY + +general_settings: + master_key: your-master-key +``` + +#### 2. Make TTS requests + +##### Simple Usage (OpenAI Parameters) + +You can use standard OpenAI-compatible parameters without any provider-specific configuration: + +```bash showLineNumbers title="Simple TTS request with curl" +curl http://localhost:4000/v1/audio/speech \ + -H "Authorization: Bearer $LITELLM_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{ + "model": "elevenlabs-tts", + "input": "Testing ElevenLabs speech via the LiteLLM proxy.", + "voice": "alloy", + "response_format": "mp3" + }' \ + --output speech.mp3 +``` + +```python showLineNumbers title="Simple TTS with OpenAI SDK" +from openai import OpenAI + +client = OpenAI( + base_url="http://localhost:4000", + api_key="your-litellm-api-key" +) + +response = client.audio.speech.create( + model="elevenlabs-tts", + input="Testing ElevenLabs speech via the LiteLLM proxy.", + voice="alloy", + response_format="mp3" +) + +# Save audio +with open("speech.mp3", "wb") as f: + f.write(response.content) +``` + +##### Advanced Usage (ElevenLabs-Specific Parameters) + +**Note**: When using the proxy, provider-specific parameters (like `pronunciation_dictionary_locators`, `voice_settings`, etc.) must be passed in the `extra_body` field. + +```bash showLineNumbers title="Advanced TTS request with curl" +curl http://localhost:4000/v1/audio/speech \ + -H "Authorization: Bearer $LITELLM_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{ + "model": "elevenlabs-tts", + "input": "Testing ElevenLabs speech via the LiteLLM proxy.", + "voice": "alloy", + "response_format": "pcm", + "extra_body": { + "pronunciation_dictionary_locators": [ + {"pronunciation_dictionary_id": "dict_123", "version_id": "v1"} + ], + "voice_settings": { + "speed": 1.1, + "stability": 0.5, + "similarity_boost": 0.75 + } + } + }' \ + --output speech.mp3 +``` + +```python showLineNumbers title="Advanced TTS with OpenAI SDK" +from openai import OpenAI + +client = OpenAI( + base_url="http://localhost:4000", + api_key="your-litellm-api-key" +) + +response = client.audio.speech.create( + model="elevenlabs-tts", + input="Testing ElevenLabs speech via the LiteLLM proxy.", + voice="alloy", + response_format="pcm", + extra_body={ + "pronunciation_dictionary_locators": [ + {"pronunciation_dictionary_id": "dict_123", "version_id": "v1"} + ], + "voice_settings": { + "speed": 1.1, + "stability": 0.5, + "similarity_boost": 0.75 + } + } +) + +# Save audio +with open("speech.mp3", "wb") as f: + f.write(response.content) +``` + + diff --git a/docs/my-website/docs/providers/gemini.md b/docs/my-website/docs/providers/gemini.md index e04225e1f85..1b21ed8d03c 100644 --- a/docs/my-website/docs/providers/gemini.md +++ b/docs/my-website/docs/providers/gemini.md @@ -74,6 +74,10 @@ Note: Reasoning cannot be turned off on Gemini 2.5 Pro models. For **Gemini 3+ models** (e.g., `gemini-3-pro-preview`), LiteLLM automatically maps `reasoning_effort` to the new `thinking_level` parameter instead of `thinking_budget`. The `thinking_level` parameter uses `"low"` or `"high"` values for better control over reasoning depth. ::: +:::warning Image Models +**Gemini image models** (e.g., `gemini-3-pro-image-preview`, `gemini-2.0-flash-exp-image-generation`) do **not** support the `thinking_level` parameter. LiteLLM automatically excludes image models from receiving thinking configuration to prevent API errors. +::: + **Mapping for Gemini 2.5 and earlier models** | reasoning_effort | thinking | Notes | diff --git a/docs/my-website/docs/providers/gemini_file_search.md b/docs/my-website/docs/providers/gemini_file_search.md new file mode 100644 index 00000000000..947715218a3 --- /dev/null +++ b/docs/my-website/docs/providers/gemini_file_search.md @@ -0,0 +1,414 @@ +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +# Gemini File Search + +Use Google Gemini's File Search for Retrieval Augmented Generation (RAG) with LiteLLM. + +Gemini File Search imports, chunks, and indexes your data to enable fast retrieval of relevant information based on user prompts. This information is then provided as context to the model for more accurate and relevant answers. + +[Official Gemini File Search Documentation](https://ai.google.dev/gemini-api/docs/file-search) + +## Features + +| Feature | Supported | Notes | +|---------|-----------|-------| +| Cost Tracking | ❌ | Cost calculation not yet implemented | +| Logging | ✅ | Full request/response logging | +| RAG Ingest API | ✅ | Upload → Chunk → Embed → Store | +| Vector Store Search | ✅ | Search with metadata filters | +| Custom Chunking | ✅ | Configure chunk size and overlap | +| Metadata Filtering | ✅ | Filter by custom metadata | +| Citations | ✅ | Extract from grounding metadata | + +## Quick Start + +### Setup + +Set your Gemini API key: + +```bash +export GEMINI_API_KEY="your-api-key" +# or +export GOOGLE_API_KEY="your-api-key" +``` + +### Basic RAG Ingest + + + + +```python +import litellm + +# Ingest a document +response = await litellm.aingest( + ingest_options={ + "name": "my-document-store", + "vector_store": { + "custom_llm_provider": "gemini" + } + }, + file_data=("document.txt", b"Your document content", "text/plain") +) + +print(f"Vector Store ID: {response['vector_store_id']}") +print(f"File ID: {response['file_id']}") +``` + + + + + +```bash +curl -X POST "http://localhost:4000/v1/rag/ingest" \ + -H "Authorization: Bearer sk-1234" \ + -H "Content-Type: application/json" \ + -d '{ + "file": { + "filename": "document.txt", + "content": "'$(base64 -i document.txt)'", + "content_type": "text/plain" + }, + "ingest_options": { + "name": "my-document-store", + "vector_store": { + "custom_llm_provider": "gemini" + } + } + }' +``` + + + + +### Search Vector Store + + + + +```python +import litellm + +# Search the vector store +response = await litellm.vector_stores.asearch( + vector_store_id="fileSearchStores/your-store-id", + query="What is the main topic?", + custom_llm_provider="gemini", + max_num_results=5 +) + +for result in response["data"]: + print(f"Score: {result.get('score')}") + print(f"Content: {result['content'][0]['text']}") +``` + + + + + +```bash +curl -X POST "http://localhost:4000/v1/vector_stores/fileSearchStores/your-store-id/search" \ + -H "Authorization: Bearer sk-1234" \ + -H "Content-Type: application/json" \ + -d '{ + "query": "What is the main topic?", + "custom_llm_provider": "gemini", + "max_num_results": 5 + }' +``` + + + + +## Advanced Features + +### Custom Chunking Configuration + +Control how documents are split into chunks: + +```python +import litellm + +response = await litellm.aingest( + ingest_options={ + "name": "custom-chunking-store", + "vector_store": { + "custom_llm_provider": "gemini" + }, + "chunking_strategy": { + "white_space_config": { + "max_tokens_per_chunk": 200, + "max_overlap_tokens": 20 + } + } + }, + file_data=("document.txt", document_content, "text/plain") +) +``` + +**Chunking Parameters:** +- `max_tokens_per_chunk`: Maximum tokens per chunk (default: 800, min: 100, max: 4096) +- `max_overlap_tokens`: Overlap between chunks (default: 400) + +### Metadata Filtering + +Attach custom metadata to files and filter searches: + +#### Attach Metadata During Ingest + +```python +import litellm + +response = await litellm.aingest( + ingest_options={ + "name": "metadata-store", + "vector_store": { + "custom_llm_provider": "gemini", + "custom_metadata": [ + {"key": "author", "string_value": "John Doe"}, + {"key": "year", "numeric_value": 2024}, + {"key": "category", "string_value": "documentation"} + ] + } + }, + file_data=("document.txt", document_content, "text/plain") +) +``` + +#### Search with Metadata Filter + +```python +import litellm + +response = await litellm.vector_stores.asearch( + vector_store_id="fileSearchStores/your-store-id", + query="What is LiteLLM?", + custom_llm_provider="gemini", + filters={"author": "John Doe", "category": "documentation"} +) +``` + +**Filter Syntax:** +- Simple equality: `{"key": "value"}` +- Gemini converts to: `key="value"` +- Multiple filters combined with AND + +### Using Existing Vector Store + +Ingest into an existing File Search store: + +```python +import litellm + +# First, create a store +create_response = await litellm.vector_stores.acreate( + name="My Persistent Store", + custom_llm_provider="gemini" +) +store_id = create_response["id"] + +# Then ingest multiple documents into it +for doc in documents: + await litellm.aingest( + ingest_options={ + "vector_store": { + "custom_llm_provider": "gemini", + "vector_store_id": store_id # Reuse existing store + } + }, + file_data=(doc["name"], doc["content"], doc["type"]) + ) +``` + +### Citation Extraction + +Gemini provides grounding metadata with citations: + +```python +import litellm + +response = await litellm.vector_stores.asearch( + vector_store_id="fileSearchStores/your-store-id", + query="Explain the concept", + custom_llm_provider="gemini" +) + +for result in response["data"]: + # Access citation information + if "attributes" in result: + print(f"URI: {result['attributes'].get('uri')}") + print(f"Title: {result['attributes'].get('title')}") + + # Content with relevance score + print(f"Score: {result.get('score')}") + print(f"Text: {result['content'][0]['text']}") +``` + +## Complete Example + +End-to-end workflow: + +```python +import litellm + +# 1. Create a File Search store +store_response = await litellm.vector_stores.acreate( + name="Knowledge Base", + custom_llm_provider="gemini" +) +store_id = store_response["id"] +print(f"Created store: {store_id}") + +# 2. Ingest documents with custom chunking and metadata +documents = [ + { + "name": "intro.txt", + "content": b"Introduction to LiteLLM...", + "metadata": [ + {"key": "section", "string_value": "intro"}, + {"key": "priority", "numeric_value": 1} + ] + }, + { + "name": "advanced.txt", + "content": b"Advanced features...", + "metadata": [ + {"key": "section", "string_value": "advanced"}, + {"key": "priority", "numeric_value": 2} + ] + } +] + +for doc in documents: + ingest_response = await litellm.aingest( + ingest_options={ + "name": f"ingest-{doc['name']}", + "vector_store": { + "custom_llm_provider": "gemini", + "vector_store_id": store_id, + "custom_metadata": doc["metadata"] + }, + "chunking_strategy": { + "white_space_config": { + "max_tokens_per_chunk": 300, + "max_overlap_tokens": 50 + } + } + }, + file_data=(doc["name"], doc["content"], "text/plain") + ) + print(f"Ingested: {doc['name']}") + +# 3. Search with filters +search_response = await litellm.vector_stores.asearch( + vector_store_id=store_id, + query="How do I get started?", + custom_llm_provider="gemini", + filters={"section": "intro"}, + max_num_results=3 +) + +# 4. Process results +for i, result in enumerate(search_response["data"]): + print(f"\nResult {i+1}:") + print(f" Score: {result.get('score')}") + print(f" File: {result.get('filename')}") + print(f" Content: {result['content'][0]['text'][:100]}...") +``` + +## Supported File Types + +Gemini File Search supports a wide range of file formats: + +### Documents +- PDF (`application/pdf`) +- Microsoft Word (`.docx`, `.doc`) +- Microsoft Excel (`.xlsx`, `.xls`) +- Microsoft PowerPoint (`.pptx`) +- OpenDocument formats (`.odt`, `.ods`, `.odp`) + +### Text Files +- Plain text (`text/plain`) +- Markdown (`text/markdown`) +- HTML (`text/html`) +- CSV (`text/csv`) +- JSON (`application/json`) +- XML (`application/xml`) + +### Code Files +- Python, JavaScript, TypeScript, Java, C/C++, Go, Rust, etc. +- Most common programming languages supported + +See [Gemini's full list of supported file types](https://ai.google.dev/gemini-api/docs/file-search#supported-file-types). + +## Pricing + +- **Indexing**: $0.15 per 1M tokens (embedding pricing) +- **Storage**: Free +- **Query embeddings**: Free +- **Retrieved tokens**: Charged as regular context tokens + +## Supported Models + +File Search works with: +- `gemini-3-pro-preview` +- `gemini-2.5-pro` +- `gemini-2.5-flash` (and preview versions) +- `gemini-2.5-flash-lite` (and preview versions) + +## Troubleshooting + +### Authentication Errors + +```python +# Ensure API key is set +import os +os.environ["GEMINI_API_KEY"] = "your-api-key" + +# Or pass explicitly +response = await litellm.aingest( + ingest_options={ + "vector_store": { + "custom_llm_provider": "gemini", + "api_key": "your-api-key" + } + }, + file_data=(...) +) +``` + +### Store Not Found + +Ensure you're using the full store name format: +- ✅ `fileSearchStores/abc123` +- ❌ `abc123` + +### Large Files + +For files >100MB, split them into smaller chunks before ingestion. + +### Slow Indexing + +After ingestion, Gemini may need time to index documents. Wait a few seconds before searching: + +```python +import time + +# After ingest +await litellm.aingest(...) + +# Wait for indexing +time.sleep(5) + +# Then search +await litellm.vector_stores.asearch(...) +``` + +## Related Resources + +- [Gemini File Search Official Docs](https://ai.google.dev/gemini-api/docs/file-search) +- [LiteLLM RAG Ingest API](/docs/rag_ingest) +- [LiteLLM Vector Store Search](/docs/vector_stores/search) +- [Using Vector Stores with Chat](/docs/completion/knowledgebase) + diff --git a/docs/my-website/docs/providers/vertex_image.md b/docs/my-website/docs/providers/vertex_image.md index 27e584cb222..c4d5d554088 100644 --- a/docs/my-website/docs/providers/vertex_image.md +++ b/docs/my-website/docs/providers/vertex_image.md @@ -1,18 +1,65 @@ # Vertex AI Image Generation -Vertex AI Image Generation uses Google's Imagen models to generate high-quality images from text descriptions. +Vertex AI supports two types of image generation: + +1. **Gemini Image Generation Models** (Nano Banana 🍌) - Conversational image generation using `generateContent` API +2. **Imagen Models** - Traditional image generation using `predict` API | Property | Details | |----------|---------| -| Description | Vertex AI Image Generation uses Google's Imagen models to generate high-quality images from text descriptions. | +| Description | Vertex AI Image Generation supports both Gemini image generation models | | Provider Route on LiteLLM | `vertex_ai/` | | Provider Doc | [Google Cloud Vertex AI Image Generation ↗](https://cloud.google.com/vertex-ai/docs/generative-ai/image/generate-images) | +| Gemini Image Generation Docs | [Gemini Image Generation ↗](https://ai.google.dev/gemini-api/docs/image-generation) | ## Quick Start -### LiteLLM Python SDK +### Gemini Image Generation Models -```python showLineNumbers title="Basic Image Generation" +Gemini image generation models support conversational image creation with features like: +- Text-to-Image generation +- Image editing (text + image → image) +- Multi-turn image refinement +- High-fidelity text rendering +- Up to 4K resolution (Gemini 3 Pro) + +```python showLineNumbers title="Gemini 2.5 Flash Image" +import litellm + +# Generate a single image +response = await litellm.aimage_generation( + prompt="A nano banana dish in a fancy restaurant with a Gemini theme", + model="vertex_ai/gemini-2.5-flash-image", + vertex_ai_project="your-project-id", + vertex_ai_location="us-central1", + n=1, + size="1024x1024", +) + +print(response.data[0].b64_json) # Gemini returns base64 images +``` + +```python showLineNumbers title="Gemini 3 Pro Image Preview (4K output)" +import litellm + +# Generate high-resolution image +response = await litellm.aimage_generation( + prompt="Da Vinci style anatomical sketch of a dissected Monarch butterfly", + model="vertex_ai/gemini-3-pro-image-preview", + vertex_ai_project="your-project-id", + vertex_ai_location="us-central1", + n=1, + size="1024x1024", + # Optional: specify image size for Gemini 3 Pro + # imageSize="4K", # Options: "1K", "2K", "4K" +) + +print(response.data[0].b64_json) +``` + +### Imagen Models + +```python showLineNumbers title="Imagen Image Generation" import litellm # Generate a single image @@ -21,9 +68,11 @@ response = await litellm.aimage_generation( model="vertex_ai/imagen-4.0-generate-001", vertex_ai_project="your-project-id", vertex_ai_location="us-central1", + n=1, + size="1024x1024", ) -print(response.data[0].url) +print(response.data[0].b64_json) # Imagen also returns base64 images ``` ### LiteLLM Proxy @@ -70,6 +119,18 @@ print(response.data[0].url) ## Supported Models +### Gemini Image Generation Models + +- `vertex_ai/gemini-2.5-flash-image` - Fast, efficient image generation (1024px resolution) +- `vertex_ai/gemini-3-pro-image-preview` - Advanced model with 4K output, Google Search grounding, and thinking mode +- `vertex_ai/gemini-2.0-flash-preview-image` - Preview model +- `vertex_ai/gemini-2.5-flash-image-preview` - Preview model + +### Imagen Models + +- `vertex_ai/imagegeneration@006` - Legacy Imagen model +- `vertex_ai/imagen-4.0-generate-001` - Latest Imagen model +- `vertex_ai/imagen-3.0-generate-001` - Imagen 3.0 model :::tip @@ -77,7 +138,5 @@ print(response.data[0].url) ::: -LiteLLM supports all Vertex AI Imagen models available through Google Cloud. - For the complete and up-to-date list of supported models, visit: [https://models.litellm.ai/](https://models.litellm.ai/) diff --git a/docs/my-website/docs/proxy/config_settings.md b/docs/my-website/docs/proxy/config_settings.md index 4d1bc549e05..5a586035bcf 100644 --- a/docs/my-website/docs/proxy/config_settings.md +++ b/docs/my-website/docs/proxy/config_settings.md @@ -104,6 +104,7 @@ general_settings: disable_responses_id_security: boolean # turn off response ID security checks that prevent users from accessing other users' responses enable_jwt_auth: boolean # allow proxy admin to auth in via jwt tokens with 'litellm_proxy_admin' in claims enforce_user_param: boolean # requires all openai endpoint requests to have a 'user' param + reject_clientside_metadata_tags: boolean # if true, rejects requests with client-side 'metadata.tags' to prevent users from influencing budgets allowed_routes: ["route1", "route2"] # list of allowed proxy API routes - a user can access. (currently JWT-Auth only) key_management_system: google_kms # either google_kms or azure_kms master_key: string @@ -201,6 +202,7 @@ router_settings: | disable_responses_id_security | boolean | If true, disables response ID security checks that prevent users from accessing response IDs from other users. When false (default), response IDs are encrypted with user information to ensure users can only access their own responses. Applies to /v1/responses endpoints | | enable_jwt_auth | boolean | allow proxy admin to auth in via jwt tokens with 'litellm_proxy_admin' in claims. [Doc on JWT Tokens](token_auth) | | enforce_user_param | boolean | If true, requires all OpenAI endpoint requests to have a 'user' param. [Doc on call hooks](call_hooks)| +| reject_clientside_metadata_tags | boolean | If true, rejects requests that contain client-side 'metadata.tags' to prevent users from influencing budgets by sending different tags. Tags can only be inherited from the API key metadata. | | allowed_routes | array of strings | List of allowed proxy API routes a user can access [Doc on controlling allowed routes](enterprise#control-available-public-private-routes)| | key_management_system | string | Specifies the key management system. [Doc Secret Managers](../secret) | | master_key | string | The master key for the proxy [Set up Virtual Keys](virtual_keys) | diff --git a/docs/my-website/docs/proxy/guardrails/pillar_security.md b/docs/my-website/docs/proxy/guardrails/pillar_security.md index 5ab9f9bf8cb..9632376768b 100644 --- a/docs/my-website/docs/proxy/guardrails/pillar_security.md +++ b/docs/my-website/docs/proxy/guardrails/pillar_security.md @@ -60,6 +60,8 @@ litellm_settings: set_verbose: true # Enable detailed logging ``` +**Note:** Virtual key context is **automatically passed** as headers - no additional configuration needed! + ### 3. Start the Proxy ```bash @@ -210,7 +212,7 @@ export PILLAR_API_KEY="your_api_key_here" export PILLAR_API_BASE="https://api.pillar.security" export PILLAR_ON_FLAGGED_ACTION="monitor" export PILLAR_FALLBACK_ON_ERROR="allow" -export PILLAR_TIMEOUT="30.0" +export PILLAR_TIMEOUT="5.0" ``` ### Session Tracking diff --git a/docs/my-website/docs/proxy/guardrails/tool_permission.md b/docs/my-website/docs/proxy/guardrails/tool_permission.md index 2e0b72a8a8a..19b674c9e55 100644 --- a/docs/my-website/docs/proxy/guardrails/tool_permission.md +++ b/docs/my-website/docs/proxy/guardrails/tool_permission.md @@ -2,9 +2,9 @@ import Image from '@theme/IdealImage'; import Tabs from '@theme/Tabs'; import TabItem from '@theme/TabItem'; -# Tool Permission Guardrail +# LiteLLM Tool Permission Guardrail -LiteLLM provides a Tool Permission Guardrail that lets you control which **tool calls** a model is allowed to invoke, using configurable allow/deny rules. This offers fine-grained, provider-agnostic control over tool execution (e.g., OpenAI Chat Completions `tool_calls`, Anthropic Messages `tool_use`, MCP tools). +LiteLLM provides the LiteLLM Tool Permission Guardrail that lets you control which **tool calls** a model is allowed to invoke, using configurable allow/deny rules. This offers fine-grained, provider-agnostic control over tool execution (e.g., OpenAI Chat Completions `tool_calls`, Anthropic Messages `tool_use`, MCP tools). ## Quick Start ### 1. Define Guardrails on your LiteLLM config.yaml diff --git a/docs/my-website/docs/proxy/reject_clientside_metadata_tags.md b/docs/my-website/docs/proxy/reject_clientside_metadata_tags.md new file mode 100644 index 00000000000..534c65939eb --- /dev/null +++ b/docs/my-website/docs/proxy/reject_clientside_metadata_tags.md @@ -0,0 +1,120 @@ +# Reject Client-Side Metadata Tags + +## Overview + +The `reject_clientside_metadata_tags` setting allows you to prevent users from passing client-side `metadata.tags` in their API requests. This ensures that tags are only inherited from the API key metadata and cannot be overridden by users to potentially influence budget tracking or routing decisions. + +## Use Case + +This feature is particularly useful in multi-tenant scenarios where: +- You want to enforce strict budget tracking based on API key tags +- You want to prevent users from manipulating routing decisions by sending custom client-side tags +- You need to ensure consistent tag-based filtering and reporting + +## Configuration + +Add the following to your `config.yaml`: + +```yaml +general_settings: + reject_clientside_metadata_tags: true # Default is false/null +``` + +## Behavior + +### When `reject_clientside_metadata_tags: true` + +**Rejected Request Example:** +```bash +curl -X POST http://localhost:4000/chat/completions \ + -H "Authorization: Bearer sk-1234" \ + -H "Content-Type: application/json" \ + -d '{ + "model": "gpt-3.5-turbo", + "messages": [{"role": "user", "content": "Hello"}], + "metadata": { + "tags": ["custom-tag"] # This will be rejected + } + }' +``` + +**Error Response:** +```json +{ + "error": { + "message": "Client-side 'metadata.tags' not allowed in request. 'reject_clientside_metadata_tags'=True. Tags can only be set via API key metadata.", + "type": "bad_request_error", + "param": "metadata.tags", + "code": 400 + } +} +``` + +**Allowed Request Example:** +```bash +curl -X POST http://localhost:4000/chat/completions \ + -H "Authorization: Bearer sk-1234" \ + -H "Content-Type: application/json" \ + -d '{ + "model": "gpt-3.5-turbo", + "messages": [{"role": "user", "content": "Hello"}], + "metadata": { + "custom_field": "value" # Other metadata fields are allowed + } + }' +``` + +### When `reject_clientside_metadata_tags: false` or not set + +All requests are allowed, including those with client-side `metadata.tags`. + +## Setting Tags via API Key + +When `reject_clientside_metadata_tags` is enabled, tags should be set on the API key metadata: + +```bash +curl -X POST http://localhost:4000/key/generate \ + -H "Authorization: Bearer sk-master-key" \ + -H "Content-Type: application/json" \ + -d '{ + "metadata": { + "tags": ["team-a", "production"] + } + }' +``` + +These tags will be automatically inherited by all requests made with that API key. + +## Complete Example Configuration + +```yaml +model_list: + - model_name: gpt-3.5-turbo + litellm_params: + model: gpt-3.5-turbo + api_key: os.environ/OPENAI_API_KEY + +general_settings: + master_key: sk-1234 + database_url: "postgresql://user:password@localhost:5432/litellm" + + # Reject client-side tags + reject_clientside_metadata_tags: true + + # Optional: Also enforce user parameter + enforce_user_param: true +``` + +## Similar Features + +- `enforce_user_param` - Requires all requests to include a 'user' parameter +- Tag-based routing - Use tags for intelligent request routing +- Budget tracking - Track spending per tag + +## Notes + +- This check only applies to LLM API routes (e.g., `/chat/completions`, `/embeddings`) +- Management endpoints (e.g., `/key/generate`) are not affected +- The check validates that client-side `metadata.tags` is not present in the request body +- Other metadata fields can still be passed in requests +- Tags set on API keys will still be applied to all requests diff --git a/docs/my-website/docs/rag_ingest.md b/docs/my-website/docs/rag_ingest.md new file mode 100644 index 00000000000..c824e2127e3 --- /dev/null +++ b/docs/my-website/docs/rag_ingest.md @@ -0,0 +1,273 @@ +# /rag/ingest + +All-in-one document ingestion pipeline: **Upload → Chunk → Embed → Vector Store** + +| Feature | Supported | +|---------|-----------| +| Cost Tracking | ❌ | +| Logging | ✅ | +| Supported Providers | `openai`, `bedrock`, `gemini` | + +## Quick Start + +### OpenAI + +```bash showLineNumbers title="Ingest to OpenAI vector store" +curl -X POST "http://localhost:4000/v1/rag/ingest" \ + -H "Authorization: Bearer sk-1234" \ + -H "Content-Type: application/json" \ + -d "{ + \"file\": { + \"filename\": \"document.txt\", + \"content\": \"$(base64 -i document.txt)\", + \"content_type\": \"text/plain\" + }, + \"ingest_options\": { + \"vector_store\": { + \"custom_llm_provider\": \"openai\" + } + } + }" +``` + +### Bedrock + +```bash showLineNumbers title="Ingest to Bedrock Knowledge Base" +curl -X POST "http://localhost:4000/v1/rag/ingest" \ + -H "Authorization: Bearer sk-1234" \ + -H "Content-Type: application/json" \ + -d "{ + \"file\": { + \"filename\": \"document.txt\", + \"content\": \"$(base64 -i document.txt)\", + \"content_type\": \"text/plain\" + }, + \"ingest_options\": { + \"vector_store\": { + \"custom_llm_provider\": \"bedrock\" + } + } + }" +``` + +### Gemini + +```bash showLineNumbers title="Ingest to Gemini File Search" +curl -X POST "http://localhost:4000/v1/rag/ingest" \ + -H "Authorization: Bearer sk-1234" \ + -H "Content-Type: application/json" \ + -d "{ + \"file\": { + \"filename\": \"document.txt\", + \"content\": \"$(base64 -i document.txt)\", + \"content_type\": \"text/plain\" + }, + \"ingest_options\": { + \"vector_store\": { + \"custom_llm_provider\": \"gemini\" + } + } + }" +``` + +**With Custom Chunking:** + +```bash showLineNumbers title="Ingest with custom chunking" +curl -X POST "http://localhost:4000/v1/rag/ingest" \ + -H "Authorization: Bearer sk-1234" \ + -H "Content-Type: application/json" \ + -d '{ + "file": { + "filename": "document.txt", + "content": "'$(base64 -i document.txt)'", + "content_type": "text/plain" + }, + "ingest_options": { + "vector_store": { + "custom_llm_provider": "gemini" + }, + "chunking_strategy": { + "white_space_config": { + "max_tokens_per_chunk": 200, + "max_overlap_tokens": 20 + } + } + } + }' +``` + +## Response + +```json +{ + "id": "ingest_abc123", + "status": "completed", + "vector_store_id": "vs_xyz789", + "file_id": "file_123" +} +``` + +## Query the Vector Store + +After ingestion, query with `/vector_stores/{vector_store_id}/search`: + +```bash showLineNumbers title="Search the vector store" +curl -X POST "http://localhost:4000/v1/vector_stores/vs_xyz789/search" \ + -H "Authorization: Bearer sk-1234" \ + -H "Content-Type: application/json" \ + -d '{ + "query": "What is the main topic?", + "max_num_results": 5 + }' +``` + +## End-to-End Example + +### OpenAI + +#### 1. Ingest Document + +```bash showLineNumbers title="Step 1: Ingest" +curl -X POST "http://localhost:4000/v1/rag/ingest" \ + -H "Authorization: Bearer sk-1234" \ + -H "Content-Type: application/json" \ + -d "{ + \"file\": { + \"filename\": \"test_document.txt\", + \"content\": \"$(base64 -i test_document.txt)\", + \"content_type\": \"text/plain\" + }, + \"ingest_options\": { + \"name\": \"test-basic-ingest\", + \"vector_store\": { + \"custom_llm_provider\": \"openai\" + } + } + }" +``` + +Response: +```json +{ + "id": "ingest_d834f544-fc5e-4751-902d-fb0bcc183b85", + "status": "completed", + "vector_store_id": "vs_692658d337c4819183f2ad8488d12fc9", + "file_id": "file-M2pJJiWH56cfUP4Fe7rJay" +} +``` + +#### 2. Query + +```bash showLineNumbers title="Step 2: Query" +curl -X POST "http://localhost:4000/v1/vector_stores/vs_692658d337c4819183f2ad8488d12fc9/search" \ + -H "Authorization: Bearer sk-1234" \ + -H "Content-Type: application/json" \ + -d '{ + "query": "What is LiteLLM?", + "custom_llm_provider": "openai" + }' +``` + +Response: +```json +{ + "object": "vector_store.search_results.page", + "search_query": ["What is LiteLLM?"], + "data": [ + { + "file_id": "file-M2pJJiWH56cfUP4Fe7rJay", + "filename": "test_document.txt", + "score": 0.4004629778869299, + "attributes": {}, + "content": [ + { + "type": "text", + "text": "Test document abc123 for RAG ingestion.\nThis is a sample document to test the RAG ingest API.\nLiteLLM provides a unified interface for vector stores." + } + ] + } + ], + "has_more": false, + "next_page": null +} +``` + +## Request Parameters + +### Top-Level + +| Parameter | Type | Required | Description | +|-----------|------|----------|-------------| +| `file` | object | One of file/file_url/file_id required | Base64-encoded file | +| `file.filename` | string | Yes | Filename with extension | +| `file.content` | string | Yes | Base64-encoded content | +| `file.content_type` | string | Yes | MIME type (e.g., `text/plain`) | +| `file_url` | string | One of file/file_url/file_id required | URL to fetch file from | +| `file_id` | string | One of file/file_url/file_id required | Existing file ID | +| `ingest_options` | object | Yes | Pipeline configuration | + +### ingest_options + +| Parameter | Type | Required | Description | +|-----------|------|----------|-------------| +| `vector_store` | object | Yes | Vector store configuration | +| `name` | string | No | Pipeline name for logging | + +### vector_store (OpenAI) + +| Parameter | Type | Default | Description | +|-----------|------|---------|-------------| +| `custom_llm_provider` | string | - | `"openai"` | +| `vector_store_id` | string | auto-create | Existing vector store ID | + +### vector_store (Bedrock) + +| Parameter | Type | Default | Description | +|-----------|------|---------|-------------| +| `custom_llm_provider` | string | - | `"bedrock"` | +| `vector_store_id` | string | auto-create | Existing Knowledge Base ID | +| `wait_for_ingestion` | boolean | `false` | Wait for indexing to complete | +| `ingestion_timeout` | integer | `300` | Timeout in seconds (if waiting) | +| `s3_bucket` | string | auto-create | S3 bucket for documents | +| `s3_prefix` | string | `"data/"` | S3 key prefix | +| `embedding_model` | string | `amazon.titan-embed-text-v2:0` | Bedrock embedding model | +| `aws_region_name` | string | `us-west-2` | AWS region | + +:::info Bedrock Auto-Creation +When `vector_store_id` is omitted, LiteLLM automatically creates: +- S3 bucket for document storage +- OpenSearch Serverless collection +- IAM role with required permissions +- Bedrock Knowledge Base +- Data Source +::: + +## Input Examples + +### File (Base64) + +```json title="Request body" +{ + "file": { + "filename": "document.txt", + "content": "", + "content_type": "text/plain" + }, + "ingest_options": { + "vector_store": {"custom_llm_provider": "openai"} + } +} +``` + +### File URL + +```bash showLineNumbers title="Ingest from URL" +curl -X POST "http://localhost:4000/v1/rag/ingest" \ + -H "Authorization: Bearer sk-1234" \ + -H "Content-Type: application/json" \ + -d '{ + "file_url": "https://example.com/document.pdf", + "ingest_options": {"vector_store": {"custom_llm_provider": "openai"}} + }' +``` + diff --git a/docs/my-website/docs/text_to_speech.md b/docs/my-website/docs/text_to_speech.md index c530e70e4be..ea2a9c2eff3 100644 --- a/docs/my-website/docs/text_to_speech.md +++ b/docs/my-website/docs/text_to_speech.md @@ -103,6 +103,7 @@ litellm --config /path/to/config.yaml | Azure AI Speech Service (AVA)| [Usage](../docs/providers/azure_ai_speech) | | Vertex AI | [Usage](../docs/providers/vertex#text-to-speech-apis) | | Gemini | [Usage](#gemini-text-to-speech) | +| ElevenLabs | [Usage](../docs/providers/elevenlabs#text-to-speech-tts) | ## `/audio/speech` to `/chat/completions` Bridge diff --git a/docs/my-website/docs/vector_stores/search.md b/docs/my-website/docs/vector_stores/search.md index 2ffc8ef12e5..3286b3b01e5 100644 --- a/docs/my-website/docs/vector_stores/search.md +++ b/docs/my-website/docs/vector_stores/search.md @@ -12,7 +12,7 @@ Search a vector store for relevant chunks based on a query and file attributes f | Cost Tracking | ✅ | Tracked per search operation | | Logging | ✅ | Works across all integrations | | End-user Tracking | ✅ | | -| Support LLM Providers | **OpenAI, Azure OpenAI, Bedrock, Vertex RAG Engine, Azure AI, Milvus** | Full vector stores API support across providers | +| Support LLM Providers | **OpenAI, Azure OpenAI, Bedrock, Vertex RAG Engine, Azure AI, Milvus, Gemini** | Full vector stores API support across providers | ## Usage @@ -164,6 +164,41 @@ print(response) [See full Milvus vector store documentation](../providers/milvus_vector_stores.md) + + + + +#### Using Gemini File Search +```python showLineNumbers title="Search Vector Store - Gemini Provider" +import litellm +import os + +# Set credentials +os.environ["GEMINI_API_KEY"] = "your-gemini-api-key" + +response = await litellm.vector_stores.asearch( + vector_store_id="fileSearchStores/your-store-id", + query="What is the capital of France?", + custom_llm_provider="gemini", + max_num_results=5 +) +print(response) +``` + +**With Metadata Filter:** +```python showLineNumbers title="Search with Metadata Filter" +response = await litellm.vector_stores.asearch( + vector_store_id="fileSearchStores/your-store-id", + query="What is LiteLLM?", + custom_llm_provider="gemini", + filters={"author": "John Doe", "category": "documentation"}, + max_num_results=5 +) +print(response) +``` + +[See full Gemini File Search documentation](../providers/gemini_file_search.md) + diff --git a/docs/my-website/sidebars.js b/docs/my-website/sidebars.js index 6fa5fdeced0..e5eba82e32f 100644 --- a/docs/my-website/sidebars.js +++ b/docs/my-website/sidebars.js @@ -412,6 +412,7 @@ const sidebars = { "proxy/pass_through" ] }, + "rag_ingest", "realtime", "rerank", "response_api", @@ -530,6 +531,7 @@ const sidebars = { items: [ "providers/bedrock", "providers/bedrock_embedding", + "providers/bedrock_imported", "providers/bedrock_image_gen", "providers/bedrock_rerank", "providers/bedrock_agentcore", diff --git a/litellm-proxy-extras/litellm_proxy_extras/utils.py b/litellm-proxy-extras/litellm_proxy_extras/utils.py index 73065b050b7..96e1a5106ac 100644 --- a/litellm-proxy-extras/litellm_proxy_extras/utils.py +++ b/litellm-proxy-extras/litellm_proxy_extras/utils.py @@ -130,6 +130,60 @@ class ProxyExtrasDBManager: capture_output=True, ) + @staticmethod + def _is_permission_error(error_message: str) -> bool: + """ + Check if the error message indicates a database permission error. + + Permission errors should NOT be marked as applied, as the migration + did not actually execute successfully. + + Args: + error_message: The error message from Prisma migrate + + Returns: + bool: True if this is a permission error, False otherwise + """ + permission_patterns = [ + r"Database error code: 42501", # PostgreSQL insufficient privilege + r"must be owner of table", + r"permission denied for schema", + r"permission denied for table", + r"must be owner of schema", + ] + + for pattern in permission_patterns: + if re.search(pattern, error_message, re.IGNORECASE): + return True + return False + + @staticmethod + def _is_idempotent_error(error_message: str) -> bool: + """ + Check if the error message indicates an idempotent operation error. + + Idempotent errors (like "column already exists") mean the migration + has effectively already been applied, so it's safe to mark as applied. + + Args: + error_message: The error message from Prisma migrate + + Returns: + bool: True if this is an idempotent error, False otherwise + """ + idempotent_patterns = [ + r"already exists", + r"column .* already exists", + r"duplicate key value violates", + r"relation .* already exists", + r"constraint .* already exists", + ] + + for pattern in idempotent_patterns: + if re.search(pattern, error_message, re.IGNORECASE): + return True + return False + @staticmethod def _resolve_all_migrations( migrations_dir: str, schema_path: str, mark_all_applied: bool = True @@ -320,29 +374,79 @@ class ProxyExtrasDBManager: ) logger.info("✅ All migrations resolved.") return True - elif ( - "P3018" in e.stderr - ): # PostgreSQL error code for duplicate column - logger.info( - "Migration already exists, resolving specific migration" - ) - # Extract the migration name from the error message - migration_match = re.search( - r"Migration name: (\d+_.*)", e.stderr - ) - if migration_match: - migration_name = migration_match.group(1) - logger.info(f"Rolling back migration {migration_name}") - ProxyExtrasDBManager._roll_back_migration( - migration_name + elif "P3018" in e.stderr: + # Check if this is a permission error or idempotent error + if ProxyExtrasDBManager._is_permission_error(e.stderr): + # Permission errors should NOT be marked as applied + # Extract migration name for logging + migration_match = re.search( + r"Migration name: (\d+_.*)", e.stderr ) + migration_name = ( + migration_match.group(1) + if migration_match + else "unknown" + ) + + logger.error( + f"❌ Migration {migration_name} failed due to insufficient permissions. " + f"Please check database user privileges. Error: {e.stderr}" + ) + + # Mark as rolled back and exit with error + if migration_match: + try: + ProxyExtrasDBManager._roll_back_migration( + migration_name + ) + logger.info( + f"Migration {migration_name} marked as rolled back" + ) + except Exception as rollback_error: + logger.warning( + f"Failed to mark migration as rolled back: {rollback_error}" + ) + + # Re-raise the error to prevent silent failures + raise RuntimeError( + f"Migration failed due to permission error. Migration {migration_name} " + f"was NOT applied. Please grant necessary database permissions and retry." + ) from e + + elif ProxyExtrasDBManager._is_idempotent_error(e.stderr): + # Idempotent errors mean the migration has effectively been applied logger.info( - f"Resolving migration {migration_name} that failed due to existing columns" + "Migration failed due to idempotent error (e.g., column already exists), " + "resolving as applied" ) - ProxyExtrasDBManager._resolve_specific_migration( - migration_name + # Extract the migration name from the error message + migration_match = re.search( + r"Migration name: (\d+_.*)", e.stderr ) - logger.info("✅ Migration resolved.") + if migration_match: + migration_name = migration_match.group(1) + logger.info( + f"Rolling back migration {migration_name}" + ) + ProxyExtrasDBManager._roll_back_migration( + migration_name + ) + logger.info( + f"Resolving migration {migration_name} that failed " + f"due to existing schema objects" + ) + ProxyExtrasDBManager._resolve_specific_migration( + migration_name + ) + logger.info("✅ Migration resolved.") + else: + # Unknown P3018 error - log and re-raise for safety + logger.warning( + f"P3018 error encountered but could not classify " + f"as permission or idempotent error. " + f"Error: {e.stderr}" + ) + raise else: # Use prisma db push with increased timeout subprocess.run( diff --git a/litellm/__init__.py b/litellm/__init__.py index 5cb135269bd..ec813cc4bf6 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -1114,6 +1114,7 @@ from .llms.openrouter.chat.transformation import OpenrouterConfig from .llms.datarobot.chat.transformation import DataRobotConfig from .llms.anthropic.chat.transformation import AnthropicConfig from .llms.anthropic.common_utils import AnthropicModelInfo +from .llms.azure.anthropic.transformation import AzureAnthropicConfig from .llms.groq.stt.transformation import GroqSTTConfig from .llms.anthropic.completion.transformation import AnthropicTextConfig from .llms.triton.completion.transformation import TritonConfig @@ -1225,6 +1226,9 @@ from .llms.bedrock.chat.invoke_transformations.amazon_titan_transformation impor from .llms.bedrock.chat.invoke_transformations.base_invoke_transformation import ( AmazonInvokeConfig, ) +from .llms.bedrock.chat.invoke_transformations.amazon_openai_transformation import ( + AmazonBedrockOpenAIConfig, +) from .llms.bedrock.image.amazon_stability1_transformation import AmazonStabilityConfig from .llms.bedrock.image.amazon_stability3_transformation import AmazonStability3Config @@ -1431,6 +1435,7 @@ from .skills.main import ( ) from .containers.main import * from .ocr.main import * +from .rag.main import * from .search.main import * from .realtime_api.main import _arealtime from .fine_tuning.main import * @@ -1466,6 +1471,9 @@ from .vector_stores.vector_store_registry import ( vector_store_registry: Optional[VectorStoreRegistry] = None vector_store_index_registry: Optional[VectorStoreIndexRegistry] = None +### RAG ### +from . import rag + ### CUSTOM LLMs ### from .types.llms.custom_llm import CustomLLMItem from .types.utils import GenericStreamingChunk diff --git a/litellm/completion_extras/litellm_responses_transformation/transformation.py b/litellm/completion_extras/litellm_responses_transformation/transformation.py index e37be860b52..07d9de5a016 100644 --- a/litellm/completion_extras/litellm_responses_transformation/transformation.py +++ b/litellm/completion_extras/litellm_responses_transformation/transformation.py @@ -234,6 +234,11 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): cast(List[Dict[str, Any]], value) ) ) + elif key == "response_format": + # Convert response_format to text.format + text_format = self._transform_response_format_to_text_format(value) + if text_format: + responses_api_request["text"] = text_format # type: ignore elif key in ResponsesAPIOptionalRequestParams.__annotations__.keys(): responses_api_request[key] = value # type: ignore elif key == "metadata": @@ -666,6 +671,63 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): return Reasoning(effort="minimal") return None + def _transform_response_format_to_text_format( + self, response_format: Union[Dict[str, Any], Any] + ) -> Optional[Dict[str, Any]]: + """ + Transform Chat Completion response_format parameter to Responses API text.format parameter. + + Chat Completion response_format structure: + { + "type": "json_schema", + "json_schema": { + "name": "schema_name", + "schema": {...}, + "strict": True + } + } + + Responses API text parameter structure: + { + "format": { + "type": "json_schema", + "name": "schema_name", + "schema": {...}, + "strict": True + } + } + """ + if not response_format: + return None + + if isinstance(response_format, dict): + format_type = response_format.get("type") + + if format_type == "json_schema": + json_schema = response_format.get("json_schema", {}) + return { + "format": { + "type": "json_schema", + "name": json_schema.get("name", "response_schema"), + "schema": json_schema.get("schema", {}), + "strict": json_schema.get("strict", False), + } + } + elif format_type == "json_object": + return { + "format": { + "type": "json_object" + } + } + elif format_type == "text": + return { + "format": { + "type": "text" + } + } + + return None + def _map_responses_status_to_finish_reason(self, status: Optional[str]) -> str: """Map responses API status to chat completion finish_reason""" if not status: diff --git a/litellm/constants.py b/litellm/constants.py index 41441e21f75..cf3d4c6e742 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -1211,3 +1211,7 @@ SENTRY_PII_DENYLIST = [ COROUTINE_CHECKER_MAX_SIZE_IN_MEMORY = int( os.getenv("COROUTINE_CHECKER_MAX_SIZE_IN_MEMORY", 1000) ) + +########################### RAG Text Splitter Constants ########################### +DEFAULT_CHUNK_SIZE = int(os.getenv("DEFAULT_CHUNK_SIZE", 1000)) +DEFAULT_CHUNK_OVERLAP = int(os.getenv("DEFAULT_CHUNK_OVERLAP", 200)) diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index 0f5195e31af..9ef26d23ce2 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -1031,6 +1031,57 @@ def completion_cost( # noqa: PLR0915 billed_units.get("search_units") or 1 ) # cohere charges per request by default. completion_tokens = search_units + elif ( + call_type == CallTypes.search.value + or call_type == CallTypes.asearch.value + ): + from litellm.search import search_provider_cost_per_query + + # Extract number_of_queries from optional_params or default to 1 + number_of_queries = 1 + if optional_params is not None: + # Check if query is a list (multiple queries) + query = optional_params.get("query") + if isinstance(query, list): + number_of_queries = len(query) + elif query is not None: + number_of_queries = 1 + + search_model = model or "" + if custom_llm_provider and "/" not in search_model: + # If model is like "tavily-search", construct "tavily/search" for cost lookup + search_model = f"{custom_llm_provider}/search" + + prompt_cost, completion_cost_result = search_provider_cost_per_query( + model=search_model, + custom_llm_provider=custom_llm_provider, + number_of_queries=number_of_queries, + optional_params=optional_params, + ) + + # Return the total cost (prompt_cost + completion_cost, but for search it's just prompt_cost) + _final_cost = prompt_cost + completion_cost_result + + # Apply discount + original_cost = _final_cost + _final_cost, discount_percent, discount_amount = _apply_cost_discount( + base_cost=_final_cost, + custom_llm_provider=custom_llm_provider, + ) + + # Store cost breakdown in logging object if available + _store_cost_breakdown_in_logging_obj( + litellm_logging_obj=litellm_logging_obj, + prompt_tokens_cost_usd_dollar=prompt_cost, + completion_tokens_cost_usd_dollar=completion_cost_result, + cost_for_built_in_tools_cost_usd_dollar=0.0, + total_cost_usd_dollar=_final_cost, + original_cost=original_cost, + discount_percent=discount_percent, + discount_amount=discount_amount, + ) + + return _final_cost elif call_type == CallTypes.arealtime.value and isinstance( completion_response, LiteLLMRealtimeStreamLoggingObject ): diff --git a/litellm/images/main.py b/litellm/images/main.py index 5b6cc995ec8..878fce83f17 100644 --- a/litellm/images/main.py +++ b/litellm/images/main.py @@ -19,6 +19,8 @@ from litellm.llms.custom_llm import CustomLLM #################### Initialize provider clients #################### llm_http_handler: BaseLLMHTTPHandler = BaseLLMHTTPHandler() +from openai.types.audio.transcription_create_params import FileTypes # type: ignore + from litellm.main import ( azure_chat_completions, base_llm_aiohttp_handler, @@ -26,7 +28,6 @@ from litellm.main import ( bedrock_image_generation, openai_chat_completions, openai_image_variations, - vertex_image_generation, ) ########################################### @@ -36,7 +37,6 @@ from litellm.types.llms.openai import ImageGenerationRequestQuality from litellm.types.router import GenericLiteLLMParams from litellm.types.utils import ( LITELLM_IMAGE_VARIATION_PROVIDERS, - FileTypes, LlmProviders, all_litellm_params, ) @@ -344,6 +344,7 @@ def image_generation( # noqa: PLR0915 litellm.LlmProviders.GEMINI, litellm.LlmProviders.FAL_AI, litellm.LlmProviders.RUNWAYML, + litellm.LlmProviders.VERTEX_AI, ): if image_generation_config is None: raise ValueError( @@ -430,46 +431,6 @@ def image_generation( # noqa: PLR0915 api_base=api_base, api_key=api_key, ) - elif custom_llm_provider == "vertex_ai": - vertex_ai_project = ( - optional_params.pop("vertex_project", None) - or optional_params.pop("vertex_ai_project", None) - or litellm.vertex_project - or get_secret_str("VERTEXAI_PROJECT") - ) - vertex_ai_location = ( - optional_params.pop("vertex_location", None) - or optional_params.pop("vertex_ai_location", None) - or litellm.vertex_location - or get_secret_str("VERTEXAI_LOCATION") - ) - vertex_credentials = ( - optional_params.pop("vertex_credentials", None) - or optional_params.pop("vertex_ai_credentials", None) - or get_secret_str("VERTEXAI_CREDENTIALS") - ) - - api_base = ( - api_base - or litellm.api_base - or get_secret_str("VERTEXAI_API_BASE") - or get_secret_str("VERTEX_API_BASE") - ) - - model_response = vertex_image_generation.image_generation( - model=model, - prompt=prompt, - timeout=timeout, - logging_obj=litellm_logging_obj, - optional_params=optional_params, - model_response=model_response, - vertex_project=vertex_ai_project, - vertex_location=vertex_ai_location, - vertex_credentials=vertex_credentials, - aimg_generation=aimg_generation, - api_base=api_base, - client=client, - ) elif ( custom_llm_provider in litellm._custom_providers ): # Assume custom LLM provider diff --git a/litellm/litellm_core_utils/get_litellm_params.py b/litellm/litellm_core_utils/get_litellm_params.py index d5675a2ac51..5279cb26b69 100644 --- a/litellm/litellm_core_utils/get_litellm_params.py +++ b/litellm/litellm_core_utils/get_litellm_params.py @@ -121,5 +121,16 @@ def get_litellm_params( "use_litellm_proxy": use_litellm_proxy, "litellm_request_debug": litellm_request_debug, "aws_region_name": kwargs.get("aws_region_name"), + # AWS credentials for Bedrock/Sagemaker + "aws_access_key_id": kwargs.get("aws_access_key_id"), + "aws_secret_access_key": kwargs.get("aws_secret_access_key"), + "aws_session_token": kwargs.get("aws_session_token"), + "aws_session_name": kwargs.get("aws_session_name"), + "aws_profile_name": kwargs.get("aws_profile_name"), + "aws_role_name": kwargs.get("aws_role_name"), + "aws_web_identity_token": kwargs.get("aws_web_identity_token"), + "aws_sts_endpoint": kwargs.get("aws_sts_endpoint"), + "aws_external_id": kwargs.get("aws_external_id"), + "aws_bedrock_runtime_endpoint": kwargs.get("aws_bedrock_runtime_endpoint"), } return litellm_params diff --git a/litellm/litellm_core_utils/get_llm_provider_logic.py b/litellm/litellm_core_utils/get_llm_provider_logic.py index eefe680217d..1efea63beb7 100644 --- a/litellm/litellm_core_utils/get_llm_provider_logic.py +++ b/litellm/litellm_core_utils/get_llm_provider_logic.py @@ -22,6 +22,19 @@ def _is_non_openai_azure_model(model: str) -> bool: return False +def _is_azure_anthropic_model(model: str) -> Optional[str]: + try: + model_parts = model.split("/", 1) + if len(model_parts) > 1: + model_name = model_parts[1].lower() + # Check if model name contains claude + if "claude" in model_name or model_name.startswith("claude"): + return model_parts[1] # Return model name without "azure/" prefix + except Exception: + pass + return None + + def handle_cohere_chat_model_custom_llm_provider( model: str, custom_llm_provider: Optional[str] = None ) -> Tuple[str, Optional[str]]: @@ -123,6 +136,11 @@ def get_llm_provider( # noqa: PLR0915 # AZURE AI-Studio Logic - Azure AI Studio supports AZURE/Cohere # If User passes azure/command-r-plus -> we should send it to cohere_chat/command-r-plus if model.split("/", 1)[0] == "azure": + # Check if it's an Azure Anthropic model (claude models) + azure_anthropic_model = _is_azure_anthropic_model(model) + if azure_anthropic_model: + custom_llm_provider = "azure_anthropic" + return azure_anthropic_model, custom_llm_provider, dynamic_api_key, api_base if _is_non_openai_azure_model(model): custom_llm_provider = "openai" return model, custom_llm_provider, dynamic_api_key, api_base diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index cf8d5f2cc2d..e9bd082b9d5 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -69,6 +69,7 @@ from litellm.litellm_core_utils.redact_messages import ( redact_message_input_output_from_logging, ) from litellm.llms.base_llm.ocr.transformation import OCRResponse +from litellm.llms.base_llm.search.transformation import SearchResponse from litellm.responses.utils import ResponseAPILoggingUtils from litellm.types.containers.main import ContainerObject from litellm.types.llms.openai import ( @@ -1314,6 +1315,7 @@ class Logging(LiteLLMLoggingBaseClass): OpenAIFileObject, LiteLLMRealtimeStreamLoggingObject, OpenAIModerationResponse, + "SearchResponse", ], cache_hit: Optional[bool] = None, litellm_model_name: Optional[str] = None, @@ -1726,8 +1728,11 @@ class Logging(LiteLLMLoggingBaseClass): or isinstance(logging_result, LiteLLMRealtimeStreamLoggingObject) or isinstance(logging_result, OpenAIModerationResponse) or isinstance(logging_result, OCRResponse) # OCR + or isinstance(logging_result, SearchResponse) # Search API or isinstance(logging_result, dict) and logging_result.get("object") == "vector_store.search_results.page" + or isinstance(logging_result, dict) + and logging_result.get("object") == "search" # Search API (dict format) or isinstance(logging_result, VideoObject) or isinstance(logging_result, ContainerObject) or (self.call_type == CallTypes.call_mcp_tool.value) diff --git a/litellm/litellm_core_utils/streaming_chunk_builder_utils.py b/litellm/litellm_core_utils/streaming_chunk_builder_utils.py index ddcf81b5ba5..c332e5f88f7 100644 --- a/litellm/litellm_core_utils/streaming_chunk_builder_utils.py +++ b/litellm/litellm_core_utils/streaming_chunk_builder_utils.py @@ -18,6 +18,7 @@ from litellm.types.utils import ( ModelResponseStream, PromptTokensDetailsWrapper, Usage, + ServerToolUse ) from litellm.utils import print_verbose, token_counter @@ -418,7 +419,8 @@ class ChunkProcessor: ## anthropic prompt caching information ## cache_creation_input_tokens: Optional[int] = None cache_read_input_tokens: Optional[int] = None - + + server_tool_use: Optional[ServerToolUse] = None web_search_requests: Optional[int] = None completion_tokens_details: Optional[CompletionTokensDetails] = None prompt_tokens_details: Optional[PromptTokensDetailsWrapper] = None @@ -462,6 +464,8 @@ class ChunkProcessor: completion_tokens_details = usage_chunk_dict[ "completion_tokens_details" ] + if hasattr(usage_chunk, 'server_tool_use') and usage_chunk.server_tool_use is not None: + server_tool_use = usage_chunk.server_tool_use if ( usage_chunk_dict["prompt_tokens_details"] is not None and getattr( @@ -483,6 +487,7 @@ class ChunkProcessor: completion_tokens=completion_tokens, cache_creation_input_tokens=cache_creation_input_tokens, cache_read_input_tokens=cache_read_input_tokens, + server_tool_use=server_tool_use, web_search_requests=web_search_requests, completion_tokens_details=completion_tokens_details, prompt_tokens_details=prompt_tokens_details, @@ -513,6 +518,9 @@ class ChunkProcessor: "cache_read_input_tokens" ] + server_tool_use: Optional[ServerToolUse] = calculated_usage_per_chunk[ + "server_tool_use" + ] web_search_requests: Optional[int] = calculated_usage_per_chunk[ "web_search_requests" ] @@ -576,6 +584,8 @@ class ChunkProcessor: if prompt_tokens_details is not None: returned_usage.prompt_tokens_details = prompt_tokens_details + if server_tool_use is not None: + returned_usage.server_tool_use = server_tool_use if web_search_requests is not None: if returned_usage.prompt_tokens_details is None: returned_usage.prompt_tokens_details = PromptTokensDetailsWrapper( diff --git a/litellm/llms/anthropic/chat/handler.py b/litellm/llms/anthropic/chat/handler.py index b7b39f10395..b363b747de5 100644 --- a/litellm/llms/anthropic/chat/handler.py +++ b/litellm/llms/anthropic/chat/handler.py @@ -42,6 +42,7 @@ from litellm.types.llms.openai import ( ChatCompletionRedactedThinkingBlock, ChatCompletionThinkingBlock, ChatCompletionToolCallChunk, + ChatCompletionToolCallFunctionChunk, ) from litellm.types.utils import ( Delta, @@ -550,15 +551,18 @@ class ModelResponseIterator: if "text" in content_block["delta"]: text = content_block["delta"]["text"] elif "partial_json" in content_block["delta"]: - tool_use = { - "id": None, - "type": "function", - "function": { - "name": None, - "arguments": content_block["delta"]["partial_json"], + tool_use = cast( + ChatCompletionToolCallChunk, + { + "id": None, + "type": "function", + "function": { + "name": None, + "arguments": content_block["delta"]["partial_json"], + }, + "index": self.tool_index, }, - "index": self.tool_index, - } + ) elif "citation" in content_block["delta"]: provider_specific_fields["citation"] = content_block["delta"]["citation"] elif ( @@ -569,7 +573,7 @@ class ModelResponseIterator: ChatCompletionThinkingBlock( type="thinking", thinking=content_block["delta"].get("thinking") or "", - signature=content_block["delta"].get("signature"), + signature=str(content_block["delta"].get("signature") or ""), ) ] provider_specific_fields["thinking_blocks"] = thinking_blocks @@ -625,7 +629,7 @@ class ModelResponseIterator: return content_block_start - def chunk_parser(self, chunk: dict) -> ModelResponseStream: + def chunk_parser(self, chunk: dict) -> ModelResponseStream: # noqa: PLR0915 try: type_chunk = chunk.get("type", "") or "" @@ -672,15 +676,32 @@ class ModelResponseIterator: text = content_block_start["content_block"]["text"] elif content_block_start["content_block"]["type"] == "tool_use": self.tool_index += 1 - tool_use = { - "id": content_block_start["content_block"]["id"], - "type": "function", - "function": { - "name": content_block_start["content_block"]["name"], - "arguments": "", - }, - "index": self.tool_index, - } + tool_use = ChatCompletionToolCallChunk( + id=content_block_start["content_block"]["id"], + type="function", + function=ChatCompletionToolCallFunctionChunk( + name=content_block_start["content_block"]["name"], + arguments="", + ), + index=self.tool_index, + ) + # Include caller information if present (for programmatic tool calling) + if "caller" in content_block_start["content_block"]: + caller_data = content_block_start["content_block"]["caller"] + if caller_data: + tool_use["caller"] = cast(Dict[str, Any], caller_data) # type: ignore[typeddict-item] + elif content_block_start["content_block"]["type"] == "server_tool_use": + # Handle server tool use (for tool search) + self.tool_index += 1 + tool_use = ChatCompletionToolCallChunk( + id=content_block_start["content_block"]["id"], + type="function", + function=ChatCompletionToolCallFunctionChunk( + name=content_block_start["content_block"]["name"], + arguments="", + ), + index=self.tool_index, + ) elif ( content_block_start["content_block"]["type"] == "redacted_thinking" ): @@ -696,17 +717,21 @@ class ModelResponseIterator: # check if tool call content block is_empty = self.check_empty_tool_call_args() if is_empty: - tool_use = { - "id": None, - "type": "function", - "function": { - "name": None, - "arguments": "{}", - }, - "index": self.tool_index, - } + tool_use = ChatCompletionToolCallChunk( + id=None, # type: ignore[typeddict-item] + type="function", + function=ChatCompletionToolCallFunctionChunk( + name=None, # type: ignore[typeddict-item] + arguments="{}", + ), + index=self.tool_index, + ) # Reset response_format tool tracking when block stops self.is_response_format_tool = False + elif type_chunk == "tool_result": + # Handle tool_result blocks (for tool search results with tool_reference) + # These are automatically handled by Anthropic API, we just pass them through + pass elif type_chunk == "message_delta": finish_reason, usage = self._handle_message_delta(chunk) elif type_chunk == "message_start": diff --git a/litellm/llms/anthropic/chat/transformation.py b/litellm/llms/anthropic/chat/transformation.py index 623a98c132f..ac1c9b1e000 100644 --- a/litellm/llms/anthropic/chat/transformation.py +++ b/litellm/llms/anthropic/chat/transformation.py @@ -54,7 +54,10 @@ from litellm.types.utils import ( CompletionTokensDetailsWrapper, ) from litellm.types.utils import Message as LitellmMessage -from litellm.types.utils import PromptTokensDetailsWrapper, ServerToolUse +from litellm.types.utils import ( + PromptTokensDetailsWrapper, + ServerToolUse, +) from litellm.utils import ( ModelResponse, Usage, @@ -187,7 +190,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): ) return _tool_choice - def _map_tool_helper( + def _map_tool_helper( # noqa: PLR0915 self, tool: ChatCompletionToolParam ) -> Tuple[Optional[AllAnthropicToolsValues], Optional[AnthropicMcpServerTool]]: returned_tool: Optional[AllAnthropicToolsValues] = None @@ -250,9 +253,10 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): returned_tool = _computer_tool elif any(tool["type"].startswith(t) for t in ANTHROPIC_HOSTED_TOOLS): - function_name = tool.get("name", tool.get("function", {}).get("name")) - if function_name is None or not isinstance(function_name, str): + function_name_obj = tool.get("name", tool.get("function", {}).get("name")) + if function_name_obj is None or not isinstance(function_name_obj, str): raise ValueError("Missing required parameter: name") + function_name = function_name_obj additional_tool_params = {} for k, v in tool.items(): @@ -268,6 +272,30 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): mcp_server = self._map_openai_mcp_server_tool( cast(OpenAIMcpServerTool, tool) ) + elif tool["type"] == "tool_search_tool_regex_20251119": + # Tool search tool using regex + from litellm.types.llms.anthropic import AnthropicToolSearchToolRegex + + tool_name_obj = tool.get("name", "tool_search_tool_regex") + if not isinstance(tool_name_obj, str): + raise ValueError("Tool search tool must have a valid name") + tool_name = tool_name_obj + returned_tool = AnthropicToolSearchToolRegex( + type="tool_search_tool_regex_20251119", + name=tool_name, + ) + elif tool["type"] == "tool_search_tool_bm25_20251119": + # Tool search tool using BM25 + from litellm.types.llms.anthropic import AnthropicToolSearchToolBM25 + + tool_name_obj = tool.get("name", "tool_search_tool_bm25") + if not isinstance(tool_name_obj, str): + raise ValueError("Tool search tool must have a valid name") + tool_name = tool_name_obj + returned_tool = AnthropicToolSearchToolBM25( + type="tool_search_tool_bm25_20251119", + name=tool_name, + ) if returned_tool is None and mcp_server is None: raise ValueError(f"Unsupported tool type: {tool['type']}") @@ -275,14 +303,67 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): _cache_control = tool.get("cache_control", None) _cache_control_function = tool.get("function", {}).get("cache_control", None) if returned_tool is not None: - if _cache_control is not None: - returned_tool["cache_control"] = _cache_control - elif _cache_control_function is not None and isinstance( - _cache_control_function, dict - ): - returned_tool["cache_control"] = ChatCompletionCachedContent( - **_cache_control_function # type: ignore - ) + # Only set cache_control on tools that support it (not tool search tools) + tool_type = returned_tool.get("type", "") + if tool_type not in ("tool_search_tool_regex_20251119", "tool_search_tool_bm25_20251119"): + if _cache_control is not None: + returned_tool["cache_control"] = _cache_control # type: ignore[typeddict-item] + elif _cache_control_function is not None and isinstance( + _cache_control_function, dict + ): + returned_tool["cache_control"] = ChatCompletionCachedContent( # type: ignore[typeddict-item] + **_cache_control_function # type: ignore + ) + + ## check if defer_loading is set in the tool + _defer_loading = tool.get("defer_loading", None) + _defer_loading_function = tool.get("function", {}).get("defer_loading", None) + if returned_tool is not None: + # Only set defer_loading on tools that support it (not tool search tools or computer tools) + tool_type = returned_tool.get("type", "") + if tool_type not in ("tool_search_tool_regex_20251119", "tool_search_tool_bm25_20251119", "computer_20241022", "computer_20250124"): + if _defer_loading is not None: + if not isinstance(_defer_loading, bool): + raise ValueError("defer_loading must be a boolean") + returned_tool["defer_loading"] = _defer_loading # type: ignore[typeddict-item] + elif _defer_loading_function is not None: + if not isinstance(_defer_loading_function, bool): + raise ValueError("defer_loading must be a boolean") + returned_tool["defer_loading"] = _defer_loading_function # type: ignore[typeddict-item] + + ## check if allowed_callers is set in the tool + _allowed_callers = tool.get("allowed_callers", None) + _allowed_callers_function = tool.get("function", {}).get("allowed_callers", None) + if returned_tool is not None: + # Only set allowed_callers on tools that support it (not tool search tools or computer tools) + tool_type = returned_tool.get("type", "") + if tool_type not in ("tool_search_tool_regex_20251119", "tool_search_tool_bm25_20251119", "computer_20241022", "computer_20250124"): + if _allowed_callers is not None: + if not isinstance(_allowed_callers, list) or not all( + isinstance(item, str) for item in _allowed_callers + ): + raise ValueError("allowed_callers must be a list of strings") + returned_tool["allowed_callers"] = _allowed_callers # type: ignore[typeddict-item] + elif _allowed_callers_function is not None: + if not isinstance(_allowed_callers_function, list) or not all( + isinstance(item, str) for item in _allowed_callers_function + ): + raise ValueError("allowed_callers must be a list of strings") + returned_tool["allowed_callers"] = _allowed_callers_function # type: ignore[typeddict-item] + + ## check if input_examples is set in the tool + _input_examples = tool.get("input_examples", None) + _input_examples_function = tool.get("function", {}).get("input_examples", None) + if returned_tool is not None: + # Only set input_examples on user-defined tools (type "custom" or no type) + tool_type = returned_tool.get("type", "") + if tool_type == "custom" or (tool_type == "" and "name" in returned_tool): + if _input_examples is not None and isinstance(_input_examples, list): + returned_tool["input_examples"] = _input_examples # type: ignore[typeddict-item] + elif _input_examples_function is not None and isinstance( + _input_examples_function, list + ): + returned_tool["input_examples"] = _input_examples_function # type: ignore[typeddict-item] return returned_tool, mcp_server @@ -334,6 +415,82 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): mcp_servers.append(mcp_server_tool) return anthropic_tools, mcp_servers + def _detect_tool_search_tools(self, tools: Optional[List]) -> bool: + """Check if tool search tools are present in the tools list.""" + if not tools: + return False + + for tool in tools: + tool_type = tool.get("type", "") + if tool_type in ["tool_search_tool_regex_20251119", "tool_search_tool_bm25_20251119"]: + return True + return False + + def _separate_deferred_tools( + self, tools: List + ) -> Tuple[List, List]: + """ + Separate tools into deferred and non-deferred lists. + + Returns: + Tuple of (non_deferred_tools, deferred_tools) + """ + non_deferred = [] + deferred = [] + + for tool in tools: + if tool.get("defer_loading", False): + deferred.append(tool) + else: + non_deferred.append(tool) + + return non_deferred, deferred + + def _expand_tool_references( + self, + content: List, + deferred_tools: List, + ) -> List: + """ + Expand tool_reference blocks to full tool definitions. + + When Anthropic's tool search returns results, it includes tool_reference blocks + that reference tools by name. This method expands those references to full + tool definitions from the deferred_tools catalog. + + Args: + content: Response content that may contain tool_reference blocks + deferred_tools: List of deferred tools that can be referenced + + Returns: + Content with tool_reference blocks expanded to full tool definitions + """ + if not deferred_tools: + return content + + # Create a mapping of tool names to tool definitions + tool_map = {} + for tool in deferred_tools: + tool_name = tool.get("name") or tool.get("function", {}).get("name") + if tool_name: + tool_map[tool_name] = tool + + # Expand tool references in content + expanded_content = [] + for item in content: + if isinstance(item, dict) and item.get("type") == "tool_reference": + tool_name = item.get("tool_name") + if tool_name and tool_name in tool_map: + # Replace reference with full tool definition + expanded_content.append(tool_map[tool_name]) + else: + # Keep the reference if we can't find the tool + expanded_content.append(item) + else: + expanded_content.append(item) + + return expanded_content + def _map_stop_sequences( self, stop: Optional[Union[str, List[str]]] ) -> Optional[List[str]]: @@ -822,6 +979,17 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): "messages": anthropic_messages, **optional_params, } + + ## Handle output_config (Anthropic-specific parameter) + if "output_config" in optional_params: + output_config = optional_params.get("output_config") + if output_config and isinstance(output_config, dict): + effort = output_config.get("effort") + if effort and effort not in ["high", "medium", "low"]: + raise ValueError( + f"Invalid effort value: {effort}. Must be one of: 'high', 'medium', 'low'" + ) + data["output_config"] = output_config return data @@ -870,18 +1038,40 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): text_content += content["text"] ## TOOL CALLING elif content["type"] == "tool_use": - tool_calls.append( - ChatCompletionToolCallChunk( - id=content["id"], - type="function", - function=ChatCompletionToolCallFunctionChunk( - name=content["name"], - arguments=json.dumps(content["input"]), - ), - index=idx, - ) + tool_call = ChatCompletionToolCallChunk( + id=content["id"], + type="function", + function=ChatCompletionToolCallFunctionChunk( + name=content["name"], + arguments=json.dumps(content["input"]), + ), + index=idx, ) - + # Include caller information if present (for programmatic tool calling) + if "caller" in content: + tool_call["caller"] = cast(Dict[str, Any], content["caller"]) # type: ignore[typeddict-item] + tool_calls.append(tool_call) + ## SERVER TOOL USE (for tool search) + elif content["type"] == "server_tool_use": + # Server tool use blocks are for tool search - treat as tool calls + tool_call = ChatCompletionToolCallChunk( + id=content["id"], + type="function", + function=ChatCompletionToolCallFunctionChunk( + name=content["name"], + arguments=json.dumps(content.get("input", {})), + ), + index=idx, + ) + # Include caller information if present (for programmatic tool calling) + if "caller" in content: + tool_call["caller"] = cast(Dict[str, Any], content["caller"]) # type: ignore[typeddict-item] + tool_calls.append(tool_call) + ## TOOL SEARCH TOOL RESULT (skip - this is metadata about tool discovery) + elif content["type"] == "tool_search_tool_result": + # This block contains tool_references that were discovered + # We don't need to include this in the response as it's internal metadata + pass elif content.get("thinking", None) is not None: if thinking_blocks is None: thinking_blocks = [] @@ -916,7 +1106,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): return text_content, citations, thinking_blocks, reasoning_content, tool_calls def calculate_usage( - self, usage_object: dict, reasoning_content: Optional[str] + self, usage_object: dict, reasoning_content: Optional[str], completion_response: Optional[dict] = None ) -> Usage: # NOTE: Sometimes the usage object has None set explicitly for token counts, meaning .get() & key access returns None, and we need to account for this prompt_tokens = usage_object.get("input_tokens", 0) or 0 @@ -926,6 +1116,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): cache_read_input_tokens: int = 0 cache_creation_token_details: Optional[CacheCreationTokenDetails] = None web_search_requests: Optional[int] = None + tool_search_requests: Optional[int] = None if ( "cache_creation_input_tokens" in _usage and _usage["cache_creation_input_tokens"] is not None @@ -946,6 +1137,25 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): web_search_requests = cast( int, _usage["server_tool_use"]["web_search_requests"] ) + if ( + "tool_search_requests" in _usage["server_tool_use"] + and _usage["server_tool_use"]["tool_search_requests"] is not None + ): + tool_search_requests = cast( + int, _usage["server_tool_use"]["tool_search_requests"] + ) + + # Count tool_search_requests from content blocks if not in usage + # Anthropic doesn't always include tool_search_requests in the usage object + if tool_search_requests is None and completion_response is not None: + tool_search_count = 0 + for content in completion_response.get("content", []): + if content.get("type") == "server_tool_use": + tool_name = content.get("name", "") + if "tool_search" in tool_name: + tool_search_count += 1 + if tool_search_count > 0: + tool_search_requests = tool_search_count if "cache_creation" in _usage and _usage["cache_creation"] is not None: cache_creation_token_details = CacheCreationTokenDetails( @@ -982,8 +1192,11 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): cache_read_input_tokens=cache_read_input_tokens, completion_tokens_details=completion_token_details, server_tool_use=( - ServerToolUse(web_search_requests=web_search_requests) - if web_search_requests is not None + ServerToolUse( + web_search_requests=web_search_requests, + tool_search_requests=tool_search_requests, + ) + if (web_search_requests is not None or tool_search_requests is not None) else None ), ) @@ -1077,6 +1290,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): usage = self.calculate_usage( usage_object=completion_response["usage"], reasoning_content=reasoning_content, + completion_response=completion_response, ) setattr(model_response, "usage", usage) # type: ignore diff --git a/litellm/llms/anthropic/common_utils.py b/litellm/llms/anthropic/common_utils.py index 0d00a3b4632..9f5688f9e01 100644 --- a/litellm/llms/anthropic/common_utils.py +++ b/litellm/llms/anthropic/common_utils.py @@ -88,6 +88,86 @@ class AnthropicModelInfo(BaseLLMModelInfo): return True return False + def is_tool_search_used(self, tools: Optional[List]) -> bool: + """ + Check if tool search tools are present in the tools list. + """ + if not tools: + return False + + for tool in tools: + tool_type = tool.get("type", "") + if tool_type in ["tool_search_tool_regex_20251119", "tool_search_tool_bm25_20251119"]: + return True + return False + + def is_programmatic_tool_calling_used(self, tools: Optional[List]) -> bool: + """ + Check if programmatic tool calling is being used (tools with allowed_callers field). + + Returns True if any tool has allowed_callers containing 'code_execution_20250825'. + """ + if not tools: + return False + + for tool in tools: + # Check top-level allowed_callers + allowed_callers = tool.get("allowed_callers", None) + if allowed_callers and isinstance(allowed_callers, list): + if "code_execution_20250825" in allowed_callers: + return True + + # Check function.allowed_callers for OpenAI format tools + function = tool.get("function", {}) + if isinstance(function, dict): + function_allowed_callers = function.get("allowed_callers", None) + if function_allowed_callers and isinstance(function_allowed_callers, list): + if "code_execution_20250825" in function_allowed_callers: + return True + + return False + + def is_input_examples_used(self, tools: Optional[List]) -> bool: + """ + Check if input_examples is being used in any tools. + + Returns True if any tool has input_examples field. + """ + if not tools: + return False + + for tool in tools: + # Check top-level input_examples + input_examples = tool.get("input_examples", None) + if input_examples and isinstance(input_examples, list) and len(input_examples) > 0: + return True + + # Check function.input_examples for OpenAI format tools + function = tool.get("function", {}) + if isinstance(function, dict): + function_input_examples = function.get("input_examples", None) + if function_input_examples and isinstance(function_input_examples, list) and len(function_input_examples) > 0: + return True + + return False + + def is_effort_used(self, optional_params: Optional[dict]) -> bool: + """ + Check if effort parameter is being used via output_config. + + Returns True if output_config with effort field is present. + """ + if not optional_params: + return False + + output_config = optional_params.get("output_config") + if output_config and isinstance(output_config, dict): + effort = output_config.get("effort") + if effort and isinstance(effort, str): + return True + + return False + def _get_user_anthropic_beta_headers( self, anthropic_beta_header: Optional[str] ) -> Optional[List[str]]: @@ -122,6 +202,10 @@ class AnthropicModelInfo(BaseLLMModelInfo): pdf_used: bool = False, file_id_used: bool = False, mcp_server_used: bool = False, + tool_search_used: bool = False, + programmatic_tool_calling_used: bool = False, + input_examples_used: bool = False, + effort_used: bool = False, is_vertex_request: bool = False, user_anthropic_beta_headers: Optional[List[str]] = None, ) -> dict: @@ -138,6 +222,15 @@ class AnthropicModelInfo(BaseLLMModelInfo): betas.add("code-execution-2025-05-22") if mcp_server_used: betas.add("mcp-client-2025-04-04") + # Tool search, programmatic tool calling, and input_examples all use the same beta header + if tool_search_used or programmatic_tool_calling_used or input_examples_used: + from litellm.types.llms.anthropic import ANTHROPIC_TOOL_SEARCH_BETA_HEADER + betas.add(ANTHROPIC_TOOL_SEARCH_BETA_HEADER) + + # Effort parameter uses a separate beta header + if effort_used: + from litellm.types.llms.anthropic import ANTHROPIC_EFFORT_BETA_HEADER + betas.add(ANTHROPIC_EFFORT_BETA_HEADER) headers = { "anthropic-version": anthropic_version or "2023-06-01", @@ -182,6 +275,10 @@ class AnthropicModelInfo(BaseLLMModelInfo): ) pdf_used = self.is_pdf_used(messages=messages) file_id_used = self.is_file_id_used(messages=messages) + tool_search_used = self.is_tool_search_used(tools=tools) + programmatic_tool_calling_used = self.is_programmatic_tool_calling_used(tools=tools) + input_examples_used = self.is_input_examples_used(tools=tools) + effort_used = self.is_effort_used(optional_params=optional_params) user_anthropic_beta_headers = self._get_user_anthropic_beta_headers( anthropic_beta_header=headers.get("anthropic-beta") ) @@ -194,6 +291,10 @@ class AnthropicModelInfo(BaseLLMModelInfo): is_vertex_request=optional_params.get("is_vertex_request", False), user_anthropic_beta_headers=user_anthropic_beta_headers, mcp_server_used=mcp_server_used, + tool_search_used=tool_search_used, + programmatic_tool_calling_used=programmatic_tool_calling_used, + input_examples_used=input_examples_used, + effort_used=effort_used, ) headers = {**headers, **anthropic_headers} diff --git a/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py b/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py index 0e905014fe2..98e57f279cf 100644 --- a/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py +++ b/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py @@ -645,7 +645,7 @@ class LiteLLMAnthropicMessagesAdapter: type="tool_use", id=choice.delta.tool_calls[0].id or str(uuid.uuid4()), name=choice.delta.tool_calls[0].function.name or "", - input={}, + input={}, # type: ignore[typeddict-item] ) elif isinstance(choice, StreamingChoices) and hasattr( choice.delta, "thinking_blocks" diff --git a/litellm/llms/azure/anthropic/__init__.py b/litellm/llms/azure/anthropic/__init__.py new file mode 100644 index 00000000000..233f22999f0 --- /dev/null +++ b/litellm/llms/azure/anthropic/__init__.py @@ -0,0 +1,12 @@ +""" +Azure Anthropic provider - supports Claude models via Azure Foundry +""" +from .handler import AzureAnthropicChatCompletion +from .transformation import AzureAnthropicConfig + +try: + from .messages_transformation import AzureAnthropicMessagesConfig + __all__ = ["AzureAnthropicChatCompletion", "AzureAnthropicConfig", "AzureAnthropicMessagesConfig"] +except ImportError: + __all__ = ["AzureAnthropicChatCompletion", "AzureAnthropicConfig"] + diff --git a/litellm/llms/azure/anthropic/handler.py b/litellm/llms/azure/anthropic/handler.py new file mode 100644 index 00000000000..cf4765190c2 --- /dev/null +++ b/litellm/llms/azure/anthropic/handler.py @@ -0,0 +1,236 @@ +""" +Azure Anthropic handler - reuses AnthropicChatCompletion logic with Azure authentication +""" +import copy +import json +from typing import TYPE_CHECKING, Callable, Union + +import httpx + +import litellm +from litellm.llms.anthropic.chat.handler import AnthropicChatCompletion +from litellm.llms.custom_httpx.http_handler import ( + AsyncHTTPHandler, + HTTPHandler, +) +from litellm.types.utils import ModelResponse +from litellm.utils import CustomStreamWrapper + +from .transformation import AzureAnthropicConfig + +if TYPE_CHECKING: + pass + + +class AzureAnthropicChatCompletion(AnthropicChatCompletion): + """ + Azure Anthropic chat completion handler. + Reuses all Anthropic logic but with Azure authentication. + """ + + def __init__(self) -> None: + super().__init__() + + def completion( + self, + model: str, + messages: list, + api_base: str, + custom_llm_provider: str, + custom_prompt_dict: dict, + model_response: ModelResponse, + print_verbose: Callable, + encoding, + api_key, + logging_obj, + optional_params: dict, + timeout: Union[float, httpx.Timeout], + litellm_params: dict, + acompletion=None, + logger_fn=None, + headers={}, + client=None, + ): + """ + Completion method that uses Azure authentication instead of Anthropic's x-api-key. + All other logic is the same as AnthropicChatCompletion. + """ + from litellm.utils import ProviderConfigManager + + optional_params = copy.deepcopy(optional_params) + stream = optional_params.pop("stream", None) + json_mode: bool = optional_params.pop("json_mode", False) + is_vertex_request: bool = optional_params.pop("is_vertex_request", False) + _is_function_call = False + messages = copy.deepcopy(messages) + + # Use AzureAnthropicConfig instead of AnthropicConfig + headers = AzureAnthropicConfig().validate_environment( + api_key=api_key, + headers=headers, + model=model, + messages=messages, + optional_params={**optional_params, "is_vertex_request": is_vertex_request}, + litellm_params=litellm_params, + ) + + config = ProviderConfigManager.get_provider_chat_config( + model=model, + provider=litellm.types.utils.LlmProviders(custom_llm_provider), + ) + if config is None: + raise ValueError( + f"Provider config not found for model: {model} and provider: {custom_llm_provider}" + ) + + data = config.transform_request( + model=model, + messages=messages, + optional_params=optional_params, + litellm_params=litellm_params, + headers=headers, + ) + + ## LOGGING + logging_obj.pre_call( + input=messages, + api_key=api_key, + additional_args={ + "complete_input_dict": data, + "api_base": api_base, + "headers": headers, + }, + ) + print_verbose(f"_is_function_call: {_is_function_call}") + if acompletion is True: + if ( + stream is True + ): # if function call - fake the streaming (need complete blocks for output parsing in openai format) + print_verbose("makes async azure anthropic streaming POST request") + data["stream"] = stream + return self.acompletion_stream_function( + model=model, + messages=messages, + data=data, + api_base=api_base, + custom_prompt_dict=custom_prompt_dict, + model_response=model_response, + print_verbose=print_verbose, + encoding=encoding, + api_key=api_key, + logging_obj=logging_obj, + optional_params=optional_params, + stream=stream, + _is_function_call=_is_function_call, + json_mode=json_mode, + litellm_params=litellm_params, + logger_fn=logger_fn, + headers=headers, + timeout=timeout, + client=( + client + if client is not None and isinstance(client, AsyncHTTPHandler) + else None + ), + ) + else: + return self.acompletion_function( + model=model, + messages=messages, + data=data, + api_base=api_base, + custom_prompt_dict=custom_prompt_dict, + model_response=model_response, + print_verbose=print_verbose, + encoding=encoding, + api_key=api_key, + provider_config=config, + logging_obj=logging_obj, + optional_params=optional_params, + stream=stream, + _is_function_call=_is_function_call, + litellm_params=litellm_params, + logger_fn=logger_fn, + headers=headers, + client=client, + json_mode=json_mode, + timeout=timeout, + ) + else: + ## COMPLETION CALL + if ( + stream is True + ): # if function call - fake the streaming (need complete blocks for output parsing in openai format) + data["stream"] = stream + # Import the make_sync_call from parent + from litellm.llms.anthropic.chat.handler import make_sync_call + + completion_stream, response_headers = make_sync_call( + client=client, + api_base=api_base, + headers=headers, # type: ignore + data=json.dumps(data), + model=model, + messages=messages, + logging_obj=logging_obj, + timeout=timeout, + json_mode=json_mode, + ) + from litellm.llms.anthropic.common_utils import ( + process_anthropic_headers, + ) + + return CustomStreamWrapper( + completion_stream=completion_stream, + model=model, + custom_llm_provider="azure_anthropic", + logging_obj=logging_obj, + _response_headers=process_anthropic_headers(response_headers), + ) + + else: + if client is None or not isinstance(client, HTTPHandler): + from litellm.llms.custom_httpx.http_handler import _get_httpx_client + + client = _get_httpx_client(params={"timeout": timeout}) + else: + client = client + + try: + response = client.post( + api_base, + headers=headers, + data=json.dumps(data), + timeout=timeout, + ) + except Exception as e: + from litellm.llms.anthropic.common_utils import AnthropicError + + status_code = getattr(e, "status_code", 500) + error_headers = getattr(e, "headers", None) + error_text = getattr(e, "text", str(e)) + error_response = getattr(e, "response", None) + if error_headers is None and error_response: + error_headers = getattr(error_response, "headers", None) + if error_response and hasattr(error_response, "text"): + error_text = getattr(error_response, "text", error_text) + raise AnthropicError( + message=error_text, + status_code=status_code, + headers=error_headers, + ) + + return config.transform_response( + model=model, + raw_response=response, + model_response=model_response, + logging_obj=logging_obj, + api_key=api_key, + request_data=data, + messages=messages, + optional_params=optional_params, + litellm_params=litellm_params, + encoding=encoding, + json_mode=json_mode, + ) + diff --git a/litellm/llms/azure/anthropic/messages_transformation.py b/litellm/llms/azure/anthropic/messages_transformation.py new file mode 100644 index 00000000000..55818cc07d6 --- /dev/null +++ b/litellm/llms/azure/anthropic/messages_transformation.py @@ -0,0 +1,117 @@ +""" +Azure Anthropic messages transformation config - extends AnthropicMessagesConfig with Azure authentication +""" +from typing import TYPE_CHECKING, Any, List, Optional, Tuple + +from litellm.llms.anthropic.experimental_pass_through.messages.transformation import ( + AnthropicMessagesConfig, +) +from litellm.llms.azure.common_utils import BaseAzureLLM +from litellm.types.router import GenericLiteLLMParams + +if TYPE_CHECKING: + pass + + +class AzureAnthropicMessagesConfig(AnthropicMessagesConfig): + """ + Azure Anthropic messages configuration that extends AnthropicMessagesConfig. + The only difference is authentication - Azure uses x-api-key header (not api-key) + and Azure endpoint format. + """ + + def validate_anthropic_messages_environment( + self, + headers: dict, + model: str, + messages: List[Any], + optional_params: dict, + litellm_params: dict, + api_key: Optional[str] = None, + api_base: Optional[str] = None, + ) -> Tuple[dict, Optional[str]]: + """ + Validate environment and set up Azure authentication headers for /v1/messages endpoint. + Azure Anthropic uses x-api-key header (not api-key). + """ + # Convert dict to GenericLiteLLMParams if needed + if isinstance(litellm_params, dict): + if api_key and "api_key" not in litellm_params: + litellm_params = {**litellm_params, "api_key": api_key} + litellm_params_obj = GenericLiteLLMParams(**litellm_params) + else: + litellm_params_obj = litellm_params or GenericLiteLLMParams() + if api_key and not litellm_params_obj.api_key: + litellm_params_obj.api_key = api_key + + # Use Azure authentication logic + headers = BaseAzureLLM._base_validate_azure_environment( + headers=headers, litellm_params=litellm_params_obj + ) + + # Azure Anthropic uses x-api-key header (not api-key) + # Convert api-key to x-api-key if present + if "api-key" in headers and "x-api-key" not in headers: + headers["x-api-key"] = headers.pop("api-key") + + # Set anthropic-version header + if "anthropic-version" not in headers: + headers["anthropic-version"] = "2023-06-01" + + # Set content-type header + if "content-type" not in headers: + headers["content-type"] = "application/json" + + # Update headers with optional anthropic beta features + headers = self._update_headers_with_optional_anthropic_beta( + headers=headers, + context_management=optional_params.get("context_management"), + ) + + return headers, api_base + + def get_complete_url( + self, + api_base: Optional[str], + api_key: Optional[str], + model: str, + optional_params: dict, + litellm_params: dict, + stream: Optional[bool] = None, + ) -> str: + """ + Get the complete URL for Azure Anthropic /v1/messages endpoint. + Azure Foundry endpoint format: https://.services.ai.azure.com/anthropic/v1/messages + """ + from litellm.secret_managers.main import get_secret_str + + api_base = api_base or get_secret_str("AZURE_API_BASE") + if api_base is None: + raise ValueError( + "Missing Azure API Base - Please set `api_base` or `AZURE_API_BASE` environment variable. " + "Expected format: https://.services.ai.azure.com/anthropic" + ) + + # Ensure the URL ends with /v1/messages + api_base = api_base.rstrip("/") + if api_base.endswith("/v1/messages"): + # Already correct + pass + elif api_base.endswith("/anthropic/v1/messages"): + # Already correct + pass + else: + # Check if /anthropic is already in the path + if "/anthropic" in api_base: + # /anthropic exists, ensure we end with /anthropic/v1/messages + # Extract the base URL up to and including /anthropic + parts = api_base.split("/anthropic", 1) + api_base = parts[0] + "/anthropic" + else: + # /anthropic not in path, add it + api_base = api_base + "/anthropic" + # Add /v1/messages + api_base = api_base + "/v1/messages" + + return api_base + diff --git a/litellm/llms/azure/anthropic/transformation.py b/litellm/llms/azure/anthropic/transformation.py new file mode 100644 index 00000000000..9bc4f130563 --- /dev/null +++ b/litellm/llms/azure/anthropic/transformation.py @@ -0,0 +1,96 @@ +""" +Azure Anthropic transformation config - extends AnthropicConfig with Azure authentication +""" +from typing import TYPE_CHECKING, Dict, List, Optional, Union + +from litellm.llms.anthropic.chat.transformation import AnthropicConfig +from litellm.llms.azure.common_utils import BaseAzureLLM +from litellm.types.llms.openai import AllMessageValues +from litellm.types.router import GenericLiteLLMParams + +if TYPE_CHECKING: + pass + + +class AzureAnthropicConfig(AnthropicConfig): + """ + Azure Anthropic configuration that extends AnthropicConfig. + The only difference is authentication - Azure uses api-key header or Azure AD token + instead of x-api-key header. + """ + + @property + def custom_llm_provider(self) -> Optional[str]: + return "azure_anthropic" + + def validate_environment( + self, + headers: dict, + model: str, + messages: List[AllMessageValues], + optional_params: dict, + litellm_params: Union[dict, GenericLiteLLMParams], + api_key: Optional[str] = None, + api_base: Optional[str] = None, + ) -> Dict: + """ + Validate environment and set up Azure authentication headers. + Azure supports: + 1. API key via 'api-key' header + 2. Azure AD token via 'Authorization: Bearer ' header + """ + # Convert dict to GenericLiteLLMParams if needed + if isinstance(litellm_params, dict): + # Ensure api_key is included if provided + if api_key and "api_key" not in litellm_params: + litellm_params = {**litellm_params, "api_key": api_key} + litellm_params_obj = GenericLiteLLMParams(**litellm_params) + else: + litellm_params_obj = litellm_params or GenericLiteLLMParams() + # Set api_key if provided and not already set + if api_key and not litellm_params_obj.api_key: + litellm_params_obj.api_key = api_key + + # Use Azure authentication logic + headers = BaseAzureLLM._base_validate_azure_environment( + headers=headers, litellm_params=litellm_params_obj + ) + + # Azure Anthropic uses x-api-key header (not api-key) + # Convert api-key to x-api-key if present + if "api-key" in headers and "x-api-key" not in headers: + headers["x-api-key"] = headers.pop("api-key") + + # Get tools and other anthropic-specific setup + tools = optional_params.get("tools") + prompt_caching_set = self.is_cache_control_set(messages=messages) + computer_tool_used = self.is_computer_tool_used(tools=tools) + mcp_server_used = self.is_mcp_server_used( + mcp_servers=optional_params.get("mcp_servers") + ) + pdf_used = self.is_pdf_used(messages=messages) + file_id_used = self.is_file_id_used(messages=messages) + user_anthropic_beta_headers = self._get_user_anthropic_beta_headers( + anthropic_beta_header=headers.get("anthropic-beta") + ) + + # Get anthropic headers (but we'll replace x-api-key with Azure auth) + anthropic_headers = self.get_anthropic_headers( + computer_tool_used=computer_tool_used, + prompt_caching_set=prompt_caching_set, + pdf_used=pdf_used, + api_key=api_key or "", # Azure auth is already in headers + file_id_used=file_id_used, + is_vertex_request=optional_params.get("is_vertex_request", False), + user_anthropic_beta_headers=user_anthropic_beta_headers, + mcp_server_used=mcp_server_used, + ) + # Merge headers - Azure auth (api-key or Authorization) takes precedence + headers = {**anthropic_headers, **headers} + + # Ensure anthropic-version header is set + if "anthropic-version" not in headers: + headers["anthropic-version"] = "2023-06-01" + + return headers + diff --git a/litellm/llms/azure/chat/gpt_5_transformation.py b/litellm/llms/azure/chat/gpt_5_transformation.py index d563a2889ca..209475730f8 100644 --- a/litellm/llms/azure/chat/gpt_5_transformation.py +++ b/litellm/llms/azure/chat/gpt_5_transformation.py @@ -2,6 +2,8 @@ from typing import List +import litellm +from litellm.exceptions import UnsupportedParamsError from litellm.llms.openai.chat.gpt_5_transformation import OpenAIGPT5Config from litellm.types.llms.openai import AllMessageValues @@ -33,7 +35,34 @@ class AzureOpenAIGPT5Config(AzureOpenAIConfig, OpenAIGPT5Config): drop_params: bool, api_version: str = "", ) -> dict: - return OpenAIGPT5Config.map_openai_params( + reasoning_effort_value = ( + non_default_params.get("reasoning_effort") + or optional_params.get("reasoning_effort") + ) + + if reasoning_effort_value == "none": + if litellm.drop_params is True or ( + drop_params is not None and drop_params is True + ): + non_default_params = non_default_params.copy() + optional_params = optional_params.copy() + if non_default_params.get("reasoning_effort") == "none": + non_default_params.pop("reasoning_effort") + if optional_params.get("reasoning_effort") == "none": + optional_params.pop("reasoning_effort") + else: + raise UnsupportedParamsError( + status_code=400, + message=( + "Azure OpenAI does not support reasoning_effort='none'. " + "Supported values are: 'low', 'medium', and 'high'. " + "To drop this parameter, set `litellm.drop_params=True` or for proxy:\n\n" + "`litellm_settings:\n drop_params: true`\n" + "Issue: https://github.com/BerriAI/litellm/issues/16704" + ), + ) + + result = OpenAIGPT5Config.map_openai_params( self, non_default_params=non_default_params, optional_params=optional_params, @@ -41,6 +70,11 @@ class AzureOpenAIGPT5Config(AzureOpenAIConfig, OpenAIGPT5Config): drop_params=drop_params, ) + if result.get("reasoning_effort") == "none": + result.pop("reasoning_effort") + + return result + def transform_request( self, model: str, diff --git a/litellm/llms/azure/videos/transformation.py b/litellm/llms/azure/videos/transformation.py index 3af9e0778bc..a6fbd8cef8b 100644 --- a/litellm/llms/azure/videos/transformation.py +++ b/litellm/llms/azure/videos/transformation.py @@ -1,9 +1,8 @@ from typing import TYPE_CHECKING, Any, Dict, Optional from litellm.types.videos.main import VideoCreateOptionalRequestParams -from litellm.secret_managers.main import get_secret_str +from litellm.types.router import GenericLiteLLMParams from litellm.llms.azure.common_utils import BaseAzureLLM -import litellm from litellm.llms.openai.videos.transformation import OpenAIVideoConfig if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj @@ -56,22 +55,27 @@ class AzureVideoConfig(OpenAIVideoConfig): headers: dict, model: str, api_key: Optional[str] = None, + litellm_params: Optional[GenericLiteLLMParams] = None, ) -> dict: - api_key = ( - api_key - or litellm.api_key - or litellm.azure_key - or get_secret_str("AZURE_OPENAI_API_KEY") - or get_secret_str("AZURE_API_KEY") + """ + Validate Azure environment and set up authentication headers. + Uses _base_validate_azure_environment to properly handle credentials from litellm_credential_name. + """ + # If litellm_params is provided, use it; otherwise create a new one + if litellm_params is None: + litellm_params = GenericLiteLLMParams() + + if api_key and not litellm_params.api_key: + litellm_params.api_key = api_key + + # Use the base Azure validation method which properly handles: + # 1. Credentials from litellm_credential_name via litellm_params + # 2. Sets the correct "api-key" header (not "Authorization: Bearer") + return BaseAzureLLM._base_validate_azure_environment( + headers=headers, + litellm_params=litellm_params ) - headers.update( - { - "Authorization": f"Bearer {api_key}", - } - ) - return headers - def get_complete_url( self, model: str, diff --git a/litellm/llms/base_llm/videos/transformation.py b/litellm/llms/base_llm/videos/transformation.py index 7e990b42650..50cada42b87 100644 --- a/litellm/llms/base_llm/videos/transformation.py +++ b/litellm/llms/base_llm/videos/transformation.py @@ -66,6 +66,7 @@ class BaseVideoConfig(ABC): headers: dict, model: str, api_key: Optional[str] = None, + litellm_params: Optional[GenericLiteLLMParams] = None, ) -> dict: return {} diff --git a/litellm/llms/bedrock/chat/converse_transformation.py b/litellm/llms/bedrock/chat/converse_transformation.py index d76a3c31b51..9e001f533d1 100644 --- a/litellm/llms/bedrock/chat/converse_transformation.py +++ b/litellm/llms/bedrock/chat/converse_transformation.py @@ -580,6 +580,20 @@ class AmazonConverseConfig(BaseConfig): non_default_params=non_default_params, optional_params=optional_params ) + final_is_thinking_enabled = self.is_thinking_enabled(optional_params) + if ( + final_is_thinking_enabled + and "tool_choice" in optional_params + ): + tool_choice_block = optional_params["tool_choice"] + if isinstance(tool_choice_block, dict): + if "any" in tool_choice_block or "tool" in tool_choice_block: + verbose_logger.info( + f"{model} does not support forced tool use (tool_choice='required' or specific tool) " + f"when reasoning is enabled. Changing tool_choice to 'auto'." + ) + optional_params["tool_choice"] = ToolChoiceValuesBlock(auto={}) + return optional_params def _translate_response_format_param( diff --git a/litellm/llms/bedrock/chat/invoke_transformations/amazon_openai_transformation.py b/litellm/llms/bedrock/chat/invoke_transformations/amazon_openai_transformation.py new file mode 100644 index 00000000000..ee07b71ef15 --- /dev/null +++ b/litellm/llms/bedrock/chat/invoke_transformations/amazon_openai_transformation.py @@ -0,0 +1,186 @@ +""" +Transformation for Bedrock imported models that use OpenAI Chat Completions format. + +Use this for models imported into Bedrock that accept the OpenAI API format. +Model format: bedrock/openai/ + +Example: bedrock/openai/arn:aws:bedrock:us-east-1:123456789012:imported-model/abc123 +""" + +from typing import TYPE_CHECKING, Any, List, Optional, Tuple, Union + +import httpx + +from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM +from litellm.llms.bedrock.common_utils import BedrockError +from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig +from litellm.types.llms.openai import AllMessageValues + +if TYPE_CHECKING: + from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + + LiteLLMLoggingObj = _LiteLLMLoggingObj +else: + LiteLLMLoggingObj = Any + + +class AmazonBedrockOpenAIConfig(OpenAIGPTConfig, BaseAWSLLM): + """ + Configuration for Bedrock imported models that use OpenAI Chat Completions format. + + This class handles the transformation of requests and responses for Bedrock + imported models that accept the OpenAI API format directly. + + Inherits from OpenAIGPTConfig to leverage standard OpenAI parameter handling + and response transformation, while adding Bedrock-specific URL generation + and AWS request signing. + + Usage: + model = "bedrock/openai/arn:aws:bedrock:us-east-1:123456789012:imported-model/abc123" + """ + + def __init__(self, **kwargs): + OpenAIGPTConfig.__init__(self, **kwargs) + BaseAWSLLM.__init__(self, **kwargs) + + @property + def custom_llm_provider(self) -> Optional[str]: + return "bedrock" + + def _get_openai_model_id(self, model: str) -> str: + """ + Extract the actual model ID from the LiteLLM model name. + + Input format: bedrock/openai/ + Returns: + """ + # Remove bedrock/ prefix if present + if model.startswith("bedrock/"): + model = model[8:] + + # Remove openai/ prefix + if model.startswith("openai/"): + model = model[7:] + + return model + + def get_complete_url( + self, + api_base: Optional[str], + api_key: Optional[str], + model: str, + optional_params: dict, + litellm_params: dict, + stream: Optional[bool] = None, + ) -> str: + """ + Get the complete URL for the Bedrock invoke endpoint. + + Uses the standard Bedrock invoke endpoint format. + """ + model_id = self._get_openai_model_id(model) + + # Get AWS region + aws_region_name = self._get_aws_region_name( + optional_params=optional_params, model=model + ) + + # Get runtime endpoint + aws_bedrock_runtime_endpoint = optional_params.get( + "aws_bedrock_runtime_endpoint", None + ) + endpoint_url, proxy_endpoint_url = self.get_runtime_endpoint( + api_base=api_base, + aws_bedrock_runtime_endpoint=aws_bedrock_runtime_endpoint, + aws_region_name=aws_region_name, + ) + + # Build the invoke URL + if stream: + endpoint_url = f"{endpoint_url}/model/{model_id}/invoke-with-response-stream" + else: + endpoint_url = f"{endpoint_url}/model/{model_id}/invoke" + + return endpoint_url + + def sign_request( + self, + headers: dict, + optional_params: dict, + request_data: dict, + api_base: str, + api_key: Optional[str] = None, + model: Optional[str] = None, + stream: Optional[bool] = None, + fake_stream: Optional[bool] = None, + ) -> Tuple[dict, Optional[bytes]]: + """ + Sign the request using AWS Signature Version 4. + """ + return self._sign_request( + service_name="bedrock", + headers=headers, + optional_params=optional_params, + request_data=request_data, + api_base=api_base, + api_key=api_key, + model=model, + stream=stream, + fake_stream=fake_stream, + ) + + def transform_request( + self, + model: str, + messages: List[AllMessageValues], + optional_params: dict, + litellm_params: dict, + headers: dict, + ) -> dict: + """ + Transform the request to OpenAI Chat Completions format for Bedrock imported models. + + Removes AWS-specific params and stream param (handled separately in URL), + then delegates to parent class for standard OpenAI request transformation. + """ + # Remove stream from optional_params as it's handled separately in URL + optional_params.pop("stream", None) + + # Remove AWS-specific params that shouldn't be in the request body + inference_params = { + k: v + for k, v in optional_params.items() + if k not in self.aws_authentication_params + } + + # Use parent class transform_request for OpenAI format + return super().transform_request( + model=self._get_openai_model_id(model), + messages=messages, + optional_params=inference_params, + litellm_params=litellm_params, + headers=headers, + ) + + def validate_environment( + self, + headers: dict, + model: str, + messages: List[AllMessageValues], + optional_params: dict, + litellm_params: dict, + api_key: Optional[str] = None, + api_base: Optional[str] = None, + ) -> dict: + """ + Validate the environment and return headers. + + For Bedrock, we don't need Bearer token auth since we use AWS SigV4. + """ + return headers + + def get_error_class( + self, error_message: str, status_code: int, headers: Union[dict, httpx.Headers] + ) -> BedrockError: + """Return the appropriate error class for Bedrock.""" + return BedrockError(status_code=status_code, message=error_message) diff --git a/litellm/llms/bedrock/common_utils.py b/litellm/llms/bedrock/common_utils.py index baaec996535..35d3d736a1c 100644 --- a/litellm/llms/bedrock/common_utils.py +++ b/litellm/llms/bedrock/common_utils.py @@ -403,6 +403,9 @@ class BedrockModelInfo(BaseLLMModelInfo): if model.startswith("invoke/"): model = model.split("/", 1)[1] + if model.startswith("openai/"): + model = model.split("/", 1)[1] + return model @staticmethod @@ -446,12 +449,12 @@ class BedrockModelInfo(BaseLLMModelInfo): @staticmethod def get_bedrock_route( model: str, - ) -> Literal["converse", "invoke", "converse_like", "agent", "agentcore", "async_invoke"]: + ) -> Literal["converse", "invoke", "converse_like", "agent", "agentcore", "async_invoke", "openai"]: """ Get the bedrock route for the given model. """ route_mappings: Dict[ - str, Literal["invoke", "converse_like", "converse", "agent", "agentcore", "async_invoke"] + str, Literal["invoke", "converse_like", "converse", "agent", "agentcore", "async_invoke", "openai"] ] = { "invoke/": "invoke", "converse_like/": "converse_like", @@ -459,6 +462,7 @@ class BedrockModelInfo(BaseLLMModelInfo): "agent/": "agent", "agentcore/": "agentcore", "async_invoke/": "async_invoke", + "openai/": "openai", } # Check explicit routes first @@ -517,6 +521,14 @@ class BedrockModelInfo(BaseLLMModelInfo): """ return "async_invoke/" in model + @staticmethod + def _explicit_openai_route(model: str) -> bool: + """ + Check if the model is an explicit openai route. + Used for Bedrock imported models that use OpenAI Chat Completions format. + """ + return "openai/" in model + @staticmethod def get_bedrock_provider_config_for_messages_api( model: str, @@ -566,6 +578,8 @@ def get_bedrock_chat_config(model: str): # Handle explicit routes first if bedrock_route == "converse" or bedrock_route == "converse_like": return litellm.AmazonConverseConfig() + elif bedrock_route == "openai": + return litellm.AmazonBedrockOpenAIConfig() elif bedrock_route == "agent": from litellm.llms.bedrock.chat.invoke_agent.transformation import ( AmazonInvokeAgentConfig, diff --git a/litellm/llms/bedrock/image/amazon_nova_canvas_transformation.py b/litellm/llms/bedrock/image/amazon_nova_canvas_transformation.py index cd33e62af16..f2b94b617c0 100644 --- a/litellm/llms/bedrock/image/amazon_nova_canvas_transformation.py +++ b/litellm/llms/bedrock/image/amazon_nova_canvas_transformation.py @@ -3,6 +3,7 @@ from typing import Any, Dict, List, Optional from openai.types.image import Image +from litellm import get_model_info from litellm.types.llms.bedrock import ( AmazonNovaCanvasColorGuidedGenerationParams, AmazonNovaCanvasColorGuidedRequest, @@ -197,3 +198,22 @@ class AmazonNovaCanvasConfig: model_response.data = openai_images return model_response + + @classmethod + def cost_calculator( + cls, + model: str, + image_response: ImageResponse, + size: Optional[str] = None, + optional_params: Optional[dict] = None, + ) -> float: + model_info = get_model_info( + model=model, + custom_llm_provider="bedrock", + ) + + output_cost_per_image: float = model_info.get("output_cost_per_image") or 0.0 + num_images: int = 0 + if image_response.data: + num_images = len(image_response.data) + return output_cost_per_image * num_images \ No newline at end of file diff --git a/litellm/llms/bedrock/image/amazon_stability1_transformation.py b/litellm/llms/bedrock/image/amazon_stability1_transformation.py index 698ecca94ba..63af32f3f56 100644 --- a/litellm/llms/bedrock/image/amazon_stability1_transformation.py +++ b/litellm/llms/bedrock/image/amazon_stability1_transformation.py @@ -1,8 +1,11 @@ +import copy +import os import types from typing import List, Optional from openai.types.image import Image +from litellm import get_model_info from litellm.types.utils import ImageResponse @@ -90,6 +93,31 @@ class AmazonStabilityConfig: return optional_params + @classmethod + def transform_request_body( + cls, + text: str, + optional_params: dict, + ) -> dict: + inference_params = copy.deepcopy(optional_params) + inference_params.pop( + "user", None + ) # make sure user is not passed in for bedrock call + + prompt = text.replace(os.linesep, " ") + ## LOAD CONFIG + config = cls.get_config() + for k, v in config.items(): + if ( + k not in inference_params + ): # completion(top_k=3) > anthropic_config(top_k=3) <- allows for dynamic variables to be passed in + inference_params[k] = v + + return { + "text_prompts": [{"text": prompt, "weight": 1}], + **inference_params, + } + @classmethod def transform_response_dict_to_openai_response( cls, model_response: ImageResponse, response_dict: dict @@ -102,3 +130,34 @@ class AmazonStabilityConfig: model_response.data = image_list return model_response + + @classmethod + def cost_calculator( + cls, + model: str, + image_response: ImageResponse, + size: Optional[str] = None, + optional_params: Optional[dict] = None, + ) -> float: + optional_params = optional_params or {} + + # see model_prices_and_context_window.json for details on how steps is used + # Reference pricing by steps for stability 1: https://aws.amazon.com/bedrock/pricing/ + _steps = optional_params.get("steps", 50) + steps = "max-steps" if _steps > 50 else "50-steps" + + # size is stored in model_prices_and_context_window.json as 1024-x-1024 + # current size has 1024x1024 + size = size or "1024-x-1024" + model = f"{size}/{steps}/{model}" + + model_info = get_model_info( + model=model, + custom_llm_provider="bedrock", + ) + + output_cost_per_image: float = model_info.get("output_cost_per_image") or 0.0 + num_images: int = 0 + if image_response.data: + num_images = len(image_response.data) + return output_cost_per_image * num_images \ No newline at end of file diff --git a/litellm/llms/bedrock/image/amazon_stability3_transformation.py b/litellm/llms/bedrock/image/amazon_stability3_transformation.py index 06e06209791..445a2fe1100 100644 --- a/litellm/llms/bedrock/image/amazon_stability3_transformation.py +++ b/litellm/llms/bedrock/image/amazon_stability3_transformation.py @@ -3,6 +3,8 @@ from typing import List, Optional from openai.types.image import Image +from litellm import get_model_info +from litellm.llms.bedrock.common_utils import BedrockError from litellm.types.llms.bedrock import ( AmazonStability3TextToImageRequest, AmazonStability3TextToImageResponse, @@ -66,12 +68,12 @@ class AmazonStability3Config: @classmethod def transform_request_body( - cls, prompt: str, optional_params: dict + cls, text: str, optional_params: dict ) -> AmazonStability3TextToImageRequest: """ Transform the request body for the Stability 3 models """ - data = AmazonStability3TextToImageRequest(prompt=prompt, **optional_params) + data = AmazonStability3TextToImageRequest(prompt=text, **optional_params) return data @classmethod @@ -92,9 +94,34 @@ class AmazonStability3Config: """ stability_3_response = AmazonStability3TextToImageResponse(**response_dict) + + finish_reasons = stability_3_response.get("finish_reasons", []) + finish_reasons = [reason for reason in finish_reasons if reason] + if len(finish_reasons) > 0: + raise BedrockError(status_code=400, message="; ".join(finish_reasons)) + openai_images: List[Image] = [] for _img in stability_3_response.get("images", []): openai_images.append(Image(b64_json=_img)) model_response.data = openai_images return model_response + + @classmethod + def cost_calculator( + cls, + model: str, + image_response: ImageResponse, + size: Optional[str] = None, + optional_params: Optional[dict] = None, + ) -> float: + model_info = get_model_info( + model=model, + custom_llm_provider="bedrock", + ) + + output_cost_per_image: float = model_info.get("output_cost_per_image") or 0.0 + num_images: int = 0 + if image_response.data: + num_images = len(image_response.data) + return output_cost_per_image * num_images diff --git a/litellm/llms/bedrock/image/amazon_titan_transformation.py b/litellm/llms/bedrock/image/amazon_titan_transformation.py index 2709f406dfd..bed9ad0c300 100644 --- a/litellm/llms/bedrock/image/amazon_titan_transformation.py +++ b/litellm/llms/bedrock/image/amazon_titan_transformation.py @@ -103,16 +103,16 @@ class AmazonTitanImageGenerationConfig: return optional_params @classmethod - def _transform_request( + def transform_request_body( cls, - input: str, + text: str, optional_params: dict, ) -> AmazonTitanImageGenerationRequestBody: from typing import Any, Dict image_generation_config = optional_params.pop("imageGenerationConfig", {}) negative_text = optional_params.pop("negativeText", None) - text_to_image_params: Dict[str, Any] = {"text": input} + text_to_image_params: Dict[str, Any] = {"text": text} if negative_text: text_to_image_params["negativeText"] = negative_text task_type = optional_params.pop("taskType", "TEXT_IMAGE") diff --git a/litellm/llms/bedrock/image/cost_calculator.py b/litellm/llms/bedrock/image/cost_calculator.py index 9b2ae8782cb..bc1a57b8aec 100644 --- a/litellm/llms/bedrock/image/cost_calculator.py +++ b/litellm/llms/bedrock/image/cost_calculator.py @@ -1,9 +1,6 @@ from typing import Optional -import litellm -from litellm.llms.bedrock.image.amazon_titan_transformation import ( - AmazonTitanImageGenerationConfig, -) +from litellm.llms.bedrock.image.image_handler import BedrockImageGeneration from litellm.types.utils import ImageResponse @@ -18,36 +15,10 @@ def cost_calculator( Handles both Stability 1 and Stability 3 models """ - if litellm.AmazonStability3Config()._is_stability_3_model(model=model): - pass - elif AmazonTitanImageGenerationConfig._is_titan_model(model=model): - return AmazonTitanImageGenerationConfig.cost_calculator( - model=model, - image_response=image_response, - size=size, - optional_params=optional_params, - ) - else: - # Stability 1 models - optional_params = optional_params or {} - - # see model_prices_and_context_window.json for details on how steps is used - # Reference pricing by steps for stability 1: https://aws.amazon.com/bedrock/pricing/ - _steps = optional_params.get("steps", 50) - steps = "max-steps" if _steps > 50 else "50-steps" - - # size is stored in model_prices_and_context_window.json as 1024-x-1024 - # current size has 1024x1024 - size = size or "1024-x-1024" - model = f"{size}/{steps}/{model}" - - _model_info = litellm.get_model_info( + config_class = BedrockImageGeneration.get_config_class(model=model) + return config_class.cost_calculator( model=model, - custom_llm_provider="bedrock", + image_response=image_response, + size=size, + optional_params=optional_params, ) - - output_cost_per_image: float = _model_info.get("output_cost_per_image") or 0.0 - num_images: int = 0 - if image_response.data: - num_images = len(image_response.data) - return output_cost_per_image * num_images diff --git a/litellm/llms/bedrock/image/image_handler.py b/litellm/llms/bedrock/image/image_handler.py index 313a1dc17bd..89e37bbdd8d 100644 --- a/litellm/llms/bedrock/image/image_handler.py +++ b/litellm/llms/bedrock/image/image_handler.py @@ -1,13 +1,12 @@ -import copy +from __future__ import annotations + import json -import os from typing import TYPE_CHECKING, Any, Optional, Union import httpx from pydantic import BaseModel import litellm -from litellm import BEDROCK_INVOKE_PROVIDERS_LITERAL from litellm._logging import verbose_logger from litellm.litellm_core_utils.litellm_logging import Logging as LitellmLogging from litellm.llms.bedrock.image.amazon_nova_canvas_transformation import ( @@ -47,11 +46,30 @@ class BedrockImagePreparedRequest(BaseModel): data: dict +BedrockImageConfigClass = Union[ + type[AmazonTitanImageGenerationConfig], + type[AmazonNovaCanvasConfig], + type[AmazonStability3Config], + type[litellm.AmazonStabilityConfig], +] + + class BedrockImageGeneration(BaseAWSLLM): """ Bedrock Image Generation handler """ + @classmethod + def get_config_class(cls, model: str | None) -> BedrockImageConfigClass: + if AmazonTitanImageGenerationConfig._is_titan_model(model): + return AmazonTitanImageGenerationConfig + elif AmazonNovaCanvasConfig._is_nova_model(model): + return AmazonNovaCanvasConfig + elif AmazonStability3Config._is_stability_3_model(model): + return AmazonStability3Config + else: + return litellm.AmazonStabilityConfig + def image_generation( self, model: str, @@ -202,7 +220,6 @@ class BedrockImageGeneration(BaseAWSLLM): model=model, prompt=prompt, optional_params=optional_params, - bedrock_provider=bedrock_provider, ) # Make POST Request @@ -241,7 +258,6 @@ class BedrockImageGeneration(BaseAWSLLM): def _get_request_body( self, model: str, - bedrock_provider: Optional[BEDROCK_INVOKE_PROVIDERS_LITERAL], prompt: str, optional_params: dict, ) -> dict: @@ -253,49 +269,9 @@ class BedrockImageGeneration(BaseAWSLLM): Returns: dict: The request body to use for the Bedrock Image Generation API """ - if bedrock_provider == "amazon" or bedrock_provider == "nova": - # Handle Amazon Nova Canvas models - provider = "amazon" - elif bedrock_provider == "stability": - provider = "stability" - else: - # Fallback to original logic for backward compatibility - provider = model.split(".")[0] - inference_params = copy.deepcopy(optional_params) - inference_params.pop( - "user", None - ) # make sure user is not passed in for bedrock call - data = {} - if provider == "stability": - if litellm.AmazonStability3Config._is_stability_3_model(model): - request_body = litellm.AmazonStability3Config.transform_request_body( - prompt=prompt, optional_params=optional_params - ) - return dict(request_body) - else: - prompt = prompt.replace(os.linesep, " ") - ## LOAD CONFIG - config = litellm.AmazonStabilityConfig.get_config() - for k, v in config.items(): - if ( - k not in inference_params - ): # completion(top_k=3) > anthropic_config(top_k=3) <- allows for dynamic variables to be passed in - inference_params[k] = v - data = { - "text_prompts": [{"text": prompt, "weight": 1}], - **inference_params, - } - elif provider == "amazon": - return dict( - litellm.AmazonNovaCanvasConfig.transform_request_body( - text=prompt, optional_params=optional_params - ) - ) - else: - raise BedrockError( - status_code=422, message=f"Unsupported model={model}, passed in" - ) - return data + config_class = self.get_config_class(model=model) + request_body = config_class.transform_request_body(text=prompt, optional_params=optional_params) + return dict(request_body) def _transform_response_dict_to_openai_response( self, @@ -323,20 +299,7 @@ class BedrockImageGeneration(BaseAWSLLM): if response_dict is None: raise ValueError("Error in response object format, got None") - config_class: Union[ - type[AmazonTitanImageGenerationConfig], - type[AmazonNovaCanvasConfig], - type[AmazonStability3Config], - type[litellm.AmazonStabilityConfig], - ] - if AmazonTitanImageGenerationConfig._is_titan_model(model=model): - config_class = AmazonTitanImageGenerationConfig - elif AmazonNovaCanvasConfig._is_nova_model(model=model): - config_class = AmazonNovaCanvasConfig - elif AmazonStability3Config._is_stability_3_model(model=model): - config_class = AmazonStability3Config - else: - config_class = litellm.AmazonStabilityConfig + config_class = self.get_config_class(model=model) config_class.transform_response_dict_to_openai_response( model_response=model_response, diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index 383dabb3931..fdd504e2f57 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -4126,6 +4126,7 @@ class BaseLLMHTTPHandler: headers=video_generation_optional_request_params.get("extra_headers", {}) or {}, model=model, + litellm_params=litellm_params, ) if extra_headers: @@ -4226,6 +4227,7 @@ class BaseLLMHTTPHandler: headers=video_generation_optional_request_params.get("extra_headers", {}) or {}, model=model, + litellm_params=litellm_params, ) if extra_headers: diff --git a/litellm/llms/elevenlabs/text_to_speech/transformation.py b/litellm/llms/elevenlabs/text_to_speech/transformation.py new file mode 100644 index 00000000000..b78d0bafc50 --- /dev/null +++ b/litellm/llms/elevenlabs/text_to_speech/transformation.py @@ -0,0 +1,332 @@ +""" +Elevenlabs Text-to-Speech transformation + +Maps OpenAI TTS spec to Elevenlabs TTS API +""" + +from typing import TYPE_CHECKING, Any, Dict, Optional, Tuple, Union +from urllib.parse import urlencode + +import httpx +from httpx import Headers + +import litellm +from litellm.types.utils import all_litellm_params +from litellm.llms.base_llm.chat.transformation import BaseLLMException +from litellm.llms.base_llm.text_to_speech.transformation import ( + BaseTextToSpeechConfig, + TextToSpeechRequestData, +) +from litellm.secret_managers.main import get_secret_str + +from ..common_utils import ElevenLabsException + + +if TYPE_CHECKING: + from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj + from litellm.types.llms.openai import HttpxBinaryResponseContent +else: + LiteLLMLoggingObj = Any + HttpxBinaryResponseContent = Any + + +class ElevenLabsTextToSpeechConfig(BaseTextToSpeechConfig): + """ + Configuration for ElevenLabs Text-to-Speech + + Reference: https://elevenlabs.io/docs/api-reference/text-to-speech/convert + """ + + TTS_BASE_URL = "https://api.elevenlabs.io" + TTS_ENDPOINT_PATH = "/v1/text-to-speech" + DEFAULT_OUTPUT_FORMAT = "pcm_44100" + VOICE_MAPPINGS = { + "alloy": "21m00Tcm4TlvDq8ikWAM", # Rachel + "amber": "5Q0t7uMcjvnagumLfvZi", # Paul + "ash": "AZnzlk1XvdvUeBnXmlld", # Domi + "august": "D38z5RcWu1voky8WS1ja", # Fin + "blue": "2EiwWnXFnvU5JabPnv8n", # Clyde + "coral": "9BWtsMINqrJLrRacOk9x", # Aria + "lily": "EXAVITQu4vr4xnSDxMaL", # Sarah + "onyx": "29vD33N1CtxCmqQRPOHJ", # Drew + "sage": "CwhRBWXzGAHq8TQ4Fs17", # Roger + "verse": "CYw3kZ02Hs0563khs1Fj", # Dave + } + + # Response format mappings from OpenAI to ElevenLabs + FORMAT_MAPPINGS = { + "mp3": "mp3_44100_128", + "pcm": "pcm_44100", + "opus": "opus_48000_128", + # ElevenLabs does not support WAV, AAC, or FLAC formats. + } + + ELEVENLABS_QUERY_PARAMS_KEY = "__elevenlabs_query_params__" + ELEVENLABS_VOICE_ID_KEY = "__elevenlabs_voice_id__" + + def get_supported_openai_params(self, model: str) -> list: + """ + ElevenLabs TTS supports these OpenAI parameters + """ + return ["voice", "response_format", "speed"] + + def _extract_voice_id(self, voice: str) -> str: + """ + Normalize the provided voice information into an ElevenLabs voice_id. + """ + normalized_voice = voice.strip() + mapped_voice = self.VOICE_MAPPINGS.get(normalized_voice.lower()) + return mapped_voice or normalized_voice + + def _resolve_voice_id( + self, + voice: Optional[Union[str, Dict[str, Any]]], + params: Dict[str, Any], + ) -> str: + """ + Determine the ElevenLabs voice_id based on provided voice input or parameters. + """ + mapped_voice: Optional[str] = None + + if isinstance(voice, str) and voice.strip(): + mapped_voice = self._extract_voice_id(voice) + elif isinstance(voice, dict): + for key in ("voice_id", "id", "name"): + candidate = voice.get(key) + if isinstance(candidate, str) and candidate.strip(): + mapped_voice = self._extract_voice_id(candidate) + break + elif voice is not None: + mapped_voice = self._extract_voice_id(str(voice)) + + if mapped_voice is None: + voice_override = params.pop("voice_id", None) + if isinstance(voice_override, str) and voice_override.strip(): + mapped_voice = self._extract_voice_id(voice_override) + + if mapped_voice is None: + raise ValueError( + "ElevenLabs voice_id is required. Pass `voice` when calling `litellm.speech()`." + ) + + return mapped_voice + + def map_openai_params( + self, + model: str, + optional_params: Dict, + voice: Optional[Union[str, Dict]] = None, + drop_params: bool = False, + kwargs: Optional[Dict[str, Any]] = None, + ) -> Tuple[Optional[str], Dict]: + """ + Map OpenAI parameters to ElevenLabs TTS parameters + """ + mapped_params: Dict[str, Any] = {} + query_params: Dict[str, Any] = {} + + # Work on a copy so we don't mutate the caller's dictionary + params = dict(optional_params) if optional_params else {} + passthrough_kwargs: Dict[str, Any] = kwargs if kwargs is not None else {} + + # Extract voice identifier + mapped_voice = self._resolve_voice_id(voice, params) + + # Response/output format → query parameter + response_format = params.pop("response_format", None) + if isinstance(response_format, str): + mapped_format = self.FORMAT_MAPPINGS.get(response_format, response_format) + query_params["output_format"] = mapped_format + + # ElevenLabs does not support OpenAI speed directly. + # Drop it to avoid sending unsupported keys unless caller already provided voice_settings. + speed = params.pop("speed", None) + if speed is not None: + speed_value: Optional[float] + try: + speed_value = float(speed) + except (TypeError, ValueError): + speed_value = None + if speed_value is not None: + if isinstance(params.get("voice_settings"), dict): + params["voice_settings"]["speed"] = speed_value # type: ignore[index] + else: + params["voice_settings"] = {"speed": speed_value} + + # Instructions parameter is OpenAI-specific; omit to prevent API errors. + params.pop("instructions", None) + self._add_elevenlabs_specific_params( + mapped_voice=mapped_voice, + query_params=query_params, + mapped_params=mapped_params, + kwargs=passthrough_kwargs, + remaining_params=params, + ) + + return mapped_voice, mapped_params + + def validate_environment( + self, + headers: dict, + model: str, + api_key: Optional[str] = None, + api_base: Optional[str] = None, + ) -> dict: + """ + Validate Azure environment and set up authentication headers + """ + api_key = ( + api_key + or litellm.api_key + or litellm.openai_key + or get_secret_str("ELEVENLABS_API_KEY") + ) + + if api_key is None: + raise ValueError( + "ElevenLabs API key is required. Set ELEVENLABS_API_KEY environment variable." + ) + + headers.update( + { + "xi-api-key": api_key, + "Content-Type": "application/json", + } + ) + + return headers + + def get_error_class( + self, error_message: str, status_code: int, headers: Union[dict, Headers] + ) -> BaseLLMException: + return ElevenLabsException( + message=error_message, status_code=status_code, headers=headers + ) + + def transform_text_to_speech_request( + self, + model: str, + input: str, + voice: Optional[str], + optional_params: Dict, + litellm_params: Dict, + headers: dict, + ) -> TextToSpeechRequestData: + """ + Build the ElevenLabs TTS request payload. + """ + params = dict(optional_params) if optional_params else {} + extra_body = params.pop("extra_body", None) + + request_body: Dict[str, Any] = { + "text": input, + "model_id": model, + } + + for key, value in params.items(): + if value is None: + continue + request_body[key] = value + + if isinstance(extra_body, dict): + for key, value in extra_body.items(): + if value is None: + continue + request_body[key] = value + + return TextToSpeechRequestData( + dict_body=request_body, + headers={"Content-Type": "application/json"}, + ) + + def _add_elevenlabs_specific_params( + self, + mapped_voice: str, + query_params: Dict[str, Any], + mapped_params: Dict[str, Any], + kwargs: Optional[Dict[str, Any]], + remaining_params: Dict[str, Any], + ) -> None: + if kwargs is None: + kwargs = {} + for key, value in remaining_params.items(): + if value is None: + continue + mapped_params[key] = value + + reserved_kwarg_keys = set(all_litellm_params) | { + self.ELEVENLABS_QUERY_PARAMS_KEY, + self.ELEVENLABS_VOICE_ID_KEY, + "voice", + "model", + "response_format", + "output_format", + "extra_body", + "user", + } + + extra_body_from_kwargs = kwargs.pop("extra_body", None) + if isinstance(extra_body_from_kwargs, dict): + for key, value in extra_body_from_kwargs.items(): + if value is None: + continue + mapped_params[key] = value + + for key in list(kwargs.keys()): + if key in reserved_kwarg_keys: + continue + value = kwargs[key] + if value is None: + continue + mapped_params[key] = value + kwargs.pop(key, None) + + if query_params: + kwargs[self.ELEVENLABS_QUERY_PARAMS_KEY] = query_params + else: + kwargs.pop(self.ELEVENLABS_QUERY_PARAMS_KEY, None) + + kwargs[self.ELEVENLABS_VOICE_ID_KEY] = mapped_voice + + def transform_text_to_speech_response( + self, + model: str, + raw_response: httpx.Response, + logging_obj: LiteLLMLoggingObj, + ) -> "HttpxBinaryResponseContent": + """ + Wrap ElevenLabs binary audio response. + """ + from litellm.types.llms.openai import HttpxBinaryResponseContent + + return HttpxBinaryResponseContent(raw_response) + + def get_complete_url( + self, + model: str, + api_base: Optional[str], + litellm_params: dict, + ) -> str: + """ + Construct the ElevenLabs endpoint URL, including path voice_id and query params. + """ + base_url = ( + api_base + or get_secret_str("ELEVENLABS_API_BASE") + or self.TTS_BASE_URL + ) + base_url = base_url.rstrip("/") + + voice_id = litellm_params.get(self.ELEVENLABS_VOICE_ID_KEY) + if not isinstance(voice_id, str) or not voice_id.strip(): + raise ValueError( + "ElevenLabs voice_id is required. Pass `voice` when calling `litellm.speech()`." + ) + + url = f"{base_url}{self.TTS_ENDPOINT_PATH}/{voice_id}" + + query_params = litellm_params.get(self.ELEVENLABS_QUERY_PARAMS_KEY, {}) + if query_params: + url = f"{url}?{urlencode(query_params)}" + + return url \ No newline at end of file diff --git a/litellm/llms/gemini/count_tokens/handler.py b/litellm/llms/gemini/count_tokens/handler.py index 4d6c7fd8864..fdb77452d4c 100644 --- a/litellm/llms/gemini/count_tokens/handler.py +++ b/litellm/llms/gemini/count_tokens/handler.py @@ -30,6 +30,10 @@ class GoogleAIStudioTokenCounter: from google.genai.types import FunctionResponse + # Handle None or empty contents + if not contents: + return contents + cleaned_contents = copy.deepcopy(contents) for content in cleaned_contents: diff --git a/litellm/llms/gemini/image_generation/transformation.py b/litellm/llms/gemini/image_generation/transformation.py index d47759d0e82..e79414394fa 100644 --- a/litellm/llms/gemini/image_generation/transformation.py +++ b/litellm/llms/gemini/image_generation/transformation.py @@ -25,6 +25,7 @@ FLASH_IMAGE_PREVIEW_MODEL_IDENTIFIERS = ( "2.0-flash-preview-image", "2.0-flash-preview-image-generation", "2.5-flash-image-preview", + "3-pro-image-preview", ) class GoogleImageGenConfig(BaseImageGenerationConfig): DEFAULT_BASE_URL: str = "https://generativelanguage.googleapis.com/v1beta" @@ -75,7 +76,7 @@ class GoogleImageGenConfig(BaseImageGenerationConfig): "1792x1024": "16:9", "1024x1792": "9:16", "1280x896": "4:3", - "896x1280": "3:4" + "896x1280": "3:4", } return aspect_ratio_map.get(size, "1:1") diff --git a/litellm/llms/gemini/vector_stores/__init__.py b/litellm/llms/gemini/vector_stores/__init__.py new file mode 100644 index 00000000000..613b5775b66 --- /dev/null +++ b/litellm/llms/gemini/vector_stores/__init__.py @@ -0,0 +1,6 @@ +"""Gemini File Search Vector Store module.""" + +from .transformation import GeminiVectorStoreConfig + +__all__ = ["GeminiVectorStoreConfig"] + diff --git a/litellm/llms/gemini/vector_stores/transformation.py b/litellm/llms/gemini/vector_stores/transformation.py new file mode 100644 index 00000000000..4d76f691e51 --- /dev/null +++ b/litellm/llms/gemini/vector_stores/transformation.py @@ -0,0 +1,357 @@ +""" +Gemini File Search Vector Store Transformation Layer. + +Implements the transformation between LiteLLM's unified vector store API +and Google Gemini's File Search API. +""" + +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union + +import httpx + +from litellm.llms.base_llm.vector_store.transformation import BaseVectorStoreConfig +from litellm.llms.gemini.common_utils import ( + GeminiError, + GeminiModelInfo, + get_api_key_from_env, +) +from litellm.types.router import GenericLiteLLMParams +from litellm.types.vector_stores import ( + VECTOR_STORE_OPENAI_PARAMS, + BaseVectorStoreAuthCredentials, + VectorStoreCreateOptionalRequestParams, + VectorStoreCreateResponse, + VectorStoreFileCounts, + VectorStoreIndexEndpoints, + VectorStoreResultContent, + VectorStoreSearchOptionalRequestParams, + VectorStoreSearchResponse, + VectorStoreSearchResult, +) + +if TYPE_CHECKING: + from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj +else: + LiteLLMLoggingObj = Any + + +class GeminiVectorStoreConfig(BaseVectorStoreConfig): + """ + Vector store configuration for Google Gemini File Search. + """ + + def __init__(self) -> None: + super().__init__() + self.model_info = GeminiModelInfo() + self._cached_api_key: Optional[str] = None + + def get_auth_credentials( + self, litellm_params: dict + ) -> BaseVectorStoreAuthCredentials: + """Gemini uses API key in query params, not headers.""" + return {} + + def get_vector_store_endpoints_by_type(self) -> VectorStoreIndexEndpoints: + """ + Gemini File Search endpoints. + + Note: Search is done via generateContent with file_search tool, + not a dedicated search endpoint. + """ + return { + "read": [("POST", "/models/{model}:generateContent")], + "write": [("POST", "/fileSearchStores")], + } + + def get_supported_openai_params( + self, model: str + ) -> List[VECTOR_STORE_OPENAI_PARAMS]: + """Supported parameters for Gemini File Search.""" + return ["max_num_results", "filters"] + + def validate_environment( + self, headers: dict, litellm_params: Optional[GenericLiteLLMParams] + ) -> dict: + """Validate and set up headers for Gemini API.""" + headers = headers or {} + headers.setdefault("Content-Type", "application/json") + if litellm_params: + api_key = litellm_params.get("api_key") or get_api_key_from_env() + if api_key: + self._cached_api_key = api_key + + return headers + + def get_complete_url(self, api_base: Optional[str], litellm_params: dict) -> str: + """ + Get the complete base URL for Gemini API. + + Note: This returns the base URL WITHOUT the API key. + The API key will be appended to specific endpoint URLs in the transform methods. + """ + if api_base is None: + api_base = GeminiModelInfo.get_api_base() + + if api_base is None: + raise ValueError("GEMINI_API_BASE is not set") + + # Ensure we're using the v1beta version for File Search + api_version = "v1beta" + return f"{api_base}/{api_version}" + + def get_error_class( + self, error_message: str, status_code: int, headers: Union[dict, httpx.Headers] + ) -> GeminiError: + """Return Gemini-specific error class.""" + return GeminiError( + status_code=status_code, + message=error_message, + headers=headers, + ) + + def transform_search_vector_store_request( + self, + vector_store_id: str, + query: Union[str, List[str]], + vector_store_search_optional_params: VectorStoreSearchOptionalRequestParams, + api_base: str, + litellm_logging_obj: LiteLLMLoggingObj, + litellm_params: dict, + ) -> Tuple[str, Dict]: + """ + Transform search request to Gemini's generateContent format. + + Gemini File Search works by calling generateContent with a file_search tool. + """ + # Convert query list to single string if needed + if isinstance(query, list): + query = " ".join(query) + + # Get model from litellm_params or use default + # Note: File Search requires gemini-2.5-flash or later + model = litellm_params.get("model") or "gemini-2.5-flash" + if model and model.startswith("gemini/"): + model = model.replace("gemini/", "") + + # Get API key - Gemini requires it as a query parameter + api_key = litellm_params.get("api_key") or GeminiModelInfo.get_api_key() + if not api_key: + raise ValueError("GEMINI_API_KEY or GOOGLE_API_KEY is required") + + # Build the URL for generateContent with API key + url = f"{api_base}/models/{model}:generateContent?key={api_key}" + + # Build file_search tool configuration (using snake_case as per Gemini docs) + file_search_config: Dict[str, Any] = { + "file_search_store_names": [vector_store_id] + } + + # Add metadata filter if provided + metadata_filter = vector_store_search_optional_params.get("filters") + if metadata_filter: + # Convert to Gemini filter syntax if it's a dict + if isinstance(metadata_filter, dict): + # Simple conversion - may need more sophisticated mapping + filter_parts = [] + for key, value in metadata_filter.items(): + if isinstance(value, str): + filter_parts.append(f'{key} = "{value}"') + else: + filter_parts.append(f'{key} = {value}') + file_search_config["metadata_filter"] = " AND ".join(filter_parts) + else: + file_search_config["metadata_filter"] = metadata_filter + + # Build request body + request_body: Dict[str, Any] = { + "contents": [ + { + "parts": [{"text": query}] + } + ], + "tools": [ + { + "file_search": file_search_config + } + ], + } + + # Add max_num_results if specified + max_results = vector_store_search_optional_params.get("max_num_results") + if max_results: + # This might need to be added to generationConfig or tool config + # depending on Gemini's API requirements + request_body.setdefault("generationConfig", {})["candidateCount"] = 1 + + litellm_logging_obj.model_call_details["query"] = query + litellm_logging_obj.model_call_details["vector_store_id"] = vector_store_id + + return url, request_body + + def transform_search_vector_store_response( + self, response: httpx.Response, litellm_logging_obj: LiteLLMLoggingObj + ) -> VectorStoreSearchResponse: + """ + Transform Gemini's generateContent response to standard format. + + Extracts grounding metadata and citations from the response. + """ + try: + response_data = response.json() + results: List[VectorStoreSearchResult] = [] + + # Extract candidates and grounding metadata + candidates = response_data.get("candidates", []) + + for candidate in candidates: + grounding_metadata = candidate.get("groundingMetadata", {}) + grounding_chunks = grounding_metadata.get("groundingChunks", []) + + # Process each grounding chunk + for chunk in grounding_chunks: + retrieved_context = chunk.get("retrievedContext") + + if retrieved_context: + # This is from file search + text = retrieved_context.get("text", "") + uri = retrieved_context.get("uri", "") + title = retrieved_context.get("title", "") + + # Extract file_id from URI if available + file_id = uri if uri else None + + results.append( + VectorStoreSearchResult( + score=None, # Gemini doesn't provide explicit scores + content=[VectorStoreResultContent(text=text, type="text")], + file_id=file_id, + filename=title if title else None, + attributes={ + "uri": uri, + "title": title, + }, + ) + ) + + # Also extract from grounding supports for more detailed citations + grounding_supports = grounding_metadata.get("groundingSupports", []) + for support in grounding_supports: + segment = support.get("segment", {}) + text = segment.get("text", "") + + grounding_chunk_indices = support.get("groundingChunkIndices", []) + confidence_scores = support.get("confidenceScores", []) + + # Use first confidence score as relevance score + score = confidence_scores[0] if confidence_scores else None + + # Only add if we have meaningful text and it's not a duplicate + if text: + already_exists = False + for record in results: + contents = record.get("content") or [] + if contents and contents[0].get("text") == text: + already_exists = True + break + if already_exists: + continue + results.append( + VectorStoreSearchResult( + score=score, + content=[VectorStoreResultContent(text=text, type="text")], + attributes={ + "grounding_chunk_indices": grounding_chunk_indices, + }, + ) + ) + + query = litellm_logging_obj.model_call_details.get("query", "") + + return VectorStoreSearchResponse( + object="vector_store.search_results.page", + search_query=query, + data=results, + ) + + except Exception as e: + raise self.get_error_class( + error_message=f"Failed to parse Gemini response: {str(e)}", + status_code=response.status_code, + headers=response.headers, + ) + + def transform_create_vector_store_request( + self, + vector_store_create_optional_params: VectorStoreCreateOptionalRequestParams, + api_base: str, + ) -> Tuple[str, Dict]: + """ + Transform create request to Gemini's fileSearchStores format. + """ + url = f"{api_base}/fileSearchStores" + + # Append API key as query parameter (required by Gemini) + api_key = self._cached_api_key or get_api_key_from_env() + if api_key: + url = f"{url}?key={api_key}" + + request_body: Dict[str, Any] = {} + + # Add display name if provided + name = vector_store_create_optional_params.get("name") + if name: + request_body["displayName"] = name + + return url, request_body + + def transform_create_vector_store_response( + self, response: httpx.Response + ) -> VectorStoreCreateResponse: + """ + Transform Gemini's fileSearchStore response to standard format. + """ + try: + response_data = response.json() + + # Extract store name (format: fileSearchStores/xxxxxxx) + store_name = response_data.get("name", "") + display_name = response_data.get("displayName", "") + create_time = response_data.get("createTime", "") + + # Convert ISO timestamp to Unix timestamp + import datetime + created_at = None + if create_time: + try: + dt = datetime.datetime.fromisoformat(create_time.replace("Z", "+00:00")) + created_at = int(dt.timestamp()) + except Exception: + created_at = None + + return VectorStoreCreateResponse( + id=store_name, + object="vector_store", + created_at=created_at or 0, + name=display_name, + bytes=0, # Gemini doesn't provide size info on creation + file_counts=VectorStoreFileCounts( + in_progress=0, + completed=0, + failed=0, + cancelled=0, + total=0, + ), + status="completed", + expires_after=None, + expires_at=None, + last_active_at=None, + metadata=None, + ) + + except Exception as e: + raise self.get_error_class( + error_message=f"Failed to parse Gemini create response: {str(e)}", + status_code=response.status_code, + headers=response.headers, + ) + diff --git a/litellm/llms/gemini/videos/transformation.py b/litellm/llms/gemini/videos/transformation.py index ce2519e9177..4120d1cad22 100644 --- a/litellm/llms/gemini/videos/transformation.py +++ b/litellm/llms/gemini/videos/transformation.py @@ -160,11 +160,16 @@ class GeminiVideoConfig(BaseVideoConfig): headers: dict, model: str, api_key: Optional[str] = None, + litellm_params: Optional[GenericLiteLLMParams] = None, ) -> dict: """ Validate environment and add Gemini API key to headers. Gemini uses x-goog-api-key header for authentication. """ + # Use api_key from litellm_params if available, otherwise fall back to other sources + if litellm_params and litellm_params.api_key: + api_key = api_key or litellm_params.api_key + api_key = ( api_key or litellm.api_key diff --git a/litellm/llms/openai/chat/gpt_5_transformation.py b/litellm/llms/openai/chat/gpt_5_transformation.py index d18f898cf1c..60a172ef817 100644 --- a/litellm/llms/openai/chat/gpt_5_transformation.py +++ b/litellm/llms/openai/chat/gpt_5_transformation.py @@ -25,6 +25,15 @@ class OpenAIGPT5Config(OpenAIGPTConfig): def is_model_gpt_5_codex_model(cls, model: str) -> bool: """Check if the model is specifically a GPT-5 Codex variant.""" return "gpt-5-codex" in model + + @classmethod + def is_model_gpt_5_1_model(cls, model: str) -> bool: + """Check if the model is a gpt-5.1 variant. + + gpt-5.1 supports temperature when reasoning_effort="none", + unlike gpt-5 which only supports temperature=1. + """ + return "gpt-5.1" in model def get_supported_openai_params(self, model: str) -> list: from litellm.utils import supports_tool_choice @@ -69,14 +78,26 @@ class OpenAIGPT5Config(OpenAIGPTConfig): if "temperature" in non_default_params: temperature_value: Optional[float] = non_default_params.pop("temperature") if temperature_value is not None: - if temperature_value == 1: + is_gpt_5_1 = self.is_model_gpt_5_1_model(model) + reasoning_effort = ( + non_default_params.get("reasoning_effort") + or optional_params.get("reasoning_effort") + ) + + # gpt-5.1 supports any temperature when reasoning_effort="none" (or not specified, as it defaults to "none") + if is_gpt_5_1 and (reasoning_effort == "none" or reasoning_effort is None): + optional_params["temperature"] = temperature_value + elif temperature_value == 1: optional_params["temperature"] = temperature_value elif litellm.drop_params or drop_params: pass else: raise litellm.utils.UnsupportedParamsError( message=( - "gpt-5 models (including gpt-5-codex) don't support temperature={}. Only temperature=1 is supported. To drop unsupported params set `litellm.drop_params = True`" + "gpt-5 models (including gpt-5-codex) don't support temperature={}. " + "Only temperature=1 is supported. " + "For gpt-5.1, temperature is supported when reasoning_effort='none' (or not specified, as it defaults to 'none'). " + "To drop unsupported params set `litellm.drop_params = True`" ).format(temperature_value), status_code=400, ) diff --git a/litellm/llms/openai/videos/transformation.py b/litellm/llms/openai/videos/transformation.py index d1d3fc2919e..abdcd2fbe7b 100644 --- a/litellm/llms/openai/videos/transformation.py +++ b/litellm/llms/openai/videos/transformation.py @@ -61,7 +61,12 @@ class OpenAIVideoConfig(BaseVideoConfig): headers: dict, model: str, api_key: Optional[str] = None, + litellm_params: Optional[GenericLiteLLMParams] = None, ) -> dict: + # Use api_key from litellm_params if available, otherwise fall back to other sources + if litellm_params and litellm_params.api_key: + api_key = api_key or litellm_params.api_key + api_key = ( api_key or litellm.api_key diff --git a/litellm/llms/runwayml/videos/transformation.py b/litellm/llms/runwayml/videos/transformation.py index 651acff6fc4..5a46ebb664b 100644 --- a/litellm/llms/runwayml/videos/transformation.py +++ b/litellm/llms/runwayml/videos/transformation.py @@ -114,11 +114,16 @@ class RunwayMLVideoConfig(BaseVideoConfig): headers: dict, model: str, api_key: Optional[str] = None, + litellm_params: Optional[GenericLiteLLMParams] = None, ) -> dict: """ Validate environment and set up authentication headers. RunwayML uses Bearer token authentication via RUNWAYML_API_SECRET. """ + # Use api_key from litellm_params if available, otherwise fall back to other sources + if litellm_params and litellm_params.api_key: + api_key = api_key or litellm_params.api_key + api_key = ( api_key or litellm.api_key diff --git a/litellm/llms/vertex_ai/common_utils.py b/litellm/llms/vertex_ai/common_utils.py index 2c534577366..dc6a3170afe 100644 --- a/litellm/llms/vertex_ai/common_utils.py +++ b/litellm/llms/vertex_ai/common_utils.py @@ -274,6 +274,57 @@ def _fix_enum_empty_strings(schema, depth=0): _fix_enum_empty_strings(items, depth=depth + 1) +def _fix_enum_types(schema, depth=0): + """Remove `enum` fields when the schema type is not string. + + Gemini / Vertex APIs only allow enums for string-typed fields. When an enum + is present on a non-string typed property (or when `anyOf` types do not + include a string type), remove the enum to avoid provider validation errors. + """ + if depth > DEFAULT_MAX_RECURSE_DEPTH: + raise ValueError( + f"Max depth of {DEFAULT_MAX_RECURSE_DEPTH} exceeded while processing schema." + ) + + if not isinstance(schema, dict): + return + + # If enum exists but type is not string (and anyOf doesn't include string), drop enum + if "enum" in schema and isinstance(schema["enum"], list): + schema_type = schema.get("type") + keep_enum = False + if isinstance(schema_type, str) and schema_type.lower() == "string": + keep_enum = True + else: + anyof = schema.get("anyOf") + if isinstance(anyof, list): + for item in anyof: + if isinstance(item, dict): + item_type = item.get("type") + if isinstance(item_type, str) and item_type.lower() == "string": + keep_enum = True + break + + if not keep_enum: + schema.pop("enum", None) + + # Recurse into nested structures + properties = schema.get("properties", None) + if properties is not None: + for _, value in properties.items(): + _fix_enum_types(value, depth=depth + 1) + + items = schema.get("items", None) + if items is not None: + _fix_enum_types(items, depth=depth + 1) + + anyof = schema.get("anyOf", None) + if anyof is not None and isinstance(anyof, list): + for item in anyof: + if isinstance(item, dict): + _fix_enum_types(item, depth=depth + 1) + + def _build_vertex_schema(parameters: dict, add_property_ordering: bool = False): """ This is a modified version of https://github.com/google-gemini/generative-ai-python/blob/8f77cc6ac99937cd3a81299ecf79608b91b06bbb/google/generativeai/types/content_types.py#L419 @@ -307,6 +358,9 @@ def _build_vertex_schema(parameters: dict, add_property_ordering: bool = False): # Handle empty strings in enum values - Gemini doesn't accept empty strings in enums _fix_enum_empty_strings(parameters) + # Remove enums for non-string typed fields (Gemini requires enum only on strings) + _fix_enum_types(parameters) + # Handle empty items objects process_items(parameters) add_object_type(parameters) diff --git a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py index ab594c79ef4..5fef8c1ec49 100644 --- a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py +++ b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py @@ -904,13 +904,15 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): if VertexGeminiConfig._is_gemini_3_or_newer(model): if "temperature" not in optional_params: optional_params["temperature"] = 1.0 - thinking_config = optional_params.get("thinkingConfig", {}) - if ( - "thinkingLevel" not in thinking_config - and "thinkingBudget" not in thinking_config - ): - thinking_config["thinkingLevel"] = "low" - optional_params["thinkingConfig"] = thinking_config + # Only add thinkingLevel if model supports it (exclude image models) + if "image" not in model.lower(): + thinking_config = optional_params.get("thinkingConfig", {}) + if ( + "thinkingLevel" not in thinking_config + and "thinkingBudget" not in thinking_config + ): + thinking_config["thinkingLevel"] = "low" + optional_params["thinkingConfig"] = thinking_config return optional_params diff --git a/litellm/llms/vertex_ai/image_generation/__init__.py b/litellm/llms/vertex_ai/image_generation/__init__.py new file mode 100644 index 00000000000..a6f6156167a --- /dev/null +++ b/litellm/llms/vertex_ai/image_generation/__init__.py @@ -0,0 +1,43 @@ +from litellm.llms.base_llm.image_generation.transformation import ( + BaseImageGenerationConfig, +) +from litellm.llms.vertex_ai.common_utils import ( + VertexAIModelRoute, + get_vertex_ai_model_route, +) + +from .vertex_gemini_transformation import VertexAIGeminiImageGenerationConfig +from .vertex_imagen_transformation import VertexAIImagenImageGenerationConfig + +__all__ = [ + "VertexAIGeminiImageGenerationConfig", + "VertexAIImagenImageGenerationConfig", + "get_vertex_ai_image_generation_config", +] + + +def get_vertex_ai_image_generation_config(model: str) -> BaseImageGenerationConfig: + """ + Get the appropriate image generation config for a Vertex AI model. + + Routes to the correct transformation class based on the model type: + - Gemini image generation models use generateContent API (VertexAIGeminiImageGenerationConfig) + - Imagen models use predict API (VertexAIImagenImageGenerationConfig) + + Args: + model: The model name (e.g., "gemini-2.5-flash-image", "imagegeneration@006") + + Returns: + BaseImageGenerationConfig: The appropriate configuration class + """ + # Determine the model route + model_route = get_vertex_ai_model_route(model) + + if model_route == VertexAIModelRoute.GEMINI: + # Gemini models use generateContent API + return VertexAIGeminiImageGenerationConfig() + else: + # Default to Imagen for other models (imagegeneration, etc.) + # This includes NON_GEMINI models like imagegeneration@006 + return VertexAIImagenImageGenerationConfig() + diff --git a/litellm/llms/vertex_ai/image_generation/vertex_gemini_transformation.py b/litellm/llms/vertex_ai/image_generation/vertex_gemini_transformation.py new file mode 100644 index 00000000000..149e0850bf0 --- /dev/null +++ b/litellm/llms/vertex_ai/image_generation/vertex_gemini_transformation.py @@ -0,0 +1,264 @@ +import os +from typing import TYPE_CHECKING, Any, Dict, List, Optional + +import httpx + +import litellm +from litellm.llms.base_llm.image_generation.transformation import ( + BaseImageGenerationConfig, +) +from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import VertexLLM +from litellm.secret_managers.main import get_secret_str +from litellm.types.llms.openai import ( + AllMessageValues, + OpenAIImageGenerationOptionalParams, +) +from litellm.types.utils import ImageObject, ImageResponse + +if TYPE_CHECKING: + from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + + LiteLLMLoggingObj = _LiteLLMLoggingObj +else: + LiteLLMLoggingObj = Any + + +class VertexAIGeminiImageGenerationConfig(BaseImageGenerationConfig, VertexLLM): + """ + Vertex AI Gemini Image Generation Configuration + + Uses generateContent API for Gemini image generation models on Vertex AI + Supports models like gemini-2.5-flash-image, gemini-3-pro-image-preview, etc. + """ + + def __init__(self) -> None: + BaseImageGenerationConfig.__init__(self) + VertexLLM.__init__(self) + + def get_supported_openai_params( + self, model: str + ) -> List[OpenAIImageGenerationOptionalParams]: + """ + Gemini image generation supported parameters + """ + return [ + "n", + "size", + ] + + def map_openai_params( + self, + non_default_params: dict, + optional_params: dict, + model: str, + drop_params: bool, + ) -> dict: + supported_params = self.get_supported_openai_params(model) + mapped_params = {} + + for k, v in non_default_params.items(): + if k not in optional_params.keys(): + if k in supported_params: + # Map OpenAI parameters to Gemini format + if k == "n": + mapped_params["candidate_count"] = v + elif k == "size": + # Map OpenAI size format to Gemini aspectRatio + mapped_params["aspectRatio"] = self._map_size_to_aspect_ratio(v) + else: + mapped_params[k] = v + + return mapped_params + + def _map_size_to_aspect_ratio(self, size: str) -> str: + """ + Map OpenAI size format to Gemini aspect ratio format + """ + aspect_ratio_map = { + "1024x1024": "1:1", + "1792x1024": "16:9", + "1024x1792": "9:16", + "1280x896": "4:3", + "896x1280": "3:4" + } + return aspect_ratio_map.get(size, "1:1") + + def _resolve_vertex_project(self) -> Optional[str]: + return ( + getattr(self, "_vertex_project", None) + or os.environ.get("VERTEXAI_PROJECT") + or getattr(litellm, "vertex_project", None) + or get_secret_str("VERTEXAI_PROJECT") + ) + + def _resolve_vertex_location(self) -> Optional[str]: + return ( + getattr(self, "_vertex_location", None) + or os.environ.get("VERTEXAI_LOCATION") + or os.environ.get("VERTEX_LOCATION") + or getattr(litellm, "vertex_location", None) + or get_secret_str("VERTEXAI_LOCATION") + or get_secret_str("VERTEX_LOCATION") + ) + + def _resolve_vertex_credentials(self) -> Optional[str]: + return ( + getattr(self, "_vertex_credentials", None) + or os.environ.get("VERTEXAI_CREDENTIALS") + or getattr(litellm, "vertex_credentials", None) + or os.environ.get("GOOGLE_APPLICATION_CREDENTIALS") + or get_secret_str("VERTEXAI_CREDENTIALS") + ) + + def get_complete_url( + self, + api_base: Optional[str], + api_key: Optional[str], + model: str, + optional_params: dict, + litellm_params: dict, + stream: Optional[bool] = None, + ) -> str: + """ + Get the complete URL for Vertex AI Gemini generateContent API + """ + vertex_project = self._resolve_vertex_project() + vertex_location = self._resolve_vertex_location() + + if not vertex_project or not vertex_location: + raise ValueError("vertex_project and vertex_location are required for Vertex AI") + + # Use the model name as provided, handling vertex_ai prefix + model_name = model + if model.startswith("vertex_ai/"): + model_name = model.replace("vertex_ai/", "") + + if api_base: + base_url = api_base.rstrip("/") + else: + base_url = f"https://{vertex_location}-aiplatform.googleapis.com" + + return f"{base_url}/v1/projects/{vertex_project}/locations/{vertex_location}/publishers/google/models/{model_name}:generateContent" + + def validate_environment( + self, + headers: dict, + model: str, + messages: List[AllMessageValues], + optional_params: dict, + litellm_params: dict, + api_key: Optional[str] = None, + api_base: Optional[str] = None, + ) -> dict: + headers = headers or {} + vertex_project = self._resolve_vertex_project() + vertex_credentials = self._resolve_vertex_credentials() + access_token, _ = self._ensure_access_token( + credentials=vertex_credentials, + project_id=vertex_project, + custom_llm_provider="vertex_ai", + ) + return self.set_headers(access_token, headers) + + def transform_image_generation_request( + self, + model: str, + prompt: str, + optional_params: dict, + litellm_params: dict, + headers: dict, + ) -> dict: + """ + Transform the image generation request to Gemini format + + Uses generateContent API with responseModalities: ["IMAGE"] + """ + # Prepare messages with the prompt + contents = [ + { + "role": "user", + "parts": [{"text": prompt}] + } + ] + + # Prepare generation config + generation_config: Dict[str, Any] = { + "responseModalities": ["IMAGE"] + } + + # Handle image-specific config parameters + image_config: Dict[str, Any] = {} + + # Map aspectRatio + if "aspectRatio" in optional_params: + image_config["aspectRatio"] = optional_params["aspectRatio"] + elif "aspect_ratio" in optional_params: + image_config["aspectRatio"] = optional_params["aspect_ratio"] + + # Map imageSize (for Gemini 3 Pro) + if "imageSize" in optional_params: + image_config["imageSize"] = optional_params["imageSize"] + elif "image_size" in optional_params: + image_config["imageSize"] = optional_params["image_size"] + + if image_config: + generation_config["imageConfig"] = image_config + + # Handle candidate_count (n parameter) + if "candidate_count" in optional_params: + generation_config["candidateCount"] = optional_params["candidate_count"] + elif "n" in optional_params: + generation_config["candidateCount"] = optional_params["n"] + + request_body: Dict[str, Any] = { + "contents": contents, + "generationConfig": generation_config + } + + return request_body + + def transform_image_generation_response( + self, + model: str, + raw_response: httpx.Response, + model_response: ImageResponse, + logging_obj: LiteLLMLoggingObj, + request_data: dict, + optional_params: dict, + litellm_params: dict, + encoding: Any, + api_key: Optional[str] = None, + json_mode: Optional[bool] = None, + ) -> ImageResponse: + """ + Transform Gemini image generation response to litellm ImageResponse format + """ + try: + response_data = raw_response.json() + except Exception as e: + raise self.get_error_class( + error_message=f"Error transforming image generation response: {e}", + status_code=raw_response.status_code, + headers=raw_response.headers, + ) + + if not model_response.data: + model_response.data = [] + + # Gemini image generation models return in candidates format + candidates = response_data.get("candidates", []) + for candidate in candidates: + content = candidate.get("content", {}) + parts = content.get("parts", []) + for part in parts: + # Look for inlineData with image + if "inlineData" in part: + inline_data = part["inlineData"] + if "data" in inline_data: + model_response.data.append(ImageObject( + b64_json=inline_data["data"], + url=None, + )) + + return model_response + diff --git a/litellm/llms/vertex_ai/image_generation/vertex_imagen_transformation.py b/litellm/llms/vertex_ai/image_generation/vertex_imagen_transformation.py new file mode 100644 index 00000000000..8c4ad5dd423 --- /dev/null +++ b/litellm/llms/vertex_ai/image_generation/vertex_imagen_transformation.py @@ -0,0 +1,230 @@ +import os +from typing import TYPE_CHECKING, Any, List, Optional + +import httpx + +import litellm +from litellm.llms.base_llm.image_generation.transformation import ( + BaseImageGenerationConfig, +) +from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import VertexLLM +from litellm.secret_managers.main import get_secret_str +from litellm.types.llms.openai import ( + AllMessageValues, + OpenAIImageGenerationOptionalParams, +) +from litellm.types.utils import ImageObject, ImageResponse + +if TYPE_CHECKING: + from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + + LiteLLMLoggingObj = _LiteLLMLoggingObj +else: + LiteLLMLoggingObj = Any + + +class VertexAIImagenImageGenerationConfig(BaseImageGenerationConfig, VertexLLM): + """ + Vertex AI Imagen Image Generation Configuration + + Uses predict API for Imagen models on Vertex AI + Supports models like imagegeneration@006 + """ + + def __init__(self) -> None: + BaseImageGenerationConfig.__init__(self) + VertexLLM.__init__(self) + + def get_supported_openai_params( + self, model: str + ) -> List[OpenAIImageGenerationOptionalParams]: + """ + Imagen API supported parameters + """ + return [ + "n", + "size" + ] + + def map_openai_params( + self, + non_default_params: dict, + optional_params: dict, + model: str, + drop_params: bool, + ) -> dict: + supported_params = self.get_supported_openai_params(model) + mapped_params = {} + + for k, v in non_default_params.items(): + if k not in optional_params.keys(): + if k in supported_params: + # Map OpenAI parameters to Imagen format + if k == "n": + mapped_params["sampleCount"] = v + elif k == "size": + # Map OpenAI size format to Imagen aspectRatio + mapped_params["aspectRatio"] = self._map_size_to_aspect_ratio(v) + else: + mapped_params[k] = v + + return mapped_params + + def _map_size_to_aspect_ratio(self, size: str) -> str: + """ + Map OpenAI size format to Imagen aspect ratio format + """ + aspect_ratio_map = { + "1024x1024": "1:1", + "1792x1024": "16:9", + "1024x1792": "9:16", + "1280x896": "4:3", + "896x1280": "3:4" + } + return aspect_ratio_map.get(size, "1:1") + + def _resolve_vertex_project(self) -> Optional[str]: + return ( + getattr(self, "_vertex_project", None) + or os.environ.get("VERTEXAI_PROJECT") + or getattr(litellm, "vertex_project", None) + or get_secret_str("VERTEXAI_PROJECT") + ) + + def _resolve_vertex_location(self) -> Optional[str]: + return ( + getattr(self, "_vertex_location", None) + or os.environ.get("VERTEXAI_LOCATION") + or os.environ.get("VERTEX_LOCATION") + or getattr(litellm, "vertex_location", None) + or get_secret_str("VERTEXAI_LOCATION") + or get_secret_str("VERTEX_LOCATION") + ) + + def _resolve_vertex_credentials(self) -> Optional[str]: + return ( + getattr(self, "_vertex_credentials", None) + or os.environ.get("VERTEXAI_CREDENTIALS") + or getattr(litellm, "vertex_credentials", None) + or os.environ.get("GOOGLE_APPLICATION_CREDENTIALS") + or get_secret_str("VERTEXAI_CREDENTIALS") + ) + + def get_complete_url( + self, + api_base: Optional[str], + api_key: Optional[str], + model: str, + optional_params: dict, + litellm_params: dict, + stream: Optional[bool] = None, + ) -> str: + """ + Get the complete URL for Vertex AI Imagen predict API + """ + vertex_project = self._resolve_vertex_project() + vertex_location = self._resolve_vertex_location() + + if not vertex_project or not vertex_location: + raise ValueError("vertex_project and vertex_location are required for Vertex AI") + + # Use the model name as provided, handling vertex_ai prefix + model_name = model + if model.startswith("vertex_ai/"): + model_name = model.replace("vertex_ai/", "") + + if api_base: + base_url = api_base.rstrip("/") + else: + base_url = f"https://{vertex_location}-aiplatform.googleapis.com" + + return f"{base_url}/v1/projects/{vertex_project}/locations/{vertex_location}/publishers/google/models/{model_name}:predict" + + def validate_environment( + self, + headers: dict, + model: str, + messages: List[AllMessageValues], + optional_params: dict, + litellm_params: dict, + api_key: Optional[str] = None, + api_base: Optional[str] = None, + ) -> dict: + headers = headers or {} + vertex_project = self._resolve_vertex_project() + vertex_credentials = self._resolve_vertex_credentials() + access_token, _ = self._ensure_access_token( + credentials=vertex_credentials, + project_id=vertex_project, + custom_llm_provider="vertex_ai", + ) + return self.set_headers(access_token, headers) + + def transform_image_generation_request( + self, + model: str, + prompt: str, + optional_params: dict, + litellm_params: dict, + headers: dict, + ) -> dict: + """ + Transform the image generation request to Imagen format + + Uses predict API with instances and parameters + """ + # Default parameters + default_params = { + "sampleCount": 1, + } + + # Merge with optional params + parameters = {**default_params, **optional_params} + + request_body = { + "instances": [{"prompt": prompt}], + "parameters": parameters, + } + + return request_body + + def transform_image_generation_response( + self, + model: str, + raw_response: httpx.Response, + model_response: ImageResponse, + logging_obj: LiteLLMLoggingObj, + request_data: dict, + optional_params: dict, + litellm_params: dict, + encoding: Any, + api_key: Optional[str] = None, + json_mode: Optional[bool] = None, + ) -> ImageResponse: + """ + Transform Imagen image generation response to litellm ImageResponse format + """ + try: + response_data = raw_response.json() + except Exception as e: + raise self.get_error_class( + error_message=f"Error transforming image generation response: {e}", + status_code=raw_response.status_code, + headers=raw_response.headers, + ) + + if not model_response.data: + model_response.data = [] + + # Imagen format - predictions with generated images + predictions = response_data.get("predictions", []) + for prediction in predictions: + # Imagen returns images as bytesBase64Encoded + if "bytesBase64Encoded" in prediction: + model_response.data.append(ImageObject( + b64_json=prediction["bytesBase64Encoded"], + url=None, + )) + + return model_response + diff --git a/litellm/llms/vertex_ai/vertex_ai_partner_models/count_tokens/handler.py b/litellm/llms/vertex_ai/vertex_ai_partner_models/count_tokens/handler.py index da76b12c371..ae1a758bf20 100644 --- a/litellm/llms/vertex_ai/vertex_ai_partner_models/count_tokens/handler.py +++ b/litellm/llms/vertex_ai/vertex_ai_partner_models/count_tokens/handler.py @@ -65,6 +65,8 @@ class VertexAIPartnerModelsTokenCounter(VertexBase): # Use custom api_base if provided, otherwise construct default if api_base: base_url = api_base + elif vertex_location == "global": + base_url = "https://aiplatform.googleapis.com" else: base_url = f"https://{vertex_location}-aiplatform.googleapis.com" diff --git a/litellm/llms/vertex_ai/videos/transformation.py b/litellm/llms/vertex_ai/videos/transformation.py index 1f657f63bf1..5d73a42286a 100644 --- a/litellm/llms/vertex_ai/videos/transformation.py +++ b/litellm/llms/vertex_ai/videos/transformation.py @@ -12,6 +12,8 @@ from typing import TYPE_CHECKING, Any, Dict, Optional, Tuple, Union, cast import httpx from httpx._types import RequestFiles +from litellm.constants import DEFAULT_GOOGLE_VIDEO_DURATION_SECONDS +from litellm.images.utils import ImageEditRequestUtils from litellm.llms.base_llm.videos.transformation import BaseVideoConfig from litellm.llms.vertex_ai.common_utils import ( _convert_vertex_datetime_to_openai_datetime, @@ -23,8 +25,6 @@ from litellm.types.videos.utils import ( encode_video_id_with_provider, extract_original_video_id, ) -from litellm.images.utils import ImageEditRequestUtils -from litellm.constants import DEFAULT_GOOGLE_VIDEO_DURATION_SECONDS if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj @@ -160,13 +160,11 @@ class VertexAIVideoConfig(BaseVideoConfig, VertexBase): def validate_environment( self, - headers: Dict, + headers: dict, model: str, api_key: Optional[str] = None, - api_base: Optional[str] = None, - litellm_params: Optional[dict] = None, - **kwargs, - ) -> Dict: + litellm_params: Optional[GenericLiteLLMParams] = None, + ) -> dict: """ Validate environment and return headers for Vertex AI OCR. diff --git a/litellm/main.py b/litellm/main.py index 4482cf5d123..afc7a36fb40 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -154,6 +154,7 @@ from .litellm_core_utils.prompt_templates.factory import ( ) from .litellm_core_utils.streaming_chunk_builder_utils import ChunkProcessor from .llms.anthropic.chat import AnthropicChatCompletion +from .llms.azure.anthropic.handler import AzureAnthropicChatCompletion from .llms.azure.audio_transcriptions import AzureAudioTranscription from .llms.azure.azure import AzureChatCompletion, _check_dynamic_azure_params from .llms.azure.chat.o_series_handler import AzureOpenAIO1ChatCompletion @@ -255,6 +256,7 @@ openai_image_variations = OpenAIImageVariationsHandler() groq_chat_completions = GroqChatCompletion() azure_ai_embedding = AzureAIEmbedding() anthropic_chat_completions = AnthropicChatCompletion() +azure_anthropic_chat_completions = AzureAnthropicChatCompletion() azure_chat_completions = AzureChatCompletion() azure_o1_chat_completions = AzureOpenAIO1ChatCompletion() azure_text_completions = AzureTextCompletion() @@ -2357,6 +2359,70 @@ def completion( # type: ignore # noqa: PLR0915 original_response=response, ) response = response + elif custom_llm_provider == "azure_anthropic": + # Azure Anthropic uses same API as Anthropic but with Azure authentication + api_key = ( + api_key + or litellm.azure_key + or litellm.api_key + or get_secret("AZURE_API_KEY") + or get_secret("AZURE_OPENAI_API_KEY") + ) + custom_prompt_dict = custom_prompt_dict or litellm.custom_prompt_dict + # Azure Foundry endpoint format: https://.services.ai.azure.com/anthropic/v1/messages + api_base = ( + api_base + or litellm.api_base + or get_secret("AZURE_API_BASE") + ) + + if api_base is None: + raise ValueError( + "Missing Azure API Base - Please set `api_base` or `AZURE_API_BASE` environment variable. " + "Expected format: https://.services.ai.azure.com/anthropic" + ) + + # Ensure the URL ends with /v1/messages + api_base = api_base.rstrip("/") + if api_base.endswith("/v1/messages"): + pass + elif api_base.endswith("/anthropic/v1/messages"): + pass + else: + if "/anthropic" in api_base: + parts = api_base.split("/anthropic", 1) + api_base = parts[0] + "/anthropic" + else: + api_base = api_base + "/anthropic" + api_base = api_base + "/v1/messages" + + response = azure_anthropic_chat_completions.completion( + model=model, + messages=messages, + api_base=api_base, + acompletion=acompletion, + custom_prompt_dict=litellm.custom_prompt_dict, + model_response=model_response, + print_verbose=print_verbose, + optional_params=optional_params, + litellm_params=litellm_params, + logger_fn=logger_fn, + encoding=encoding, # for calculating input/output tokens + api_key=api_key, + logging_obj=logging, + headers=headers, + timeout=timeout, + client=client, + custom_llm_provider=custom_llm_provider, + ) + if optional_params.get("stream", False) or acompletion is True: + ## LOGGING + logging.post_call( + input=messages, + api_key=api_key, + original_response=response, + ) + response = response elif custom_llm_provider == "nlp_cloud": nlp_cloud_key = ( api_key @@ -4019,7 +4085,11 @@ def embedding( # noqa: PLR0915 azure_ad_token_provider = kwargs.get("azure_ad_token_provider", None) aembedding: Optional[bool] = kwargs.get("aembedding", None) extra_headers = kwargs.get("extra_headers", None) - headers = kwargs.get("headers", None) + headers = kwargs.get("headers", None) or extra_headers + if headers is None: + headers = {} + if extra_headers is not None: + headers.update(extra_headers) ### CUSTOM MODEL COST ### input_cost_per_token = kwargs.get("input_cost_per_token", None) output_cost_per_token = kwargs.get("output_cost_per_token", None) @@ -4330,7 +4400,7 @@ def embedding( # noqa: PLR0915 litellm_params={}, api_base=api_base, print_verbose=print_verbose, - extra_headers=extra_headers, + extra_headers=headers, api_key=api_key, ) elif custom_llm_provider == "triton": @@ -5764,7 +5834,9 @@ def speech( # noqa: PLR0915 custom_llm_provider: Optional[str] = None, aspeech: Optional[bool] = None, **kwargs, -) -> HttpxBinaryResponseContent: +) -> Union[ + HttpxBinaryResponseContent, Coroutine[Any, Any, HttpxBinaryResponseContent] +]: user = kwargs.get("user", None) litellm_call_id: Optional[str] = kwargs.get("litellm_call_id", None) proxy_server_request = kwargs.get("proxy_server_request", None) @@ -5824,7 +5896,11 @@ def speech( # noqa: PLR0915 }, custom_llm_provider=custom_llm_provider, ) - response: Optional[HttpxBinaryResponseContent] = None + response: Union[ + HttpxBinaryResponseContent, + Coroutine[Any, Any, HttpxBinaryResponseContent], + None, + ] = None if ( custom_llm_provider == "openai" or custom_llm_provider in litellm.openai_compatible_providers @@ -5962,6 +6038,58 @@ def speech( # noqa: PLR0915 aspeech=aspeech, litellm_params=litellm_params_dict, ) + elif custom_llm_provider == "elevenlabs": + from litellm.llms.elevenlabs.text_to_speech.transformation import ( + ElevenLabsTextToSpeechConfig, + ) + + if text_to_speech_provider_config is None: + text_to_speech_provider_config = ElevenLabsTextToSpeechConfig() + + elevenlabs_config = cast( + ElevenLabsTextToSpeechConfig, text_to_speech_provider_config + ) + + voice_id = voice if isinstance(voice, str) else None + if voice_id is None or not voice_id.strip(): + raise litellm.BadRequestError( + message="'voice' must resolve to an ElevenLabs voice id for ElevenLabs TTS", + model=model, + llm_provider=custom_llm_provider, + ) + voice_id = voice_id.strip() + + query_params = kwargs.pop( + ElevenLabsTextToSpeechConfig.ELEVENLABS_QUERY_PARAMS_KEY, None + ) + if isinstance(query_params, dict): + litellm_params_dict[ + ElevenLabsTextToSpeechConfig.ELEVENLABS_QUERY_PARAMS_KEY + ] = query_params + + litellm_params_dict[ + ElevenLabsTextToSpeechConfig.ELEVENLABS_VOICE_ID_KEY + ] = voice_id + + if api_base is not None: + litellm_params_dict["api_base"] = api_base + if api_key is not None: + litellm_params_dict["api_key"] = api_key + + response = base_llm_http_handler.text_to_speech_handler( + model=model, + input=input, + voice=voice_id, + text_to_speech_provider_config=elevenlabs_config, + text_to_speech_optional_params=optional_params, + custom_llm_provider=custom_llm_provider, + litellm_params=litellm_params_dict, + logging_obj=logging_obj, + timeout=timeout, + extra_headers=extra_headers, + client=client, + _is_async=aspeech or False, + ) elif custom_llm_provider == "vertex_ai" or custom_llm_provider == "vertex_ai_beta": generic_optional_params = GenericLiteLLMParams(**kwargs) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 3b1a31d5018..1fe299815cc 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -1143,6 +1143,60 @@ "output_cost_per_token": 1.5e-05, "supports_function_calling": true }, + "azure/claude-haiku-4-5": { + "input_cost_per_token": 1e-06, + "litellm_provider": "azure_anthropic", + "max_input_tokens": 200000, + "max_output_tokens": 64000, + "max_tokens": 64000, + "mode": "chat", + "output_cost_per_token": 5e-06, + "supports_assistant_prefill": true, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "azure/claude-opus-4-1": { + "input_cost_per_token": 1.5e-05, + "litellm_provider": "azure_anthropic", + "max_input_tokens": 200000, + "max_output_tokens": 32000, + "max_tokens": 32000, + "mode": "chat", + "output_cost_per_token": 7.5e-05, + "supports_assistant_prefill": true, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "azure/claude-sonnet-4-5": { + "input_cost_per_token": 3e-06, + "litellm_provider": "azure_anthropic", + "max_input_tokens": 200000, + "max_output_tokens": 64000, + "max_tokens": 64000, + "mode": "chat", + "output_cost_per_token": 1.5e-05, + "supports_assistant_prefill": true, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, "azure/computer-use-preview": { "input_cost_per_token": 3e-06, "litellm_provider": "azure", @@ -9468,6 +9522,15 @@ "output_cost_per_token": 0.0, "supports_embedding_image_input": true }, + "embed-multilingual-light-v3.0": { + "input_cost_per_token": 1e-04, + "litellm_provider": "cohere", + "max_input_tokens": 1024, + "max_tokens": 1024, + "mode": "embedding", + "output_cost_per_token": 0.0, + "supports_embedding_image_input": true + }, "eu.amazon.nova-lite-v1:0": { "input_cost_per_token": 7.8e-08, "litellm_provider": "bedrock_converse", @@ -24551,6 +24614,58 @@ "supports_tool_choice": true, "supports_vision": true }, + "vertex_ai/claude-opus-4-5": { + "cache_creation_input_token_cost": 6.25e-06, + "cache_read_input_token_cost": 5e-07, + "input_cost_per_token": 5e-06, + "litellm_provider": "vertex_ai-anthropic_models", + "max_input_tokens": 200000, + "max_output_tokens": 64000, + "max_tokens": 64000, + "mode": "chat", + "output_cost_per_token": 2.5e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_assistant_prefill": true, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "tool_use_system_prompt_tokens": 159 + }, + "vertex_ai/claude-opus-4-5@20251101": { + "cache_creation_input_token_cost": 6.25e-06, + "cache_read_input_token_cost": 5e-07, + "input_cost_per_token": 5e-06, + "litellm_provider": "vertex_ai-anthropic_models", + "max_input_tokens": 200000, + "max_output_tokens": 64000, + "max_tokens": 64000, + "mode": "chat", + "output_cost_per_token": 2.5e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_assistant_prefill": true, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "tool_use_system_prompt_tokens": 159 + }, "vertex_ai/claude-sonnet-4-5": { "cache_creation_input_token_cost": 3.75e-06, "cache_read_input_token_cost": 3e-07, diff --git a/litellm/passthrough/main.py b/litellm/passthrough/main.py index cc57ceac50e..3df3037ed58 100644 --- a/litellm/passthrough/main.py +++ b/litellm/passthrough/main.py @@ -258,7 +258,7 @@ def llm_passthrough_route( model=model, messages=[], optional_params={}, - litellm_params={}, + litellm_params=litellm_params_dict, api_key=provider_api_key, api_base=base_target_url, ) diff --git a/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py b/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py index 4a0d25e24f3..25b5211464c 100644 --- a/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py +++ b/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py @@ -1976,7 +1976,7 @@ class MCPServerManager: verbose_logger.debug( f"Adding server to registry: {server.server_id} ({server.server_name})" ) - self.add_update_server(server) + await self.add_update_server(server) verbose_logger.debug( f"Registry now contains {len(self.get_registry())} servers" diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 8a8bbdfe2ef..fd2b12e063e 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -348,6 +348,8 @@ class LiteLLMRoutes(enum.Enum): # search "/search", "/v1/search", + "/search/{search_tool_name}", + "/v1/search/{search_tool_name}", # OCR "/ocr", "/v1/ocr", @@ -1889,6 +1891,10 @@ class ConfigGeneralSettings(LiteLLMPydanticObjectBase): allowed_routes: Optional[List] = Field( None, description="Proxy API Endpoints you want users to be able to access" ) + reject_clientside_metadata_tags: Optional[bool] = Field( + None, + description="When set to True, rejects requests that contain client-side 'metadata.tags' to prevent users from influencing budgets by sending different tags. Tags can only be inherited from the API key metadata.", + ) enable_public_model_hub: bool = Field( default=False, description="Public model hub for users to see what models they have access to, supported openai params, etc.", diff --git a/litellm/proxy/anthropic_endpoints/endpoints.py b/litellm/proxy/anthropic_endpoints/endpoints.py index c450b655a2c..abea9e6fee1 100644 --- a/litellm/proxy/anthropic_endpoints/endpoints.py +++ b/litellm/proxy/anthropic_endpoints/endpoints.py @@ -154,8 +154,13 @@ async def anthropic_response( # noqa: PLR0915 response = responses[1] + # Extract model_id from request metadata (set by router during routing) + litellm_metadata = data.get("litellm_metadata", {}) or {} + model_info = litellm_metadata.get("model_info", {}) or {} + model_id = model_info.get("id", "") or "" + + # Get other metadata from hidden_params hidden_params = getattr(response, "_hidden_params", {}) or {} - model_id = hidden_params.get("model_id", None) or "" cache_key = hidden_params.get("cache_key", None) or "" api_base = hidden_params.get("api_base", None) or "" response_cost = hidden_params.get("response_cost", None) or "" @@ -216,12 +221,32 @@ async def anthropic_response( # noqa: PLR0915 str(e) ) ) + + # Extract model_id from request metadata (same as success path) + litellm_metadata = data.get("litellm_metadata", {}) or {} + model_info = litellm_metadata.get("model_info", {}) or {} + model_id = model_info.get("id", "") or "" + + # Get headers + headers = ProxyBaseLLMRequestProcessing.get_custom_headers( + user_api_key_dict=user_api_key_dict, + call_id=data.get("litellm_call_id", ""), + model_id=model_id, + version=version, + response_cost=0, + model_region=getattr(user_api_key_dict, "allowed_model_region", ""), + request_data=data, + timeout=getattr(e, "timeout", None), + litellm_logging_obj=None, + ) + error_msg = f"{str(e)}" raise ProxyException( message=getattr(e, "message", error_msg), type=getattr(e, "type", "None"), param=getattr(e, "param", "None"), code=getattr(e, "status_code", 500), + headers=headers, ) diff --git a/litellm/proxy/auth/auth_checks.py b/litellm/proxy/auth/auth_checks.py index 32795a1874d..c9774b18b88 100644 --- a/litellm/proxy/auth/auth_checks.py +++ b/litellm/proxy/auth/auth_checks.py @@ -186,6 +186,24 @@ async def common_checks( raise Exception( f"'user' param not passed in. 'enforce_user_param'={general_settings['enforce_user_param']}" ) + + # 6.1 [OPTIONAL] If 'reject_clientside_metadata_tags' enabled - reject request if it has client-side 'metadata.tags' + if ( + general_settings.get("reject_clientside_metadata_tags", None) is not None + and general_settings["reject_clientside_metadata_tags"] is True + ): + if ( + RouteChecks.is_llm_api_route(route=route) + and "metadata" in request_body + and isinstance(request_body["metadata"], dict) + and "tags" in request_body["metadata"] + ): + raise ProxyException( + message=f"Client-side 'metadata.tags' not allowed in request. 'reject_clientside_metadata_tags'={general_settings['reject_clientside_metadata_tags']}. Tags can only be set via API key metadata.", + type=ProxyErrorTypes.bad_request_error, + param="metadata.tags", + code=status.HTTP_400_BAD_REQUEST, + ) # 7. [OPTIONAL] If 'litellm.max_budget' is set (>0), is proxy under budget if ( litellm.max_budget > 0 diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index 1cdeb3b99ea..d2b04410026 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -330,6 +330,7 @@ class ProxyBaseLLMRequestProcessing: "avideo_remix", "acreate_container", "alist_containers", + "aingest", "aretrieve_container", "adelete_container", "acreate_skill", @@ -344,6 +345,7 @@ class ProxyBaseLLMRequestProcessing: user_max_tokens: Optional[int] = None, user_api_base: Optional[str] = None, model: Optional[str] = None, + llm_router: Optional[Router] = None, ) -> Tuple[dict, LiteLLMLoggingObj]: start_time = datetime.now() # start before calling guardrail hooks @@ -452,6 +454,7 @@ class ProxyBaseLLMRequestProcessing: "avideo_remix", "acreate_container", "alist_containers", + "aingest", "aretrieve_container", "adelete_container", "acreate_skill", @@ -498,6 +501,7 @@ class ProxyBaseLLMRequestProcessing: user_api_base=user_api_base, model=model, route_type=route_type, + llm_router=llm_router, ) tasks = [] @@ -536,6 +540,13 @@ class ProxyBaseLLMRequestProcessing: hidden_params = getattr(response, "_hidden_params", {}) or {} model_id = hidden_params.get("model_id", None) or "" + + # Fallback: extract model_id from litellm_metadata if not in hidden_params + if not model_id: + litellm_metadata = self.data.get("litellm_metadata", {}) or {} + model_info = litellm_metadata.get("model_info", {}) or {} + model_id = model_info.get("id", "") or "" + cache_key = hidden_params.get("cache_key", None) or "" api_base = hidden_params.get("api_base", None) or "" response_cost = hidden_params.get("response_cost", None) or "" @@ -756,11 +767,19 @@ class ProxyBaseLLMRequestProcessing: _litellm_logging_obj: Optional[LiteLLMLoggingObj] = self.data.get( "litellm_logging_obj", None ) + + # Attempt to get model_id from logging object + # + # Note: We check the direct model_info path first (not nested in metadata) because that's where the router sets it. + # The nested metadata path is only a fallback for cases where model_info wasn't set at the top level. + model_id = self.maybe_get_model_id(_litellm_logging_obj) + custom_headers = ProxyBaseLLMRequestProcessing.get_custom_headers( user_api_key_dict=user_api_key_dict, call_id=( _litellm_logging_obj.litellm_call_id if _litellm_logging_obj else None ), + model_id=model_id, version=version, response_cost=0, model_region=getattr(user_api_key_dict, "allowed_model_region", ""), @@ -1073,3 +1092,50 @@ class ProxyBaseLLMRequestProcessing: obj.setdefault("usage", {})["cost"] = cost_val return obj return None + + def maybe_get_model_id(self, _logging_obj: Optional[LiteLLMLoggingObj]) -> Optional[str]: + """ + Get model_id from logging object or request metadata. + + The router sets model_info.id when selecting a deployment. This tries multiple locations + where the ID might be stored depending on the request lifecycle stage. + """ + model_id = None + if _logging_obj: + # 1. Try getting from litellm_params (updated during call) + if ( + hasattr(_logging_obj, "litellm_params") + and _logging_obj.litellm_params + ): + # First check direct model_info path (set by router.py with selected deployment) + model_info = _logging_obj.litellm_params.get("model_info") or {} + model_id = model_info.get("id", None) + + # Fallback to nested metadata path + if not model_id: + metadata = _logging_obj.litellm_params.get("metadata") or {} + model_info = metadata.get("model_info") or {} + model_id = model_info.get("id", None) + + # 2. Fallback to kwargs (initial) + if not model_id: + _kwargs = getattr(_logging_obj, "kwargs", None) + if _kwargs: + litellm_params = _kwargs.get("litellm_params", {}) + # First check direct model_info path + model_info = litellm_params.get("model_info") or {} + model_id = model_info.get("id", None) + + # Fallback to nested metadata path + if not model_id: + metadata = litellm_params.get("metadata") or {} + model_info = metadata.get("model_info") or {} + model_id = model_info.get("id", None) + + # 3. Final fallback to self.data["litellm_metadata"] (for routes like /v1/responses that populate data before error) + if not model_id: + litellm_metadata = self.data.get("litellm_metadata", {}) or {} + model_info = litellm_metadata.get("model_info", {}) or {} + model_id = model_info.get("id", None) + + return model_id diff --git a/litellm/proxy/common_utils/http_parsing_utils.py b/litellm/proxy/common_utils/http_parsing_utils.py index 8b602c525d6..8d8d176e232 100644 --- a/litellm/proxy/common_utils/http_parsing_utils.py +++ b/litellm/proxy/common_utils/http_parsing_utils.py @@ -39,6 +39,8 @@ async def _read_request_body(request: Optional[Request]) -> Dict: if "form" in content_type: parsed_body = dict(await request.form()) + if "metadata" in parsed_body and isinstance(parsed_body["metadata"], str): + parsed_body["metadata"] = json.loads(parsed_body["metadata"]) else: # Read the request body body = await request.body() diff --git a/litellm/proxy/example_config_yaml/reject_clientside_metadata_tags_config.yaml b/litellm/proxy/example_config_yaml/reject_clientside_metadata_tags_config.yaml new file mode 100644 index 00000000000..3c43c3c5374 --- /dev/null +++ b/litellm/proxy/example_config_yaml/reject_clientside_metadata_tags_config.yaml @@ -0,0 +1,14 @@ +model_list: + - model_name: gpt-3.5-turbo + litellm_params: + model: gpt-3.5-turbo + api_key: os.environ/OPENAI_API_KEY + +general_settings: + master_key: sk-1234 + database_url: "postgresql://user:password@localhost:5432/litellm" + + # Reject requests that contain client-side metadata.tags + # This prevents users from influencing budgets by sending different tags + # Tags can only be inherited from the API key metadata + reject_clientside_metadata_tags: true diff --git a/litellm/proxy/guardrails/guardrail_endpoints.py b/litellm/proxy/guardrails/guardrail_endpoints.py index e64fbe9084e..a1cfead9bb2 100644 --- a/litellm/proxy/guardrails/guardrail_endpoints.py +++ b/litellm/proxy/guardrails/guardrail_endpoints.py @@ -31,6 +31,7 @@ from litellm.types.guardrails import ( PiiEntityType, PresidioPresidioConfigModelUserInterface, SupportedGuardrailIntegrations, + ToolPermissionGuardrailConfigModel, ) #### GUARDRAILS ENDPOINTS #### @@ -635,7 +636,9 @@ async def get_guardrail_info(guardrail_id: str): raise HTTPException(status_code=500, detail="Prisma client not initialized") try: - guardrail_definition_location: GUARDRAIL_DEFINITION_LOCATION = GUARDRAIL_DEFINITION_LOCATION.DB + guardrail_definition_location: GUARDRAIL_DEFINITION_LOCATION = ( + GUARDRAIL_DEFINITION_LOCATION.DB + ) result = await GUARDRAIL_REGISTRY.get_guardrail_by_id_from_db( guardrail_id=guardrail_id, prisma_client=prisma_client ) @@ -702,10 +705,12 @@ async def get_guardrail_ui_settings(): # Convert the PII_ENTITY_CATEGORIES_MAP to the format expected by the UI category_maps = [] for category, entities in PII_ENTITY_CATEGORIES_MAP.items(): - category_maps.append({ - "category": category.value, - "entities": [entity.value for entity in entities] - }) + category_maps.append( + { + "category": category.value, + "entities": [entity.value for entity in entities], + } + ) return GuardrailUIAddGuardrailSettings( supported_entities=[entity.value for entity in PiiEntityType], @@ -728,20 +733,20 @@ async def get_guardrail_ui_settings(): async def validate_blocked_words_file(request: Dict[str, str]): """ Validate a blocked_words YAML file content. - + Args: request: Dictionary with 'file_content' key containing the YAML string - + Returns: Dictionary with 'valid' boolean and either 'message'/'errors' depending on result - + Example Request: ```json { "file_content": "blocked_words:\\n - keyword: \\"test\\"\\n action: \\"BLOCK\\"" } ``` - + Example Success Response: ```json { @@ -749,7 +754,7 @@ async def validate_blocked_words_file(request: Dict[str, str]): "message": "Valid YAML file with 2 blocked words" } ``` - + Example Error Response: ```json { @@ -759,56 +764,54 @@ async def validate_blocked_words_file(request: Dict[str, str]): ``` """ import yaml - + try: file_content = request.get("file_content", "") if not file_content: - return { - "valid": False, - "error": "No file content provided" - } - + return {"valid": False, "error": "No file content provided"} + data = yaml.safe_load(file_content) - + if not isinstance(data, dict) or "blocked_words" not in data: return { "valid": False, - "error": "Invalid format: file must contain 'blocked_words' key with a list" + "error": "Invalid format: file must contain 'blocked_words' key with a list", } - + blocked_words_list = data["blocked_words"] if not isinstance(blocked_words_list, list): - return { - "valid": False, - "error": "'blocked_words' must be a list" - } - + return {"valid": False, "error": "'blocked_words' must be a list"} + # Validate each entry errors = [] for idx, word_data in enumerate(blocked_words_list): if not isinstance(word_data, dict): errors.append(f"Entry {idx}: must be an object") continue - + if "keyword" not in word_data: errors.append(f"Entry {idx}: missing 'keyword' field") elif not isinstance(word_data["keyword"], str): errors.append(f"Entry {idx}: 'keyword' must be a string") - + if "action" not in word_data: errors.append(f"Entry {idx}: missing 'action' field") elif word_data["action"] not in ["BLOCK", "MASK"]: - errors.append(f"Entry {idx}: action must be 'BLOCK' or 'MASK', got '{word_data['action']}'") - - if "description" in word_data and not isinstance(word_data["description"], str): + errors.append( + f"Entry {idx}: action must be 'BLOCK' or 'MASK', got '{word_data['action']}'" + ) + + if "description" in word_data and not isinstance( + word_data["description"], str + ): errors.append(f"Entry {idx}: 'description' must be a string") - + if errors: return {"valid": False, "errors": errors} - + return { "valid": True, - "message": f"Valid YAML file with {len(blocked_words_list)} blocked word(s)" + "message": f"Valid YAML file with {len(blocked_words_list)} blocked word(s)", } except yaml.YAMLError as e: return {"valid": False, "error": f"Invalid YAML syntax: {str(e)}"} @@ -931,30 +934,32 @@ def _should_skip_optional_params(field_name: str, field_annotation: Any) -> bool """Check if optional_params field should be skipped (not meaningfully overridden).""" if field_name != "optional_params": return False - + if field_annotation is None: return True - + # Check if the annotation is still a generic TypeVar (not specialized) if isinstance(field_annotation, TypeVar) or ( hasattr(field_annotation, "__origin__") and field_annotation.__origin__ is TypeVar ): return True - + # Also skip if it's a generic type that wasn't specialized if hasattr(field_annotation, "__name__") and field_annotation.__name__ in ( "T", "TypeVar", ): return True - + # Handle Optional[T] where T is still a TypeVar if hasattr(field_annotation, "__args__"): - non_none_args = [arg for arg in field_annotation.__args__ if arg is not type(None)] + non_none_args = [ + arg for arg in field_annotation.__args__ if arg is not type(None) + ] if non_none_args and isinstance(non_none_args[0], TypeVar): return True - + return False @@ -1041,9 +1046,11 @@ def _extract_fields_recursive( for field_name, field in model.model_fields.items(): field_annotation = field.annotation - + # Skip optional_params if it's not meaningfully overridden - if _should_skip_optional_params(field_name=field_name, field_annotation=field_annotation): + if _should_skip_optional_params( + field_name=field_name, field_annotation=field_annotation + ): continue # Handle Optional types and get the actual type @@ -1153,12 +1160,18 @@ async def get_provider_specific_params(): bedrock_fields = _get_fields_from_model(BedrockGuardrailConfigModel) presidio_fields = _get_fields_from_model(PresidioPresidioConfigModelUserInterface) lakera_v2_fields = _get_fields_from_model(LakeraV2GuardrailConfigModel) + tool_permission_fields = _get_fields_from_model(ToolPermissionGuardrailConfigModel) + + tool_permission_fields[ + "ui_friendly_name" + ] = ToolPermissionGuardrailConfigModel.ui_friendly_name() # Return the provider-specific parameters provider_params = { SupportedGuardrailIntegrations.BEDROCK.value: bedrock_fields, SupportedGuardrailIntegrations.PRESIDIO.value: presidio_fields, SupportedGuardrailIntegrations.LAKERA_V2.value: lakera_v2_fields, + SupportedGuardrailIntegrations.TOOL_PERMISSION.value: tool_permission_fields, } ### get the config model for the guardrail - go through the registry and get the config model for the guardrail @@ -1175,6 +1188,7 @@ async def get_provider_specific_params(): return provider_params + @router.post("/guardrails/apply_guardrail", response_model=ApplyGuardrailResponse) @router.post("/apply_guardrail", response_model=ApplyGuardrailResponse) async def apply_guardrail( @@ -1183,11 +1197,11 @@ async def apply_guardrail( ): """ Apply a guardrail to text input and return the processed result. - + This endpoint allows testing guardrails by applying them to custom text inputs. """ from litellm.proxy.utils import handle_exception_on_proxy - + try: active_guardrail: Optional[ CustomGuardrail @@ -1207,4 +1221,3 @@ async def apply_guardrail( return ApplyGuardrailResponse(response_text=response_text) except Exception as e: raise handle_exception_on_proxy(e) - diff --git a/litellm/proxy/guardrails/guardrail_hooks/pillar/pillar.py b/litellm/proxy/guardrails/guardrail_hooks/pillar/pillar.py index 812fc6a5767..74903cc52a4 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/pillar/pillar.py +++ b/litellm/proxy/guardrails/guardrail_hooks/pillar/pillar.py @@ -90,6 +90,10 @@ class PillarGuardrail(CustomGuardrail): fallback_on_error: Action when API errors occur ('allow' or 'block') timeout: Timeout for API calls in seconds **kwargs: Additional arguments passed to parent class + + Note: + LiteLLM virtual key context (user_id, team_id, key_alias, etc.) is always + automatically passed as X-LiteLLM-* headers to enable application/user tracking. """ self.async_handler = get_async_httpx_client(llm_provider=httpxSpecialProvider.GuardrailCallback) self.api_key = api_key or os.environ.get("PILLAR_API_KEY") @@ -222,7 +226,7 @@ class PillarGuardrail(CustomGuardrail): return data verbose_proxy_logger.debug("Pillar Guardrail: Pre-call hook") - result = await self.run_pillar_guardrail(data) + result = await self.run_pillar_guardrail(data, user_api_key_dict) # Add guardrail name to response headers add_guardrail_to_applied_guardrails_header(request_data=data, guardrail_name=self.guardrail_name) @@ -265,7 +269,7 @@ class PillarGuardrail(CustomGuardrail): return data verbose_proxy_logger.debug("Pillar Guardrail: During-call moderation hook") - result = await self.run_pillar_guardrail(data) + result = await self.run_pillar_guardrail(data, user_api_key_dict) # Add guardrail name to response headers add_guardrail_to_applied_guardrails_header(request_data=data, guardrail_name=self.guardrail_name) @@ -315,7 +319,7 @@ class PillarGuardrail(CustomGuardrail): post_call_data["messages"] = data.get("messages", []) + response_messages # Reuse the existing guardrail logic - zero duplication! - await self.run_pillar_guardrail(post_call_data) + await self.run_pillar_guardrail(post_call_data, user_api_key_dict) # Add guardrail name to response headers add_guardrail_to_applied_guardrails_header(request_data=data, guardrail_name=self.guardrail_name) @@ -326,12 +330,13 @@ class PillarGuardrail(CustomGuardrail): # CORE LOGIC METHOD # ========================================================================= - async def run_pillar_guardrail(self, data: dict) -> dict: + async def run_pillar_guardrail(self, data: dict, user_api_key_dict: UserAPIKeyAuth) -> dict: """ Core method to run the Pillar guardrail scan. Args: data: Request data containing messages and metadata + user_api_key_dict: User API key authentication info containing key context Returns: Original data if safe or in monitor mode @@ -345,7 +350,7 @@ class PillarGuardrail(CustomGuardrail): return data try: - headers = self._prepare_headers() + headers = self._prepare_headers(user_api_key_dict) payload = self._prepare_payload(data) response = await self._call_pillar_api( @@ -403,8 +408,16 @@ class PillarGuardrail(CustomGuardrail): }, ) - def _prepare_headers(self) -> Dict[str, str]: - """Prepare headers for the Pillar API request.""" + def _prepare_headers(self, user_api_key_dict: UserAPIKeyAuth) -> Dict[str, str]: + """ + Prepare headers for the Pillar API request. + + Args: + user_api_key_dict: User API key authentication info containing key context + + Returns: + Dictionary of headers to send to Pillar API + """ if not self.api_key: msg = ( "Couldn't get Pillar API key, either set the `PILLAR_API_KEY` in the environment or " @@ -415,7 +428,7 @@ class PillarGuardrail(CustomGuardrail): headers: Dict[str, str] = { "Authorization": f"Bearer {self.api_key}", "Content-Type": "application/json", - } + } # Add Pillar-specific headers based on configuration self._set_bool_header(headers, "plr_scanners", self.include_scanners) @@ -423,6 +436,20 @@ class PillarGuardrail(CustomGuardrail): self._set_bool_header(headers, "plr_async", self.async_mode) self._set_bool_header(headers, "plr_persist", self.persist_session) + # Always add LiteLLM virtual key context headers (metadata excluded for security) + context_mapping = { + "X-LiteLLM-Key-Name": user_api_key_dict.key_name, + "X-LiteLLM-Key-Alias": user_api_key_dict.key_alias, + "X-LiteLLM-User-Id": user_api_key_dict.user_id, + "X-LiteLLM-User-Email": user_api_key_dict.user_email, + "X-LiteLLM-Team-Id": user_api_key_dict.team_id, + "X-LiteLLM-Team-Name": user_api_key_dict.team_alias, + "X-LiteLLM-Org-Id": user_api_key_dict.org_id, + } + for header_name, value in context_mapping.items(): + if value: + headers[header_name] = str(value) + return headers def _set_bool_header(self, headers: Dict[str, str], header_name: str, value: Optional[bool]) -> None: @@ -517,6 +544,14 @@ class PillarGuardrail(CustomGuardrail): """ Prepare the payload for the Pillar API request following the /api/v1/protect contract. + This method supports multi-modal content (images, files, audio, video, etc.) as messages + are passed through without modification. The messages array can contain any OpenAI-compatible + message structure including: + - Text content (string) + - Multi-modal content blocks (image_url, image_file, audio, video, document, file) + - Attachments + - Tool calls + Args: data: Request data diff --git a/litellm/proxy/guardrails/guardrail_hooks/prompt_security/prompt_security.py b/litellm/proxy/guardrails/guardrail_hooks/prompt_security/prompt_security.py index daee50f30cc..23b9da4714c 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/prompt_security/prompt_security.py +++ b/litellm/proxy/guardrails/guardrail_hooks/prompt_security/prompt_security.py @@ -1,13 +1,18 @@ -import os -import re import asyncio import base64 +import os +import re from typing import TYPE_CHECKING, Any, AsyncGenerator, Optional, Type, Union + from fastapi import HTTPException + from litellm import DualCache from litellm._logging import verbose_proxy_logger from litellm.integrations.custom_guardrail import CustomGuardrail -from litellm.llms.custom_httpx.http_handler import get_async_httpx_client, httpxSpecialProvider +from litellm.llms.custom_httpx.http_handler import ( + get_async_httpx_client, + httpxSpecialProvider, +) from litellm.proxy._types import UserAPIKeyAuth from litellm.types.utils import ( Choices, @@ -15,7 +20,7 @@ from litellm.types.utils import ( EmbeddingResponse, ImageResponse, ModelResponse, - ModelResponseStream + ModelResponseStream, ) if TYPE_CHECKING: @@ -267,8 +272,10 @@ class PromptSecurityGuardrail(CustomGuardrail): content = msg.get('content', '') # Handle both string and list content types if isinstance(content, str): - if content.startswith('### '): return False - if '"follow_ups": [' in content: return False + if content.startswith('### '): + return False + if '"follow_ups": [' in content: + return False return True messages = list(filter(lambda msg: good_msg(msg), messages)) diff --git a/litellm/proxy/guardrails/guardrail_hooks/tool_permission.py b/litellm/proxy/guardrails/guardrail_hooks/tool_permission.py index 95d8f894dc2..02e06acbda4 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/tool_permission.py +++ b/litellm/proxy/guardrails/guardrail_hooks/tool_permission.py @@ -23,6 +23,7 @@ from litellm.types.proxy.guardrails.guardrail_hooks.tool_permission import ( ToolResult, ) from litellm.types.utils import ( + CallTypesLiteral, ChatCompletionMessageToolCall, Choices, LLMResponseTypes, @@ -62,8 +63,11 @@ class ToolPermissionGuardrail(CustomGuardrail): self.rules: List[ToolPermissionRule] = [] self._compiled_rule_patterns: Dict[str, Dict[str, re.Pattern]] = {} if rules: - for rule_dict in rules: - rule = ToolPermissionRule(**rule_dict) + for rule_item in rules: + if isinstance(rule_item, ToolPermissionRule): + rule = rule_item + else: + rule = ToolPermissionRule(**rule_item) self.rules.append(rule) if rule.allowed_param_patterns: @@ -88,6 +92,14 @@ class ToolPermissionGuardrail(CustomGuardrail): self.default_action, ) + @staticmethod + def get_config_model(): + from litellm.types.proxy.guardrails.guardrail_hooks.tool_permission import ( + ToolPermissionGuardrailConfigModel, + ) + + return ToolPermissionGuardrailConfigModel + def _matches_pattern(self, tool_name: str, pattern: str) -> bool: """ Check if a tool name matches a pattern @@ -191,16 +203,21 @@ class ToolPermissionGuardrail(CustomGuardrail): return {} def _collect_argument_paths( - self, value: Any, current_path: str, collected: Dict[str, List[Any]] + self, value: Any, current_path: str, collected: Dict[str, List[Any]], depth: int = 0 ) -> None: + from litellm.constants import DEFAULT_MAX_RECURSE_DEPTH + + if depth > DEFAULT_MAX_RECURSE_DEPTH: + return + if isinstance(value, dict): for key, sub_value in value.items(): next_path = f"{current_path}.{key}" if current_path else key - self._collect_argument_paths(sub_value, next_path, collected) + self._collect_argument_paths(sub_value, next_path, collected, depth + 1) elif isinstance(value, list): list_path = f"{current_path}[]" if current_path else "[]" for item in value: - self._collect_argument_paths(item, list_path, collected) + self._collect_argument_paths(item, list_path, collected, depth + 1) else: if not current_path: return @@ -426,18 +443,7 @@ class ToolPermissionGuardrail(CustomGuardrail): user_api_key_dict: UserAPIKeyAuth, cache: DualCache, data: dict, - call_type: Literal[ - "completion", - "text_completion", - "embeddings", - "image_generation", - "moderation", - "audio_transcription", - "pass_through_endpoint", - "rerank", - "mcp_call", - "anthropic_messages", - ], + call_type: CallTypesLiteral, ) -> Union[Exception, str, dict, None]: """ """ verbose_proxy_logger.debug("Tool Permission Guardrail Pre-Call Hook") diff --git a/litellm/proxy/guardrails/guardrail_initializers.py b/litellm/proxy/guardrails/guardrail_initializers.py index f2083e9c67e..9bb965ef14e 100644 --- a/litellm/proxy/guardrails/guardrail_initializers.py +++ b/litellm/proxy/guardrails/guardrail_initializers.py @@ -1,4 +1,6 @@ # litellm/proxy/guardrails/guardrail_initializers.py +from typing import Any, Dict, List, Optional + import litellm from litellm.proxy._types import CommonProxyErrors from litellm.types.guardrails import * @@ -128,10 +130,19 @@ def initialize_tool_permission(litellm_params: LitellmParams, guardrail: Guardra ToolPermissionGuardrail, ) + rules: Optional[List[Dict[str, Any]]] = None + if litellm_params.rules: + rules = [] + for rule in litellm_params.rules: + if hasattr(rule, "model_dump"): + rules.append(rule.model_dump()) + else: + rules.append(dict(rule)) + _tool_permission_callback = ToolPermissionGuardrail( guardrail_name=guardrail.get("guardrail_name", ""), event_hook=litellm_params.mode, - rules=litellm_params.rules, + rules=rules, default_action=getattr(litellm_params, "default_action", "deny"), on_disallowed_action=getattr(litellm_params, "on_disallowed_action", "block"), default_on=litellm_params.default_on, diff --git a/litellm/proxy/image_endpoints/endpoints.py b/litellm/proxy/image_endpoints/endpoints.py index 16aa8f16571..a1453e10dbf 100644 --- a/litellm/proxy/image_endpoints/endpoints.py +++ b/litellm/proxy/image_endpoints/endpoints.py @@ -215,9 +215,11 @@ async def image_generation( async def image_edit_api( request: Request, fastapi_response: Response, - image: List[UploadFile] = File(...), - mask: Optional[List[UploadFile]] = File(None), user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), + image: Optional[List[UploadFile]] = File(None), + image_array: Optional[List[UploadFile]] = File(None, alias="image[]"), + mask: Optional[List[UploadFile]] = File(None), + mask_array: Optional[List[UploadFile]] = File(None, alias="mask[]"), model: Optional[str] = None, ): """ @@ -233,6 +235,18 @@ async def image_edit_api( -F 'prompt=Create a studio ghibli image of this' ``` """ + if image is not None and image_array is not None: + raise HTTPException(status_code=422, detail="Cannot specify both 'image' and 'image[]'") + if mask is not None and mask_array is not None: + raise HTTPException(status_code=422, detail="Cannot specify both 'mask' and 'mask[]'") + if image is None and image_array is not None: + image = image_array + if mask is None and mask_array is not None: + mask = mask_array + + if image is None: + raise HTTPException(status_code=422, detail="Field required: image") + from litellm.proxy.proxy_server import ( _read_request_body, general_settings, diff --git a/litellm/proxy/management_endpoints/team_endpoints.py b/litellm/proxy/management_endpoints/team_endpoints.py index 3a66f95e812..6d4faae5fd8 100644 --- a/litellm/proxy/management_endpoints/team_endpoints.py +++ b/litellm/proxy/management_endpoints/team_endpoints.py @@ -1878,6 +1878,15 @@ async def team_member_delete( where={"team_id": data.team_id, "user_id": _uid} ) + ## DELETE KEYS CREATED BY USER FOR THIS TEAM + if user_ids_to_delete: + await prisma_client.db.litellm_verificationtoken.delete_many( + where={ + "user_id": {"in": list(user_ids_to_delete)}, + "team_id": data.team_id, + } + ) + return existing_team_row diff --git a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/cohere_passthrough_logging_handler.py b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/cohere_passthrough_logging_handler.py index a8228de6e01..743f4e4f96a 100644 --- a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/cohere_passthrough_logging_handler.py +++ b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/cohere_passthrough_logging_handler.py @@ -1,14 +1,30 @@ +from datetime import datetime from typing import List, Optional, Union +import httpx + +import litellm from litellm import stream_chunk_builder from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj +from litellm.litellm_core_utils.litellm_logging import ( + get_standard_logging_object_payload, +) from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper from litellm.llms.base_llm.chat.transformation import BaseConfig from litellm.llms.cohere.chat.v2_transformation import CohereV2ChatConfig from litellm.llms.cohere.common_utils import ( ModelResponseIterator as CohereModelResponseIterator, ) -from litellm.types.utils import LlmProviders, ModelResponse, TextCompletionResponse +from litellm.llms.cohere.embed.v1_transformation import CohereEmbeddingConfig +from litellm.proxy._types import PassThroughEndpointLoggingTypedDict +from litellm.types.passthrough_endpoints.pass_through_endpoints import ( + PassthroughStandardLoggingPayload, +) +from litellm.types.utils import ( + LlmProviders, + ModelResponse, + TextCompletionResponse, +) from .base_passthrough_logging_handler import BasePassthroughLoggingHandler @@ -54,3 +70,123 @@ class CoherePassthroughLoggingHandler(BasePassthroughLoggingHandler): break complete_streaming_response = stream_chunk_builder(chunks=all_openai_chunks) return complete_streaming_response + + def cohere_passthrough_handler( # noqa: PLR0915 + self, + httpx_response: httpx.Response, + response_body: dict, + logging_obj: LiteLLMLoggingObj, + url_route: str, + result: str, + start_time: datetime, + end_time: datetime, + cache_hit: bool, + request_body: dict, + **kwargs, + ) -> PassThroughEndpointLoggingTypedDict: + """ + Handle Cohere passthrough logging with route detection and cost tracking. + """ + # Check if this is an embed endpoint + if "/v1/embed" in url_route: + model = request_body.get("model", response_body.get("model", "")) + try: + cohere_embed_config = CohereEmbeddingConfig() + litellm_model_response = litellm.EmbeddingResponse() + handler_instance = CoherePassthroughLoggingHandler() + + input_texts = request_body.get("texts", []) + if not input_texts: + input_texts = request_body.get("input", []) + + # Transform the response + litellm_model_response = cohere_embed_config._transform_response( + response=httpx_response, + api_key="", + logging_obj=logging_obj, + data=request_body, + model_response=litellm_model_response, + model=model, + encoding=litellm.encoding, + input=input_texts, + ) + + # Calculate cost using LiteLLM's cost calculator + response_cost = litellm.completion_cost( + completion_response=litellm_model_response, + model=model, + custom_llm_provider="cohere", + call_type="aembedding", + ) + + # Set the calculated cost in _hidden_params to prevent recalculation + if not hasattr(litellm_model_response, "_hidden_params"): + litellm_model_response._hidden_params = {} + litellm_model_response._hidden_params["response_cost"] = response_cost + + kwargs["response_cost"] = response_cost + kwargs["model"] = model + kwargs["custom_llm_provider"] = "cohere" + + # Extract user information for tracking + passthrough_logging_payload: Optional[ + PassthroughStandardLoggingPayload + ] = kwargs.get("passthrough_logging_payload") + if passthrough_logging_payload: + user = handler_instance._get_user_from_metadata( + passthrough_logging_payload=passthrough_logging_payload, + ) + if user: + kwargs.setdefault("litellm_params", {}) + kwargs["litellm_params"].update( + {"proxy_server_request": {"body": {"user": user}}} + ) + + # Create standard logging object + if litellm_model_response is not None: + get_standard_logging_object_payload( + kwargs=kwargs, + init_response_obj=litellm_model_response, + start_time=start_time, + end_time=end_time, + logging_obj=logging_obj, + status="success", + ) + + # Update logging object with cost information + logging_obj.model_call_details["model"] = model + logging_obj.model_call_details["custom_llm_provider"] = "cohere" + logging_obj.model_call_details["response_cost"] = response_cost + + return { + "result": litellm_model_response, + "kwargs": kwargs, + } + except Exception: + # For other routes (e.g., /v2/chat), fall back to chat handler + return super().passthrough_chat_handler( + httpx_response=httpx_response, + response_body=response_body, + logging_obj=logging_obj, + url_route=url_route, + result=result, + start_time=start_time, + end_time=end_time, + cache_hit=cache_hit, + request_body=request_body, + **kwargs, + ) + + # For non-embed routes (e.g., /v2/chat), fall back to chat handler + return super().passthrough_chat_handler( + httpx_response=httpx_response, + response_body=response_body, + logging_obj=logging_obj, + url_route=url_route, + result=result, + start_time=start_time, + end_time=end_time, + cache_hit=cache_hit, + request_body=request_body, + **kwargs, + ) diff --git a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/vertex_passthrough_logging_handler.py b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/vertex_passthrough_logging_handler.py index dae2a7f081a..407cbd2f3d0 100644 --- a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/vertex_passthrough_logging_handler.py +++ b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/vertex_passthrough_logging_handler.py @@ -148,6 +148,8 @@ class VertexPassthroughLoggingHandler: logging_obj.model = model logging_obj.model_call_details["model"] = logging_obj.model + logging_obj.model_call_details["custom_llm_provider"] = "vertex_ai" + logging_obj.custom_llm_provider = "vertex_ai" response_cost = litellm.completion_cost( completion_response=litellm_prediction_response, model=model, @@ -156,6 +158,7 @@ class VertexPassthroughLoggingHandler: kwargs["response_cost"] = response_cost kwargs["model"] = model + kwargs["custom_llm_provider"] = "vertex_ai" logging_obj.model_call_details["response_cost"] = response_cost return { diff --git a/litellm/proxy/pass_through_endpoints/success_handler.py b/litellm/proxy/pass_through_endpoints/success_handler.py index 6a0cfd44438..cc50d2c2d8e 100644 --- a/litellm/proxy/pass_through_endpoints/success_handler.py +++ b/litellm/proxy/pass_through_endpoints/success_handler.py @@ -48,7 +48,7 @@ class PassThroughEndpointLogging: self.TRACKED_ANTHROPIC_ROUTES = ["/messages"] # Cohere - self.TRACKED_COHERE_ROUTES = ["/v2/chat"] + self.TRACKED_COHERE_ROUTES = ["/v2/chat", "/v1/embed"] self.assemblyai_passthrough_logging_handler = ( AssemblyAIPassthroughLoggingHandler() ) @@ -177,7 +177,7 @@ class PassThroughEndpointLogging: kwargs = anthropic_passthrough_logging_handler_result["kwargs"] elif self.is_cohere_route(url_route): cohere_passthrough_logging_handler_result = ( - cohere_passthrough_logging_handler.passthrough_chat_handler( + cohere_passthrough_logging_handler.cohere_passthrough_handler( httpx_response=httpx_response, response_body=response_body or {}, logging_obj=logging_obj, diff --git a/litellm/proxy/proxy_config.yaml b/litellm/proxy/proxy_config.yaml index 014bcdc1670..26e867dc33e 100644 --- a/litellm/proxy/proxy_config.yaml +++ b/litellm/proxy/proxy_config.yaml @@ -1,22 +1,7 @@ model_list: - - model_name: aws/anthropic/bedrock-claude-3-5-sonnet-v1 + - model_name: qwen-25vl-72b litellm_params: - model: bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0 - aws_region_name: us-east-1 - custom_llm_provider: bedrock - - model_name: aws/anthropic/bedrock-claude-3-5-sonnet-v1 - litellm_params: - model: bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0 - aws_region_name: us-west-2 - custom_llm_provider: bedrock - - model_name: bedrock/* - litellm_params: - model: bedrock/* - custom_llm_provider: bedrock - aws_region_name: us-west-2 - - model_name: runwayml/* - litellm_params: - model: runwayml/* + model: bedrock/openai/arn:aws:bedrock:us-east-1:046319184608:imported-model/0m2lasirsp6z diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 0c0ced82ba1..67e874fc55d 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -362,6 +362,7 @@ from litellm.proxy.pass_through_endpoints.pass_through_endpoints import ( ) from litellm.proxy.prompts.prompt_endpoints import router as prompts_router from litellm.proxy.public_endpoints import router as public_endpoints_router +from litellm.proxy.rag_endpoints.endpoints import router as rag_router from litellm.proxy.rerank_endpoints.endpoints import router as rerank_router from litellm.proxy.response_api_endpoints.endpoints import router as response_router from litellm.proxy.route_llm_request import route_request @@ -1240,7 +1241,7 @@ def cost_tracking(): global prisma_client if prisma_client is not None: litellm.logging_callback_manager.add_litellm_callback(_ProxyDBLogger()) - + litellm.logging_callback_manager.add_litellm_async_success_callback(_ProxyDBLogger()) async def update_cache( # noqa: PLR0915 token: Optional[str], @@ -2010,6 +2011,16 @@ class ProxyConfig: ) print(f"\033[32m {search_tool_name} ({search_provider})\033[0m") # noqa + # Handle os.environ/ variables in litellm_params + litellm_params = search_tool.get("litellm_params", {}) + if litellm_params: + for k, v in litellm_params.items(): + if isinstance(v, str) and v.startswith("os.environ/"): + _v = v.replace("os.environ/", "") + v = get_secret(_v) + litellm_params[k] = v + search_tool["litellm_params"] = litellm_params + # Cast to SearchToolTypedDict for type safety try: search_tool_typed: SearchToolTypedDict = SearchToolTypedDict(**search_tool) # type: ignore @@ -3761,6 +3772,7 @@ class ProxyConfig: async def _init_search_tools_in_db(self, prisma_client: PrismaClient): """ Initialize search tools from database into the router on startup. + Only updates router if there are tools in the database, otherwise preserves config-loaded tools. """ global llm_router @@ -3778,17 +3790,24 @@ class ProxyConfig: f"Loading {len(search_tools)} search tool(s) from database into router" ) - if llm_router is not None: - # Add search tools to the router - await SearchAPIRouter.update_router_search_tools( - router_instance=llm_router, search_tools=search_tools - ) - verbose_proxy_logger.info( - f"Successfully loaded {len(search_tools)} search tool(s) into router" - ) + # Only update router if there are tools in the database + # This prevents overwriting config-loaded tools with an empty list + if len(search_tools) > 0: + if llm_router is not None: + # Add search tools to the router + await SearchAPIRouter.update_router_search_tools( + router_instance=llm_router, search_tools=search_tools + ) + verbose_proxy_logger.info( + f"Successfully loaded {len(search_tools)} search tool(s) into router" + ) + else: + verbose_proxy_logger.debug( + "Router not initialized yet, search tools will be added when router is created" + ) else: verbose_proxy_logger.debug( - "Router not initialized yet, search tools will be added when router is created" + "No search tools found in database, keeping config-loaded search tools (if any)" ) except Exception as e: @@ -5465,6 +5484,7 @@ async def audio_transcriptions( file_object = io.BytesIO(file_content) file_object.name = file.filename data["file"] = file_object + try: ### CALL HOOKS ### - modify incoming data / reject request before calling the model data = await proxy_logging_obj.pre_call_hook( @@ -5482,7 +5502,7 @@ async def audio_transcriptions( ) response = await llm_call except Exception as e: - raise HTTPException(status_code=500, detail=str(e)) + raise e finally: file_object.close() # close the file read in by io library @@ -10138,6 +10158,7 @@ app.include_router(batches_router) app.include_router(public_endpoints_router) app.include_router(rerank_router) app.include_router(ocr_router) +app.include_router(rag_router) app.include_router(video_router) app.include_router(container_router) app.include_router(search_router) diff --git a/litellm/proxy/rag_endpoints/__init__.py b/litellm/proxy/rag_endpoints/__init__.py new file mode 100644 index 00000000000..4586e4ec72a --- /dev/null +++ b/litellm/proxy/rag_endpoints/__init__.py @@ -0,0 +1,6 @@ +"""RAG Endpoints for LiteLLM Proxy.""" + +from litellm.proxy.rag_endpoints.endpoints import router + +__all__ = ["router"] + diff --git a/litellm/proxy/rag_endpoints/endpoints.py b/litellm/proxy/rag_endpoints/endpoints.py new file mode 100644 index 00000000000..c0b5103f47f --- /dev/null +++ b/litellm/proxy/rag_endpoints/endpoints.py @@ -0,0 +1,200 @@ +""" +RAG Ingest Endpoints for LiteLLM Proxy. + +Provides an all-in-one API for document ingestion: +Upload -> (OCR) -> Chunk -> Embed -> Vector Store +""" + +import base64 +from typing import Any, Dict, Optional, Tuple + +import orjson +from fastapi import APIRouter, Depends, HTTPException, Request, Response +from fastapi.responses import ORJSONResponse + +import litellm +from litellm._logging import verbose_proxy_logger +from litellm.proxy._types import * +from litellm.proxy.auth.user_api_key_auth import UserAPIKeyAuth, user_api_key_auth +from litellm.proxy.common_utils.http_parsing_utils import ( + _read_request_body, + _safe_get_request_headers, + get_form_data, +) + +router = APIRouter() + + +async def parse_rag_ingest_request( + request: Request, +) -> Tuple[Dict[str, Any], Optional[Tuple[str, bytes, str]], Optional[str], Optional[str]]: + """ + Parse RAG ingest request. + + Supports: + - Form: file + request JSON in form field + - JSON body for URL-based ingestion + + Returns: + Tuple of (ingest_options, file_data, file_url, file_id) + """ + headers = _safe_get_request_headers(request) + content_type = headers.get("content-type", "") + + file_data = None + file_url = None + file_id = None + ingest_options: Dict[str, Any] = {} + + if "multipart/form-data" in content_type: + # Form upload + form_data = await get_form_data(request) + + # Get file + file_obj = form_data.get("file") + if file_obj is not None and hasattr(file_obj, "read"): + file_content = await file_obj.read() + file_data = (file_obj.filename, file_content, file_obj.content_type) + + # Parse JSON from 'request' form field (contains full request body as JSON) + request_json_str = form_data.get("request") + if request_json_str: + request_data = orjson.loads(request_json_str) + ingest_options = request_data.get("ingest_options", {}) + file_url = request_data.get("file_url") + file_id = request_data.get("file_id") + + else: + # JSON body + data = await _read_request_body(request) + ingest_options = data.get("ingest_options", {}) + file_url = data.get("file_url") + file_id = data.get("file_id") + + # Handle base64-encoded file in JSON body + file_obj = data.get("file") + if file_obj and isinstance(file_obj, dict): + filename = file_obj.get("filename") + content_b64 = file_obj.get("content") + content_type = file_obj.get("content_type", "application/octet-stream") + + if filename and content_b64: + try: + file_content = base64.b64decode(content_b64) + file_data = (filename, file_content, content_type) + except Exception as e: + raise HTTPException( + status_code=400, + detail={"error": f"Invalid base64 content: {e}"}, + ) + + # Validate + if file_data is None and file_url is None and file_id is None: + raise HTTPException( + status_code=400, + detail={"error": "Must provide file, file_url, or file_id"}, + ) + + if "vector_store" not in ingest_options: + raise HTTPException( + status_code=400, + detail={"error": "ingest_options must contain 'vector_store' configuration"}, + ) + + return ingest_options, file_data, file_url, file_id + + +@router.post( + "/v1/rag/ingest", + dependencies=[Depends(user_api_key_auth)], + response_class=ORJSONResponse, + tags=["rag"], +) +@router.post( + "/rag/ingest", + dependencies=[Depends(user_api_key_auth)], + response_class=ORJSONResponse, + tags=["rag"], +) +async def rag_ingest( + request: Request, + fastapi_response: Response, + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), +): + """ + RAG Ingest endpoint - all-in-one document ingestion pipeline. + + Supports form upload (for files) or JSON body (for URLs). + + ## Form upload (for files): + ```bash + curl -X POST "http://localhost:4000/v1/rag/ingest" \\ + -H "Authorization: Bearer sk-1234" \\ + -F file="@document.pdf" \\ + -F 'ingest_options={"vector_store": {"custom_llm_provider": "openai"}}' + ``` + + ## JSON body (for URLs): + ```bash + curl -X POST "http://localhost:4000/v1/rag/ingest" \\ + -H "Authorization: Bearer sk-1234" \\ + -H "Content-Type: application/json" \\ + -d '{ + "file_url": "https://example.com/document.pdf", + "ingest_options": {"vector_store": {"custom_llm_provider": "openai"}} + }' + ``` + + ## Bedrock: + ```bash + curl -X POST "http://localhost:4000/v1/rag/ingest" \\ + -H "Authorization: Bearer sk-1234" \\ + -F file="@document.pdf" \\ + -F 'ingest_options={"vector_store": {"custom_llm_provider": "bedrock"}}' + ``` + """ + from litellm.proxy.proxy_server import ( + add_litellm_data_to_request, + general_settings, + llm_router, + proxy_config, + version, + ) + + try: + # Parse request + ingest_options, file_data, file_url, file_id = await parse_rag_ingest_request(request) + + # Add litellm data + request_data: Dict[str, Any] = {} + request_data = await add_litellm_data_to_request( + data=request_data, + request=request, + general_settings=general_settings, + user_api_key_dict=user_api_key_dict, + version=version, + proxy_config=proxy_config, + ) + + verbose_proxy_logger.debug(f"RAG Ingest - options: {ingest_options}") + + # Call ingest + response = await litellm.aingest( + ingest_options=ingest_options, + file_data=file_data, + file_url=file_url, + file_id=file_id, + router=llm_router, + **request_data, + ) + + return response + + except HTTPException: + raise + except Exception as e: + verbose_proxy_logger.exception(f"RAG Ingest failed: {e}") + raise HTTPException( + status_code=500, + detail={"error": str(e)}, + ) diff --git a/litellm/proxy/route_llm_request.py b/litellm/proxy/route_llm_request.py index 6b7324a60ed..79e316d4702 100644 --- a/litellm/proxy/route_llm_request.py +++ b/litellm/proxy/route_llm_request.py @@ -40,6 +40,7 @@ ROUTE_ENDPOINT_MAPPING = { "alist_skills": "/skills", "aget_skill": "/skills/{skill_id}", "adelete_skill": "/skills/{skill_id}", + "aingest": "/rag/ingest", } @@ -134,6 +135,7 @@ async def route_request( "alist_skills", "aget_skill", "adelete_skill", + "aingest", ], ): """ @@ -190,6 +192,7 @@ async def route_request( "alist_skills", "aget_skill", "adelete_skill", + "aingest", ] and (data.get("model") is None or data.get("model") == ""): # These endpoints don't need a model, use custom_llm_provider directly return getattr(litellm, f"{route_type}")(**data) diff --git a/litellm/proxy/search_endpoints/endpoints.py b/litellm/proxy/search_endpoints/endpoints.py index da5388f389d..8cfc8bb5106 100644 --- a/litellm/proxy/search_endpoints/endpoints.py +++ b/litellm/proxy/search_endpoints/endpoints.py @@ -131,6 +131,27 @@ async def search( if search_tool_name is not None: data["search_tool_name"] = search_tool_name + if "search_tool_name" in data and data["search_tool_name"]: + data["model"] = data["search_tool_name"] + + if llm_router is not None and hasattr(llm_router, "search_tools"): + search_tool_name_value = data["search_tool_name"] + matching_tools = [ + tool for tool in llm_router.search_tools + if tool.get("search_tool_name") == search_tool_name_value + ] + + if matching_tools: + search_tool = matching_tools[0] + search_provider = search_tool.get("litellm_params", {}).get("search_provider") + + if search_provider: + data["custom_llm_provider"] = search_provider + + if "metadata" not in data: + data["metadata"] = {} + data["metadata"]["model_group"] = search_tool_name_value + # Process request using ProxyBaseLLMRequestProcessing processor = ProxyBaseLLMRequestProcessing(data=data) try: diff --git a/litellm/proxy/spend_tracking/spend_management_endpoints.py b/litellm/proxy/spend_tracking/spend_management_endpoints.py index a167f564fd9..5dfedcc0b87 100644 --- a/litellm/proxy/spend_tracking/spend_management_endpoints.py +++ b/litellm/proxy/spend_tracking/spend_management_endpoints.py @@ -1427,7 +1427,7 @@ async def _get_spend_report_for_time_range( LEFT JOIN "LiteLLM_TeamTable" t ON s.team_id = t.team_id WHERE - s."startTime"::DATE >= $1::date AND s."startTime"::DATE <= $2::date + s."startTime" >= $1::date AND s."startTime" < ($2::date + INTERVAL '1 day') GROUP BY t.team_alias ORDER BY @@ -1441,7 +1441,7 @@ async def _get_spend_report_for_time_range( jsonb_array_elements_text(request_tags) AS individual_request_tag, SUM(spend) AS total_spend FROM "LiteLLM_SpendLogs" - WHERE "startTime"::DATE >= $1::date AND "startTime"::DATE <= $2::date + WHERE "startTime" >= $1::date AND "startTime" < ($2::date + INTERVAL '1 day') GROUP BY individual_request_tag ORDER BY total_spend DESC; """ diff --git a/litellm/proxy/spend_tracking/spend_tracking_utils.py b/litellm/proxy/spend_tracking/spend_tracking_utils.py index 08e56248853..59f712e5b6f 100644 --- a/litellm/proxy/spend_tracking/spend_tracking_utils.py +++ b/litellm/proxy/spend_tracking/spend_tracking_utils.py @@ -228,7 +228,8 @@ def get_logging_payload( # noqa: PLR0915 if call_type in ["ocr", "aocr"]: usage = _extract_usage_for_ocr_call(response_obj, response_obj_dict) else: - usage = cast(dict, response_obj).get("usage", None) or {} + # Use response_obj_dict instead of response_obj to avoid calling .get() on Pydantic models + usage = response_obj_dict.get("usage", None) or {} if isinstance(usage, litellm.Usage): usage = dict(usage) diff --git a/litellm/proxy/vector_store_endpoints/endpoints.py b/litellm/proxy/vector_store_endpoints/endpoints.py index 509869d9bb0..6e22d66c35d 100644 --- a/litellm/proxy/vector_store_endpoints/endpoints.py +++ b/litellm/proxy/vector_store_endpoints/endpoints.py @@ -50,11 +50,11 @@ def _update_request_data_with_litellm_managed_vector_store_registry( @router.post( - "/v1/vector_stores/{vector_store_id}/search", + "/v1/vector_stores/{vector_store_id:path}/search", dependencies=[Depends(user_api_key_auth)], ) @router.post( - "/vector_stores/{vector_store_id}/search", dependencies=[Depends(user_api_key_auth)] + "/vector_stores/{vector_store_id:path}/search", dependencies=[Depends(user_api_key_auth)] ) async def vector_store_search( request: Request, diff --git a/litellm/rag/__init__.py b/litellm/rag/__init__.py new file mode 100644 index 00000000000..f87e72f0c17 --- /dev/null +++ b/litellm/rag/__init__.py @@ -0,0 +1,22 @@ +""" +LiteLLM RAG (Retrieval Augmented Generation) Module. + +Provides an all-in-one API for document ingestion: +Upload -> (OCR) -> Chunk -> Embed -> Vector Store +""" + +from litellm.rag.main import aingest, ingest + +__all__ = ["ingest", "aingest"] + + +# Expose at litellm.rag level for convenience +async def arag_ingest(*args, **kwargs): + """Alias for aingest.""" + return await aingest(*args, **kwargs) + + +def rag_ingest(*args, **kwargs): + """Alias for ingest.""" + return ingest(*args, **kwargs) + diff --git a/litellm/rag/ingestion/__init__.py b/litellm/rag/ingestion/__init__.py new file mode 100644 index 00000000000..e34a715085c --- /dev/null +++ b/litellm/rag/ingestion/__init__.py @@ -0,0 +1,14 @@ +""" +RAG Ingestion classes for different providers. +""" + +from litellm.rag.ingestion.base_ingestion import BaseRAGIngestion +from litellm.rag.ingestion.bedrock_ingestion import BedrockRAGIngestion +from litellm.rag.ingestion.openai_ingestion import OpenAIRAGIngestion + +__all__ = [ + "BaseRAGIngestion", + "BedrockRAGIngestion", + "OpenAIRAGIngestion", +] + diff --git a/litellm/rag/ingestion/base_ingestion.py b/litellm/rag/ingestion/base_ingestion.py new file mode 100644 index 00000000000..3d2e72f8538 --- /dev/null +++ b/litellm/rag/ingestion/base_ingestion.py @@ -0,0 +1,319 @@ +""" +Base RAG Ingestion class. + +Provides abstract methods for: +- OCR +- Chunking +- Embedding +- Vector Store operations + +Providers can inherit and override methods as needed. +""" + +from __future__ import annotations + +import base64 +from abc import ABC, abstractmethod +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, cast + +import httpx + +import litellm +from litellm._logging import verbose_logger +from litellm._uuid import uuid4 +from litellm.constants import DEFAULT_CHUNK_OVERLAP, DEFAULT_CHUNK_SIZE +from litellm.rag.text_splitters import RecursiveCharacterTextSplitter +from litellm.types.rag import RAGIngestOptions, RAGIngestResponse + +if TYPE_CHECKING: + from litellm import Router + + +class BaseRAGIngestion(ABC): + """ + Base class for RAG ingestion. + + Providers should inherit from this class and override methods as needed. + For example, OpenAI handles embedding internally when attaching files to + vector stores, so it overrides the embedding step to be a no-op. + """ + + def __init__( + self, + ingest_options: RAGIngestOptions, + router: Optional["Router"] = None, + ): + self.ingest_options = ingest_options + self.router = router + self.ingest_id = f"ingest_{uuid4()}" + + # Extract configs from options + self.ocr_config = ingest_options.get("ocr") + self.chunking_strategy: Dict[str, Any] = cast( + Dict[str, Any], + ingest_options.get("chunking_strategy") or {"type": "auto"}, + ) + self.embedding_config = ingest_options.get("embedding") + self.vector_store_config: Dict[str, Any] = cast( + Dict[str, Any], ingest_options.get("vector_store") or {} + ) + self.ingest_name = ingest_options.get("name") + + @property + def custom_llm_provider(self) -> str: + """Get the vector store provider.""" + return self.vector_store_config.get("custom_llm_provider", "openai") + + async def upload( + self, + file_data: Optional[Tuple[str, bytes, str]] = None, + file_url: Optional[str] = None, + file_id: Optional[str] = None, + ) -> Tuple[Optional[str], Optional[bytes], Optional[str], Optional[str]]: + """ + Upload / prepare file for ingestion. + + Args: + file_data: Tuple of (filename, content_bytes, content_type) + file_url: URL to fetch file from + file_id: Existing file ID to use + + Returns: + Tuple of (filename, file_content, content_type, existing_file_id) + """ + if file_data: + filename, file_content, content_type = file_data + return filename, file_content, content_type, None + + if file_url: + async with httpx.AsyncClient() as http_client: + response = await http_client.get(file_url) + response.raise_for_status() + file_content = response.content + filename = file_url.split("/")[-1] or "document" + content_type = response.headers.get("content-type", "application/octet-stream") + return filename, file_content, content_type, None + + if file_id: + return None, None, None, file_id + + raise ValueError("Must provide file_data, file_url, or file_id") + + async def ocr( + self, + file_content: Optional[bytes], + content_type: Optional[str], + ) -> Optional[str]: + """ + Perform OCR on file content to extract text. + + Args: + file_content: Raw file bytes + content_type: MIME type of the file + + Returns: + Extracted text or None if OCR not configured/needed + """ + if not self.ocr_config or not file_content: + return None + + ocr_model = self.ocr_config.get("model", "mistral/mistral-ocr-latest") + + # Determine document type + if content_type and "image" in content_type: + doc_type, url_key = "image_url", "image_url" + else: + doc_type, url_key = "document_url", "document_url" + + # Encode as base64 data URL + b64_content = base64.b64encode(file_content).decode("utf-8") + data_url = f"data:{content_type};base64,{b64_content}" + + # Use router if available + if self.router is not None: + ocr_response = await self.router.aocr( + model=ocr_model, + document={"type": doc_type, url_key: data_url}, + ) + else: + ocr_response = await litellm.aocr( + model=ocr_model, + document={"type": doc_type, url_key: data_url}, + ) + + # Extract text from pages + if hasattr(ocr_response, "pages") and ocr_response.pages: # type: ignore + return "\n\n".join( + page.markdown for page in ocr_response.pages if hasattr(page, "markdown") # type: ignore + ) + + return None + + def chunk( + self, + text: Optional[str], + file_content: Optional[bytes], + ocr_was_used: bool, + ) -> List[str]: + """ + Split text into chunks using RecursiveCharacterTextSplitter. + + Args: + text: Text from OCR (if used) + file_content: Raw file content bytes + ocr_was_used: Whether OCR was performed + + Returns: + List of text chunks + """ + # Get text to chunk + text_to_chunk: Optional[str] = None + if text: + text_to_chunk = text + elif file_content and not ocr_was_used: + try: + text_to_chunk = file_content.decode("utf-8") + except UnicodeDecodeError: + verbose_logger.debug("Binary file detected, skipping text chunking") + return [] + + if not text_to_chunk: + return [] + + # Extract RecursiveCharacterTextSplitter args + splitter_args = self.chunking_strategy or {} + chunk_size = splitter_args.get("chunk_size", DEFAULT_CHUNK_SIZE) + chunk_overlap = splitter_args.get("chunk_overlap", DEFAULT_CHUNK_OVERLAP) + separators = splitter_args.get("separators", None) + + # Build splitter kwargs + splitter_kwargs: Dict[str, Any] = { + "chunk_size": chunk_size, + "chunk_overlap": chunk_overlap, + } + if separators: + splitter_kwargs["separators"] = separators + + text_splitter = RecursiveCharacterTextSplitter(**splitter_kwargs) + return text_splitter.split_text(text_to_chunk) + + async def embed( + self, + chunks: List[str], + ) -> Optional[List[List[float]]]: + """ + Generate embeddings for text chunks. + + Args: + chunks: List of text chunks + + Returns: + List of embeddings or None + """ + if not self.embedding_config or not chunks: + return None + + embedding_model = self.embedding_config.get("model", "text-embedding-3-small") + + if self.router is not None: + response = await self.router.aembedding(model=embedding_model, input=chunks) + else: + response = await litellm.aembedding(model=embedding_model, input=chunks) + + return [item["embedding"] for item in response.data] + + @abstractmethod + async def store( + self, + file_content: Optional[bytes], + filename: Optional[str], + content_type: Optional[str], + chunks: List[str], + embeddings: Optional[List[List[float]]], + ) -> Tuple[Optional[str], Optional[str]]: + """ + Store content in vector store. + + This method must be implemented by provider-specific subclasses. + + Args: + file_content: Raw file bytes + filename: Name of the file + content_type: MIME type + chunks: Text chunks (if chunking was done locally) + embeddings: Embeddings (if embedding was done locally) + + Returns: + Tuple of (vector_store_id, file_id) + """ + pass + + async def ingest( + self, + file_data: Optional[Tuple[str, bytes, str]] = None, + file_url: Optional[str] = None, + file_id: Optional[str] = None, + ) -> RAGIngestResponse: + """ + Execute the full ingestion pipeline. + + Args: + file_data: Tuple of (filename, content_bytes, content_type) + file_url: URL to fetch file from + file_id: Existing file ID to use + + Returns: + RAGIngestResponse with status and IDs + + Raises: + ValueError: If no input source is provided + """ + # Step 1: Upload (raises ValueError if no input provided) + filename, file_content, content_type, existing_file_id = await self.upload( + file_data=file_data, + file_url=file_url, + file_id=file_id, + ) + + try: + # Step 2: OCR (optional) + extracted_text = await self.ocr( + file_content=file_content, + content_type=content_type, + ) + + # Step 3: Chunking + chunks = self.chunk( + text=extracted_text, + file_content=file_content, + ocr_was_used=self.ocr_config is not None, + ) + + # Step 4: Embedding (optional - some providers handle this internally) + embeddings = await self.embed(chunks=chunks) + + # Step 5: Store in vector store + vector_store_id, result_file_id = await self.store( + file_content=file_content, + filename=filename, + content_type=content_type, + chunks=chunks, + embeddings=embeddings, + ) + + return RAGIngestResponse( + id=self.ingest_id, + status="completed", + vector_store_id=vector_store_id or "", + file_id=result_file_id or existing_file_id, + ) + + except Exception as e: + verbose_logger.exception(f"RAG Pipeline failed: {e}") + return RAGIngestResponse( + id=self.ingest_id, + status="failed", + vector_store_id="", + file_id=None, + ) + diff --git a/litellm/rag/ingestion/bedrock_ingestion.py b/litellm/rag/ingestion/bedrock_ingestion.py new file mode 100644 index 00000000000..5fa6145e248 --- /dev/null +++ b/litellm/rag/ingestion/bedrock_ingestion.py @@ -0,0 +1,619 @@ +""" +Bedrock-specific RAG Ingestion implementation. + +Bedrock Knowledge Bases handle embedding internally when files are ingested, +so this implementation uploads files to S3 and triggers ingestion jobs. + +Supports two modes: +1. Use existing KB: Provide vector_store_id (KB ID) +2. Auto-create KB: Don't provide vector_store_id - creates all AWS resources automatically +""" + +from __future__ import annotations + +import json +import time +import uuid +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple + +from litellm._logging import verbose_logger +from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM +from litellm.rag.ingestion.base_ingestion import BaseRAGIngestion + +if TYPE_CHECKING: + from litellm import Router + from litellm.types.rag import RAGIngestOptions + + +def _get_str_or_none(value: Any) -> Optional[str]: + """Cast config value to Optional[str].""" + return str(value) if value is not None else None + + +def _get_int(value: Any, default: int) -> int: + """Cast config value to int with default.""" + if value is None: + return default + return int(value) + + +class BedrockRAGIngestion(BaseRAGIngestion, BaseAWSLLM): + """ + Bedrock Knowledge Base RAG ingestion. + + Supports two modes: + 1. **Use existing KB**: Provide vector_store_id + 2. **Auto-create KB**: Don't provide vector_store_id - creates S3 bucket, + OpenSearch Serverless collection, IAM role, KB, and data source automatically + + Optional config: + - vector_store_id: Existing KB ID (if not provided, auto-creates) + - s3_bucket: S3 bucket (auto-created if not provided) + - embedding_model: Bedrock embedding model (default: amazon.titan-embed-text-v2:0) + - wait_for_ingestion: Wait for completion (default: True) + - ingestion_timeout: Max seconds to wait (default: 300) + + AWS Auth (uses BaseAWSLLM): + - aws_access_key_id, aws_secret_access_key, aws_session_token + - aws_region_name (default: us-west-2) + - aws_role_name, aws_session_name, aws_profile_name + - aws_web_identity_token, aws_sts_endpoint, aws_external_id + """ + + def __init__( + self, + ingest_options: "RAGIngestOptions", + router: Optional["Router"] = None, + ): + BaseRAGIngestion.__init__(self, ingest_options=ingest_options, router=router) + BaseAWSLLM.__init__(self) + + # Use vector_store_id as unified param (maps to knowledge_base_id) + self.knowledge_base_id = self.vector_store_config.get( + "vector_store_id" + ) or self.vector_store_config.get("knowledge_base_id") + + # Optional config + self._data_source_id = self.vector_store_config.get("data_source_id") + self._s3_bucket = self.vector_store_config.get("s3_bucket") + self._s3_prefix: Optional[str] = str(self.vector_store_config.get("s3_prefix")) if self.vector_store_config.get("s3_prefix") else None + self.embedding_model = self.vector_store_config.get( + "embedding_model" + ) or "amazon.titan-embed-text-v2:0" + + self.wait_for_ingestion = self.vector_store_config.get("wait_for_ingestion", False) + self.ingestion_timeout: int = _get_int(self.vector_store_config.get("ingestion_timeout"), 300) + + # Get AWS region using BaseAWSLLM method + _aws_region = self.vector_store_config.get("aws_region_name") + self.aws_region_name = self.get_aws_region_name_for_non_llm_api_calls( + aws_region_name=str(_aws_region) if _aws_region else None + ) + + # Will be set during initialization + self.data_source_id: Optional[str] = None + self.s3_bucket: Optional[str] = None + self.s3_prefix: str = self._s3_prefix or "data/" + self._config_initialized = False + + # Track resources we create (for cleanup if needed) + self._created_resources: Dict[str, Any] = {} + + def _ensure_config_initialized(self): + """Lazily initialize KB config - either detect from existing or create new.""" + if self._config_initialized: + return + + if self.knowledge_base_id: + # Use existing KB - auto-detect data source and S3 bucket + self._auto_detect_config() + else: + # No KB provided - create everything from scratch + self._create_knowledge_base_infrastructure() + + self._config_initialized = True + + def _auto_detect_config(self): + """Auto-detect data source ID and S3 bucket from existing Knowledge Base.""" + verbose_logger.debug( + f"Auto-detecting data source and S3 bucket for KB={self.knowledge_base_id}" + ) + + bedrock_agent = self._get_boto3_client("bedrock-agent") + + # List data sources for this KB + ds_response = bedrock_agent.list_data_sources( + knowledgeBaseId=self.knowledge_base_id + ) + data_sources = ds_response.get("dataSourceSummaries", []) + + if not data_sources: + raise ValueError( + f"No data sources found for Knowledge Base {self.knowledge_base_id}. " + "Please create a data source first or provide data_source_id and s3_bucket." + ) + + # Use first data source (or user-provided override) + if self._data_source_id: + self.data_source_id = self._data_source_id + else: + self.data_source_id = data_sources[0]["dataSourceId"] + verbose_logger.info(f"Auto-detected data source: {self.data_source_id}") + + # Get data source details for S3 bucket + ds_details = bedrock_agent.get_data_source( + knowledgeBaseId=self.knowledge_base_id, + dataSourceId=self.data_source_id, + ) + + s3_config = ( + ds_details.get("dataSource", {}) + .get("dataSourceConfiguration", {}) + .get("s3Configuration", {}) + ) + + bucket_arn = s3_config.get("bucketArn", "") + if bucket_arn: + # Extract bucket name from ARN: arn:aws:s3:::bucket-name + self.s3_bucket = self._s3_bucket or bucket_arn.split(":")[-1] + verbose_logger.info(f"Auto-detected S3 bucket: {self.s3_bucket}") + + # Use inclusion prefix if available + prefixes = s3_config.get("inclusionPrefixes", []) + if prefixes and not self._s3_prefix: + self.s3_prefix = prefixes[0] + else: + if not self._s3_bucket: + raise ValueError( + f"Could not auto-detect S3 bucket for data source {self.data_source_id}. " + "Please provide s3_bucket in config." + ) + self.s3_bucket = self._s3_bucket + + def _create_knowledge_base_infrastructure(self): + """Create all AWS resources needed for a new Knowledge Base.""" + verbose_logger.info("Creating new Bedrock Knowledge Base infrastructure...") + + # Generate unique names + unique_id = uuid.uuid4().hex[:8] + kb_name = self.ingest_name or f"litellm-kb-{unique_id}" + + # Get AWS account ID + sts = self._get_boto3_client("sts") + account_id = sts.get_caller_identity()["Account"] + + # Step 1: Create S3 bucket (if not provided) + self.s3_bucket = self._s3_bucket or self._create_s3_bucket(unique_id) + + # Step 2: Create OpenSearch Serverless collection + collection_name, collection_arn = self._create_opensearch_collection( + unique_id, account_id + ) + + # Step 3: Create OpenSearch index + self._create_opensearch_index(collection_name) + + # Step 4: Create IAM role for Bedrock + role_arn = self._create_bedrock_role(unique_id, account_id, collection_arn) + + # Step 5: Create Knowledge Base + self.knowledge_base_id = self._create_knowledge_base( + kb_name, role_arn, collection_arn + ) + + # Step 6: Create Data Source + self.data_source_id = self._create_data_source(kb_name) + + verbose_logger.info( + f"Created KB infrastructure: kb_id={self.knowledge_base_id}, " + f"ds_id={self.data_source_id}, bucket={self.s3_bucket}" + ) + + def _create_s3_bucket(self, unique_id: str) -> str: + """Create S3 bucket for KB data source.""" + s3 = self._get_boto3_client("s3") + bucket_name = f"litellm-kb-{unique_id}" + + verbose_logger.debug(f"Creating S3 bucket: {bucket_name}") + + create_params: Dict[str, Any] = {"Bucket": bucket_name} + if self.aws_region_name != "us-east-1": + create_params["CreateBucketConfiguration"] = { + "LocationConstraint": self.aws_region_name + } + + s3.create_bucket(**create_params) + self._created_resources["s3_bucket"] = bucket_name + + verbose_logger.info(f"Created S3 bucket: {bucket_name}") + return bucket_name + + def _create_opensearch_collection( + self, unique_id: str, account_id: str + ) -> Tuple[str, str]: + """Create OpenSearch Serverless collection for vector storage.""" + oss = self._get_boto3_client("opensearchserverless") + collection_name = f"litellm-kb-{unique_id}" + + verbose_logger.debug(f"Creating OpenSearch Serverless collection: {collection_name}") + + # Create encryption policy + oss.create_security_policy( + name=f"{collection_name}-enc", + type="encryption", + policy=json.dumps({ + "Rules": [{"ResourceType": "collection", "Resource": [f"collection/{collection_name}"]}], + "AWSOwnedKey": True, + }), + ) + + # Create network policy (public access for simplicity) + oss.create_security_policy( + name=f"{collection_name}-net", + type="network", + policy=json.dumps([{ + "Rules": [{"ResourceType": "collection", "Resource": [f"collection/{collection_name}"]}, + {"ResourceType": "dashboard", "Resource": [f"collection/{collection_name}"]}], + "AllowFromPublic": True, + }]), + ) + + # Create data access policy + oss.create_access_policy( + name=f"{collection_name}-access", + type="data", + policy=json.dumps([{ + "Rules": [ + {"ResourceType": "index", "Resource": [f"index/{collection_name}/*"], "Permission": ["aoss:*"]}, + {"ResourceType": "collection", "Resource": [f"collection/{collection_name}"], "Permission": ["aoss:*"]}, + ], + "Principal": [f"arn:aws:iam::{account_id}:root"], + }]), + ) + + # Create collection + response = oss.create_collection( + name=collection_name, + type="VECTORSEARCH", + ) + collection_id = response["createCollectionDetail"]["id"] + self._created_resources["opensearch_collection"] = collection_name + + # Wait for collection to be active + verbose_logger.debug("Waiting for OpenSearch collection to be active...") + for _ in range(60): # 5 min timeout + status_response = oss.batch_get_collection(ids=[collection_id]) + status = status_response["collectionDetails"][0]["status"] + if status == "ACTIVE": + break + time.sleep(5) + else: + raise TimeoutError("OpenSearch collection did not become active in time") + + collection_arn = status_response["collectionDetails"][0]["arn"] + verbose_logger.info(f"Created OpenSearch collection: {collection_name}") + + return collection_name, collection_arn + + def _create_opensearch_index(self, collection_name: str): + """Create vector index in OpenSearch collection.""" + from opensearchpy import OpenSearch, RequestsHttpConnection + from requests_aws4auth import AWS4Auth + + # Get credentials for signing + credentials = self.get_credentials( + aws_access_key_id=_get_str_or_none(self.vector_store_config.get("aws_access_key_id")), + aws_secret_access_key=_get_str_or_none(self.vector_store_config.get("aws_secret_access_key")), + aws_session_token=_get_str_or_none(self.vector_store_config.get("aws_session_token")), + aws_region_name=self.aws_region_name, + ) + + # Get collection endpoint + oss = self._get_boto3_client("opensearchserverless") + collections = oss.batch_get_collection(names=[collection_name]) + endpoint = collections["collectionDetails"][0]["collectionEndpoint"] + host = endpoint.replace("https://", "") + + auth = AWS4Auth( + credentials.access_key, + credentials.secret_key, + self.aws_region_name, + "aoss", + session_token=credentials.token, + ) + + client = OpenSearch( + hosts=[{"host": host, "port": 443}], + http_auth=auth, + use_ssl=True, + verify_certs=True, + connection_class=RequestsHttpConnection, + ) + + index_name = "bedrock-kb-index" + index_body = { + "settings": { + "index": {"knn": True, "knn.algo_param.ef_search": 512} + }, + "mappings": { + "properties": { + "bedrock-knowledge-base-default-vector": { + "type": "knn_vector", + "dimension": 1024, + "method": {"engine": "faiss", "name": "hnsw", "space_type": "l2"}, + }, + "AMAZON_BEDROCK_METADATA": {"type": "text", "index": False}, + "AMAZON_BEDROCK_TEXT_CHUNK": {"type": "text"}, + } + }, + } + + client.indices.create(index=index_name, body=index_body) + verbose_logger.info(f"Created OpenSearch index: {index_name}") + + def _create_bedrock_role( + self, unique_id: str, account_id: str, collection_arn: str + ) -> str: + """Create IAM role for Bedrock KB.""" + iam = self._get_boto3_client("iam") + role_name = f"litellm-bedrock-kb-{unique_id}" + + verbose_logger.debug(f"Creating IAM role: {role_name}") + + trust_policy = { + "Version": "2012-10-17", + "Statement": [{ + "Effect": "Allow", + "Principal": {"Service": "bedrock.amazonaws.com"}, + "Action": "sts:AssumeRole", + "Condition": { + "StringEquals": {"aws:SourceAccount": account_id}, + "ArnLike": {"aws:SourceArn": f"arn:aws:bedrock:{self.aws_region_name}:{account_id}:knowledge-base/*"}, + }, + }], + } + + response = iam.create_role( + RoleName=role_name, + AssumeRolePolicyDocument=json.dumps(trust_policy), + ) + role_arn = response["Role"]["Arn"] + self._created_resources["iam_role"] = role_name + + # Attach permissions policy + permissions_policy = { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": ["bedrock:InvokeModel"], + "Resource": [f"arn:aws:bedrock:{self.aws_region_name}::foundation-model/{self.embedding_model}"], + }, + { + "Effect": "Allow", + "Action": ["aoss:APIAccessAll"], + "Resource": [collection_arn], + }, + { + "Effect": "Allow", + "Action": ["s3:GetObject", "s3:ListBucket"], + "Resource": [f"arn:aws:s3:::{self.s3_bucket}", f"arn:aws:s3:::{self.s3_bucket}/*"], + }, + ], + } + + iam.put_role_policy( + RoleName=role_name, + PolicyName=f"{role_name}-policy", + PolicyDocument=json.dumps(permissions_policy), + ) + + # Wait for role to propagate + time.sleep(10) + + verbose_logger.info(f"Created IAM role: {role_arn}") + return role_arn + + def _create_knowledge_base( + self, kb_name: str, role_arn: str, collection_arn: str + ) -> str: + """Create Bedrock Knowledge Base.""" + bedrock_agent = self._get_boto3_client("bedrock-agent") + + verbose_logger.debug(f"Creating Knowledge Base: {kb_name}") + + response = bedrock_agent.create_knowledge_base( + name=kb_name, + roleArn=role_arn, + knowledgeBaseConfiguration={ + "type": "VECTOR", + "vectorKnowledgeBaseConfiguration": { + "embeddingModelArn": f"arn:aws:bedrock:{self.aws_region_name}::foundation-model/{self.embedding_model}", + }, + }, + storageConfiguration={ + "type": "OPENSEARCH_SERVERLESS", + "opensearchServerlessConfiguration": { + "collectionArn": collection_arn, + "fieldMapping": { + "metadataField": "AMAZON_BEDROCK_METADATA", + "textField": "AMAZON_BEDROCK_TEXT_CHUNK", + "vectorField": "bedrock-knowledge-base-default-vector", + }, + "vectorIndexName": "bedrock-kb-index", + }, + }, + ) + kb_id = response["knowledgeBase"]["knowledgeBaseId"] + self._created_resources["knowledge_base"] = kb_id + + # Wait for KB to be active + verbose_logger.debug("Waiting for Knowledge Base to be active...") + for _ in range(30): + kb_status = bedrock_agent.get_knowledge_base(knowledgeBaseId=kb_id) + status = kb_status["knowledgeBase"]["status"] + if status == "ACTIVE": + break + time.sleep(2) + else: + raise TimeoutError("Knowledge Base did not become active in time") + + verbose_logger.info(f"Created Knowledge Base: {kb_id}") + return kb_id + + def _create_data_source(self, kb_name: str) -> str: + """Create Data Source for the Knowledge Base.""" + bedrock_agent = self._get_boto3_client("bedrock-agent") + + verbose_logger.debug(f"Creating Data Source for KB: {self.knowledge_base_id}") + + response = bedrock_agent.create_data_source( + knowledgeBaseId=self.knowledge_base_id, + name=f"{kb_name}-s3-source", + dataSourceConfiguration={ + "type": "S3", + "s3Configuration": { + "bucketArn": f"arn:aws:s3:::{self.s3_bucket}", + "inclusionPrefixes": [self.s3_prefix], + }, + }, + ) + ds_id = response["dataSource"]["dataSourceId"] + self._created_resources["data_source"] = ds_id + + verbose_logger.info(f"Created Data Source: {ds_id}") + return ds_id + + def _get_boto3_client(self, service_name: str): + """Get a boto3 client for the specified service using BaseAWSLLM auth.""" + try: + import boto3 + except ImportError: + raise ImportError("boto3 is required for Bedrock ingestion. Install with: pip install boto3") + + # Get credentials using BaseAWSLLM's get_credentials method + credentials = self.get_credentials( + aws_access_key_id=_get_str_or_none(self.vector_store_config.get("aws_access_key_id")), + aws_secret_access_key=_get_str_or_none(self.vector_store_config.get("aws_secret_access_key")), + aws_session_token=_get_str_or_none(self.vector_store_config.get("aws_session_token")), + aws_region_name=self.aws_region_name, + aws_session_name=_get_str_or_none(self.vector_store_config.get("aws_session_name")), + aws_profile_name=_get_str_or_none(self.vector_store_config.get("aws_profile_name")), + aws_role_name=_get_str_or_none(self.vector_store_config.get("aws_role_name")), + aws_web_identity_token=_get_str_or_none(self.vector_store_config.get("aws_web_identity_token")), + aws_sts_endpoint=_get_str_or_none(self.vector_store_config.get("aws_sts_endpoint")), + aws_external_id=_get_str_or_none(self.vector_store_config.get("aws_external_id")), + ) + + # Create session with credentials + session = boto3.Session( + aws_access_key_id=credentials.access_key, + aws_secret_access_key=credentials.secret_key, + aws_session_token=credentials.token, + region_name=self.aws_region_name, + ) + + return session.client(service_name) + + async def embed( + self, + chunks: List[str], + ) -> Optional[List[List[float]]]: + """ + Bedrock handles embedding internally - skip this step. + + Returns: + None (Bedrock embeds when files are ingested) + """ + return None + + async def store( + self, + file_content: Optional[bytes], + filename: Optional[str], + content_type: Optional[str], + chunks: List[str], + embeddings: Optional[List[List[float]]], + ) -> Tuple[Optional[str], Optional[str]]: + """ + Store content in Bedrock Knowledge Base. + + Bedrock workflow: + 1. Auto-detect data source and S3 bucket (if not provided) + 2. Upload file to S3 bucket + 3. Start ingestion job + 4. (Optional) Wait for ingestion to complete + + Args: + file_content: Raw file bytes + filename: Name of the file + content_type: MIME type + chunks: Ignored - Bedrock handles chunking + embeddings: Ignored - Bedrock handles embedding + + Returns: + Tuple of (knowledge_base_id, file_key) + """ + # Auto-detect data source and S3 bucket if needed + self._ensure_config_initialized() + + if not file_content or not filename: + verbose_logger.warning("No file content or filename provided for Bedrock ingestion") + return _get_str_or_none(self.knowledge_base_id), None + + # Step 1: Upload file to S3 + s3_client = self._get_boto3_client("s3") + s3_key = f"{self.s3_prefix.rstrip('/')}/{filename}" + + verbose_logger.debug(f"Uploading file to s3://{self.s3_bucket}/{s3_key}") + s3_client.put_object( + Bucket=self.s3_bucket, + Key=s3_key, + Body=file_content, + ContentType=content_type or "application/octet-stream", + ) + verbose_logger.info(f"Uploaded file to s3://{self.s3_bucket}/{s3_key}") + + # Step 2: Start ingestion job + bedrock_agent = self._get_boto3_client("bedrock-agent") + + verbose_logger.debug( + f"Starting ingestion job for KB={self.knowledge_base_id}, DS={self.data_source_id}" + ) + ingestion_response = bedrock_agent.start_ingestion_job( + knowledgeBaseId=self.knowledge_base_id, + dataSourceId=self.data_source_id, + ) + job_id = ingestion_response["ingestionJob"]["ingestionJobId"] + verbose_logger.info(f"Started ingestion job: {job_id}") + + # Step 3: Wait for ingestion (optional) + if self.wait_for_ingestion: + start_time = time.time() + while time.time() - start_time < self.ingestion_timeout: + job_status = bedrock_agent.get_ingestion_job( + knowledgeBaseId=self.knowledge_base_id, + dataSourceId=self.data_source_id, + ingestionJobId=job_id, + ) + status = job_status["ingestionJob"]["status"] + verbose_logger.debug(f"Ingestion job {job_id} status: {status}") + + if status == "COMPLETE": + stats = job_status["ingestionJob"].get("statistics", {}) + verbose_logger.info( + f"Ingestion complete: {stats.get('numberOfNewDocumentsIndexed', 0)} docs indexed" + ) + break + elif status == "FAILED": + failure_reasons = job_status["ingestionJob"].get("failureReasons", []) + verbose_logger.error(f"Ingestion failed: {failure_reasons}") + break + elif status in ("STARTING", "IN_PROGRESS"): + time.sleep(2) + else: + verbose_logger.warning(f"Unknown ingestion status: {status}") + break + + return str(self.knowledge_base_id) if self.knowledge_base_id else None, s3_key + diff --git a/litellm/rag/ingestion/gemini_ingestion.py b/litellm/rag/ingestion/gemini_ingestion.py new file mode 100644 index 00000000000..5f6f5fc992e --- /dev/null +++ b/litellm/rag/ingestion/gemini_ingestion.py @@ -0,0 +1,319 @@ +""" +Gemini-specific RAG Ingestion implementation. + +Gemini handles embedding and chunking internally when files are uploaded to File Search stores, +so this implementation skips the embedding step and directly uploads files. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, cast + +import httpx + +from litellm._logging import verbose_logger +from litellm.llms.gemini.common_utils import GeminiModelInfo +from litellm.rag.ingestion.base_ingestion import BaseRAGIngestion + +if TYPE_CHECKING: + from litellm import Router + from litellm.types.rag import RAGIngestOptions + + +class GeminiRAGIngestion(BaseRAGIngestion): + """ + Gemini-specific RAG ingestion using File Search API. + + Key differences from base: + - Embedding is handled by Gemini when files are uploaded to File Search stores + - Files are uploaded using uploadToFileSearchStore API + - Chunking is done by Gemini's File Search (supports custom white_space_config) + - Supports custom metadata attachment + """ + + def __init__( + self, + ingest_options: "RAGIngestOptions", + router: Optional["Router"] = None, + ): + super().__init__(ingest_options=ingest_options, router=router) + self.model_info = GeminiModelInfo() + + async def embed( + self, + chunks: List[str], + ) -> Optional[List[List[float]]]: + """ + Gemini handles embedding internally - skip this step. + + Returns: + None (Gemini embeds when files are uploaded to File Search store) + """ + # Gemini handles embedding when files are uploaded to File Search stores + return None + + async def store( + self, + file_content: Optional[bytes], + filename: Optional[str], + content_type: Optional[str], + chunks: List[str], + embeddings: Optional[List[List[float]]], + ) -> Tuple[Optional[str], Optional[str]]: + """ + Store content in Gemini File Search store. + + Gemini workflow: + 1. Create File Search store (if not provided) + 2. Upload file using uploadToFileSearchStore (Gemini handles chunking/embedding) + + Args: + file_content: Raw file bytes + filename: Name of the file + content_type: MIME type + chunks: Ignored - Gemini handles chunking + embeddings: Ignored - Gemini handles embedding + + Returns: + Tuple of (vector_store_id, file_id) + """ + vector_store_id = self.vector_store_config.get("vector_store_id") + + vector_store_config = cast(Dict[str, Any], self.vector_store_config) + + # Get API credentials + api_key = cast(Optional[str], vector_store_config.get("api_key")) or GeminiModelInfo.get_api_key() + api_base = cast(Optional[str], vector_store_config.get("api_base")) or GeminiModelInfo.get_api_base() + + if not api_key: + raise ValueError("GEMINI_API_KEY or GOOGLE_API_KEY is required for Gemini File Search") + + if not api_base: + raise ValueError("GEMINI_API_BASE is required") + + api_version = "v1beta" + base_url = f"{api_base}/{api_version}" + + # Create File Search store if not provided + if not vector_store_id: + vector_store_id = await self._create_file_search_store( + api_key=api_key, + base_url=base_url, + display_name=self.ingest_name or "litellm-rag-ingest", + ) + + # Upload file to File Search store + result_file_id = None + if file_content and filename and vector_store_id: + result_file_id = await self._upload_to_file_search_store( + api_key=api_key, + base_url=base_url, + vector_store_id=vector_store_id, + filename=filename, + file_content=file_content, + content_type=content_type, + ) + + return vector_store_id, result_file_id + + async def _create_file_search_store( + self, + api_key: str, + base_url: str, + display_name: str, + ) -> str: + """ + Create a Gemini File Search store. + + Args: + api_key: Gemini API key + base_url: Base URL for Gemini API + display_name: Display name for the store + + Returns: + Store name (format: fileSearchStores/xxxxxxx) + """ + url = f"{base_url}/fileSearchStores?key={api_key}" + + request_body = { + "displayName": display_name + } + + async with httpx.AsyncClient() as client: + response = await client.post( + url, + json=request_body, + headers={"Content-Type": "application/json"}, + timeout=60.0, + ) + + if response.status_code != 200: + error_msg = f"Failed to create File Search store: {response.text}" + verbose_logger.error(error_msg) + raise Exception(error_msg) + + response_data = response.json() + store_name = response_data.get("name", "") + + verbose_logger.debug(f"Created File Search store: {store_name}") + return store_name + + async def _upload_to_file_search_store( + self, + api_key: str, + base_url: str, + vector_store_id: str, + filename: str, + file_content: bytes, + content_type: Optional[str], + ) -> str: + """ + Upload a file to Gemini File Search store using resumable upload. + + Args: + api_key: Gemini API key + base_url: Base URL for Gemini API + vector_store_id: File Search store name + filename: Name of the file + file_content: File content bytes + content_type: MIME type + + Returns: + File ID or document name + """ + # Step 1: Initiate resumable upload + upload_url = await self._initiate_resumable_upload( + api_key=api_key, + base_url=base_url, + vector_store_id=vector_store_id, + filename=filename, + file_size=len(file_content), + content_type=content_type or "application/octet-stream", + ) + + # Step 2: Upload the file content + file_id = await self._upload_file_content( + upload_url=upload_url, + file_content=file_content, + ) + + return file_id + + async def _initiate_resumable_upload( + self, + api_key: str, + base_url: str, + vector_store_id: str, + filename: str, + file_size: int, + content_type: str, + ) -> str: + """ + Initiate a resumable upload session. + + Returns: + Upload URL for the resumable session + """ + # Construct the upload URL - need to use the full upload endpoint + # base_url is like: https://generativelanguage.googleapis.com/v1beta + # We need: https://generativelanguage.googleapis.com/upload/v1beta/{store_id}:uploadToFileSearchStore + api_base = base_url.replace("/v1beta", "") # Get base without version + url = f"{api_base}/upload/v1beta/{vector_store_id}:uploadToFileSearchStore?key={api_key}" + + # Build request body with chunking config and metadata if provided + request_body: Dict[str, Any] = { + "displayName": filename + } + + # Add chunking configuration if provided + chunking_strategy = self.chunking_strategy + if chunking_strategy and isinstance(chunking_strategy, dict): + white_space_config = chunking_strategy.get("white_space_config") + if white_space_config: + request_body["chunkingConfig"] = { + "whiteSpaceConfig": { + "maxTokensPerChunk": white_space_config.get("max_tokens_per_chunk", 800), + "maxOverlapTokens": white_space_config.get("max_overlap_tokens", 400), + } + } + + # Add custom metadata if provided in vector_store_config + custom_metadata = cast(Optional[List[Dict[str, Any]]], self.vector_store_config.get("custom_metadata")) + if custom_metadata: + request_body["customMetadata"] = custom_metadata + + headers = { + "X-Goog-Upload-Protocol": "resumable", + "X-Goog-Upload-Command": "start", + "X-Goog-Upload-Header-Content-Length": str(file_size), + "X-Goog-Upload-Header-Content-Type": content_type, + "Content-Type": "application/json", + } + + verbose_logger.debug(f"Initiating resumable upload: {url}") + + async with httpx.AsyncClient() as client: + response = await client.post( + url, + json=request_body, + headers=headers, + timeout=60.0, + ) + + if response.status_code not in [200, 201]: + error_msg = f"Failed to initiate upload: {response.text}" + verbose_logger.error(error_msg) + raise Exception(error_msg) + verbose_logger.debug(f"Initiate resumable upload response: {response.headers}") + # Extract upload URL from response headers + upload_url = response.headers.get("x-goog-upload-url") + if not upload_url: + raise Exception("No upload URL returned in response headers") + + verbose_logger.debug(f"Got upload URL: {upload_url}") + return upload_url + + async def _upload_file_content( + self, + upload_url: str, + file_content: bytes, + ) -> str: + """ + Upload file content to the resumable upload URL. + + Returns: + File ID or document name from the response + """ + headers = { + "Content-Length": str(len(file_content)), + "X-Goog-Upload-Offset": "0", + "X-Goog-Upload-Command": "upload, finalize", + } + + verbose_logger.debug(f"Uploading file content ({len(file_content)} bytes)") + + async with httpx.AsyncClient() as client: + response = await client.put( + upload_url, + content=file_content, + headers=headers, + timeout=300.0, # Longer timeout for large files + ) + + if response.status_code not in [200, 201]: + error_msg = f"Failed to upload file: {response.text}" + verbose_logger.error(error_msg) + raise Exception(error_msg) + + # Parse response to get file/document ID + try: + response_data = response.json() + # The response should contain the document name or file reference + file_id = response_data.get("name", "") or response_data.get("file", {}).get("name", "") + verbose_logger.debug(f"Upload complete. File ID: {file_id}") + return file_id + except Exception as e: + verbose_logger.warning(f"Could not parse upload response: {e}") + # Return a placeholder if we can't get the ID + return "uploaded" + diff --git a/litellm/rag/ingestion/openai_ingestion.py b/litellm/rag/ingestion/openai_ingestion.py new file mode 100644 index 00000000000..034ad38d466 --- /dev/null +++ b/litellm/rag/ingestion/openai_ingestion.py @@ -0,0 +1,111 @@ +""" +OpenAI-specific RAG Ingestion implementation. + +OpenAI handles embedding internally when files are attached to vector stores, +so this implementation skips the embedding step and directly uploads files. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, cast + +import litellm +from litellm.rag.ingestion.base_ingestion import BaseRAGIngestion +from litellm.vector_store_files.main import acreate as vector_store_file_acreate +from litellm.vector_stores.main import acreate as vector_store_acreate + +if TYPE_CHECKING: + from litellm import Router + from litellm.types.rag import RAGIngestOptions + + +class OpenAIRAGIngestion(BaseRAGIngestion): + """ + OpenAI-specific RAG ingestion. + + Key differences from base: + - Embedding is handled by OpenAI when attaching files to vector stores + - Files are uploaded and attached to vector stores directly + - Chunking is done by OpenAI's vector store (uses 'auto' strategy) + """ + + def __init__( + self, + ingest_options: "RAGIngestOptions", + router: Optional["Router"] = None, + ): + super().__init__(ingest_options=ingest_options, router=router) + + async def embed( + self, + chunks: List[str], + ) -> Optional[List[List[float]]]: + """ + OpenAI handles embedding internally - skip this step. + + Returns: + None (OpenAI embeds when files are attached to vector store) + """ + # OpenAI handles embedding when files are attached to vector stores + return None + + async def store( + self, + file_content: Optional[bytes], + filename: Optional[str], + content_type: Optional[str], + chunks: List[str], + embeddings: Optional[List[List[float]]], + ) -> Tuple[Optional[str], Optional[str]]: + """ + Store content in OpenAI vector store. + + OpenAI workflow: + 1. Create vector store (if not provided) + 2. Upload file to OpenAI + 3. Attach file to vector store (OpenAI handles chunking/embedding) + + Args: + file_content: Raw file bytes + filename: Name of the file + content_type: MIME type + chunks: Ignored - OpenAI handles chunking + embeddings: Ignored - OpenAI handles embedding + + Returns: + Tuple of (vector_store_id, file_id) + """ + vector_store_id = self.vector_store_config.get("vector_store_id") + ttl_days = self.vector_store_config.get("ttl_days") + + # Create vector store if not provided + if not vector_store_id: + expires_after = {"anchor": "last_active_at", "days": ttl_days} if ttl_days else None + create_response = await vector_store_acreate( + name=self.ingest_name or "litellm-rag-ingest", + custom_llm_provider="openai", + expires_after=expires_after, + ) + vector_store_id = create_response.get("id") + + # Upload file and attach to vector store + result_file_id = None + if file_content and filename and vector_store_id: + # Upload file to OpenAI + file_response = await litellm.acreate_file( + file=(filename, file_content, content_type or "application/octet-stream"), + purpose="assistants", + custom_llm_provider="openai", + ) + result_file_id = file_response.id + + # Attach file to vector store (OpenAI handles chunking/embedding) + await vector_store_file_acreate( + vector_store_id=vector_store_id, + file_id=result_file_id, + custom_llm_provider="openai", + chunking_strategy=cast(Optional[Dict[str, Any]], self.chunking_strategy), + ) + + return vector_store_id, result_file_id + diff --git a/litellm/rag/main.py b/litellm/rag/main.py new file mode 100644 index 00000000000..fc7cc45a771 --- /dev/null +++ b/litellm/rag/main.py @@ -0,0 +1,242 @@ +""" +RAG Ingest API for LiteLLM. + +Provides an all-in-one API for document ingestion: +Upload -> (OCR) -> Chunk -> Embed -> Vector Store +""" + +from __future__ import annotations + +__all__ = ["ingest", "aingest"] + +import asyncio +import contextvars +from functools import partial +from typing import TYPE_CHECKING, Any, Coroutine, Dict, Optional, Tuple, Type, Union + +import httpx + +import litellm +from litellm.rag.ingestion.base_ingestion import BaseRAGIngestion +from litellm.rag.ingestion.bedrock_ingestion import BedrockRAGIngestion +from litellm.rag.ingestion.gemini_ingestion import GeminiRAGIngestion +from litellm.rag.ingestion.openai_ingestion import OpenAIRAGIngestion +from litellm.types.rag import RAGIngestOptions, RAGIngestResponse +from litellm.utils import client + +if TYPE_CHECKING: + from litellm import Router + + +# Registry of provider-specific ingestion classes +INGESTION_REGISTRY: Dict[str, Type[BaseRAGIngestion]] = { + "openai": OpenAIRAGIngestion, + "bedrock": BedrockRAGIngestion, + "gemini": GeminiRAGIngestion, +} + + +def get_ingestion_class(provider: str) -> Type[BaseRAGIngestion]: + """ + Get the ingestion class for a given provider. + + Args: + provider: The vector store provider name (e.g., 'openai') + + Returns: + The ingestion class for the provider + + Raises: + ValueError: If provider is not supported + """ + ingestion_class = INGESTION_REGISTRY.get(provider) + if ingestion_class is None: + supported = ", ".join(INGESTION_REGISTRY.keys()) + raise ValueError( + f"Provider '{provider}' is not supported for RAG ingestion. " + f"Supported providers: {supported}" + ) + return ingestion_class + + +async def _execute_ingest_pipeline( + ingest_options: RAGIngestOptions, + file_data: Optional[Tuple[str, bytes, str]] = None, + file_url: Optional[str] = None, + file_id: Optional[str] = None, + router: Optional["Router"] = None, +) -> RAGIngestResponse: + """ + Execute the RAG ingest pipeline using provider-specific implementation. + + Args: + ingest_options: Configuration for the ingest pipeline + file_data: Tuple of (filename, content_bytes, content_type) + file_url: URL to fetch file from + file_id: Existing file ID to use + router: Optional LiteLLM router for load balancing + + Returns: + RAGIngestResponse with status and IDs + """ + # Get provider from vector store config + vector_store_config = ingest_options.get("vector_store") or {} + provider = vector_store_config.get("custom_llm_provider", "openai") + + # Get provider-specific ingestion class + ingestion_class = get_ingestion_class(provider) + + # Create ingestion instance + ingestion = ingestion_class( + ingest_options=ingest_options, + router=router, + ) + + # Execute ingestion pipeline + return await ingestion.ingest( + file_data=file_data, + file_url=file_url, + file_id=file_id, + ) + + +####### PUBLIC API ################### + + +@client +async def aingest( + ingest_options: Dict[str, Any], + file_data: Optional[Tuple[str, bytes, str]] = None, + file: Optional[Dict[str, str]] = None, + file_url: Optional[str] = None, + file_id: Optional[str] = None, + timeout: Optional[Union[float, httpx.Timeout]] = None, + **kwargs, +) -> RAGIngestResponse: + """ + Async: Ingest a document into a vector store. + + Args: + ingest_options: Configuration for the ingest pipeline + file_data: Tuple of (filename, content_bytes, content_type) + file: Dict with {filename, content (base64), content_type} - for JSON API + file_url: URL to fetch file from + file_id: Existing file ID to use + + Example: + ```python + response = await litellm.aingest( + ingest_options={ + "vector_store": {"custom_llm_provider": "openai"} + }, + file_url="https://example.com/doc.pdf", + ) + ``` + """ + local_vars = locals() + try: + loop = asyncio.get_event_loop() + kwargs["aingest"] = True + + func = partial( + ingest, + ingest_options=ingest_options, + file_data=file_data, + file=file, + file_url=file_url, + file_id=file_id, + timeout=timeout, + **kwargs, + ) + + ctx = contextvars.copy_context() + func_with_context = partial(ctx.run, func) + init_response = await loop.run_in_executor(None, func_with_context) + + if asyncio.iscoroutine(init_response): + response = await init_response + else: + response = init_response + + return response + except Exception as e: + raise litellm.exception_type( + model=None, + custom_llm_provider=ingest_options.get("vector_store", {}).get("custom_llm_provider"), + original_exception=e, + completion_kwargs=local_vars, + extra_kwargs=kwargs, + ) + + +@client +def ingest( + ingest_options: Dict[str, Any], + file_data: Optional[Tuple[str, bytes, str]] = None, + file: Optional[Dict[str, str]] = None, + file_url: Optional[str] = None, + file_id: Optional[str] = None, + timeout: Optional[Union[float, httpx.Timeout]] = None, + **kwargs, +) -> Union[RAGIngestResponse, Coroutine[Any, Any, RAGIngestResponse]]: + """ + Ingest a document into a vector store. + + Args: + ingest_options: Configuration for the ingest pipeline + file_data: Tuple of (filename, content_bytes, content_type) + file: Dict with {filename, content (base64), content_type} - for JSON API + file_url: URL to fetch file from + file_id: Existing file ID to use + + Example: + ```python + response = litellm.ingest( + ingest_options={ + "vector_store": {"custom_llm_provider": "openai"} + }, + file_data=("doc.txt", b"Hello world", "text/plain"), + ) + ``` + """ + import base64 + + local_vars = locals() + try: + _is_async = kwargs.pop("aingest", False) is True + router: Optional["Router"] = kwargs.get("router") + + # Convert file dict to file_data tuple if provided + if file is not None and file_data is None: + filename = file.get("filename", "document") + content_b64 = file.get("content", "") + content_type = file.get("content_type", "application/octet-stream") + content_bytes = base64.b64decode(content_b64) + file_data = (filename, content_bytes, content_type) + + if _is_async: + return _execute_ingest_pipeline( + ingest_options=ingest_options, # type: ignore + file_data=file_data, + file_url=file_url, + file_id=file_id, + router=router, + ) + else: + return asyncio.get_event_loop().run_until_complete( + _execute_ingest_pipeline( + ingest_options=ingest_options, # type: ignore + file_data=file_data, + file_url=file_url, + file_id=file_id, + router=router, + ) + ) + except Exception as e: + raise litellm.exception_type( + model=None, + custom_llm_provider=ingest_options.get("vector_store", {}).get("custom_llm_provider"), + original_exception=e, + completion_kwargs=local_vars, + extra_kwargs=kwargs, + ) diff --git a/litellm/rag/text_splitters/__init__.py b/litellm/rag/text_splitters/__init__.py new file mode 100644 index 00000000000..04802b438c7 --- /dev/null +++ b/litellm/rag/text_splitters/__init__.py @@ -0,0 +1,10 @@ +""" +Text splitting utilities for RAG ingestion. +""" + +from litellm.rag.text_splitters.recursive_character_text_splitter import ( + RecursiveCharacterTextSplitter, +) + +__all__ = ["RecursiveCharacterTextSplitter"] + diff --git a/litellm/rag/text_splitters/recursive_character_text_splitter.py b/litellm/rag/text_splitters/recursive_character_text_splitter.py new file mode 100644 index 00000000000..2107b1f0683 --- /dev/null +++ b/litellm/rag/text_splitters/recursive_character_text_splitter.py @@ -0,0 +1,135 @@ +""" +RecursiveCharacterTextSplitter for RAG ingestion. + +A simple implementation that splits text recursively by different separators. +""" + +from typing import List, Optional + +from litellm.constants import DEFAULT_CHUNK_OVERLAP, DEFAULT_CHUNK_SIZE + + +class RecursiveCharacterTextSplitter: + """ + Split text recursively by different separators. + + Tries to split by the first separator, then recursively splits + by subsequent separators if chunks are still too large. + """ + + def __init__( + self, + chunk_size: int = DEFAULT_CHUNK_SIZE, + chunk_overlap: int = DEFAULT_CHUNK_OVERLAP, + separators: Optional[List[str]] = None, + ): + self.chunk_size = chunk_size + self.chunk_overlap = chunk_overlap + self.separators = separators or ["\n\n", "\n", " ", ""] + + def split_text(self, text: str) -> List[str]: + """Split text into chunks.""" + return self._split_text(text, self.separators) + + def _split_text(self, text: str, separators: List[str], depth: int = 0) -> List[str]: + """Recursively split text using separators.""" + from litellm.constants import DEFAULT_MAX_RECURSE_DEPTH + + if depth > DEFAULT_MAX_RECURSE_DEPTH: + # Max depth reached, return text as-is split into chunk_size pieces + return [text[i:i + self.chunk_size] for i in range(0, len(text), self.chunk_size)] + + final_chunks: List[str] = [] + + # Get the appropriate separator + separator = separators[-1] + new_separators: List[str] = [] + + for i, sep in enumerate(separators): + if sep == "": + separator = sep + break + if sep in text: + separator = sep + new_separators = separators[i + 1 :] + break + + # Split by the chosen separator + if separator: + splits = text.split(separator) + else: + splits = list(text) + + # Merge splits into chunks + good_splits: List[str] = [] + for split in splits: + if len(split) < self.chunk_size: + good_splits.append(split) + else: + # Chunk is too big, merge what we have and recurse + if good_splits: + merged = self._merge_splits(good_splits, separator) + final_chunks.extend(merged) + good_splits = [] + + if new_separators: + # Recursively split with finer separators + other_chunks = self._split_text(split, new_separators, depth + 1) + final_chunks.extend(other_chunks) + else: + # No more separators, force split + final_chunks.extend(self._force_split(split)) + + # Merge remaining good splits + if good_splits: + merged = self._merge_splits(good_splits, separator) + final_chunks.extend(merged) + + return final_chunks + + def _merge_splits(self, splits: List[str], separator: str) -> List[str]: + """Merge splits into chunks respecting chunk_size and chunk_overlap.""" + chunks: List[str] = [] + current_chunk: List[str] = [] + current_length = 0 + + for split in splits: + split_len = len(split) + sep_len = len(separator) if current_chunk else 0 + + if current_length + split_len + sep_len > self.chunk_size: + if current_chunk: + chunk_text = separator.join(current_chunk).strip() + if chunk_text: + chunks.append(chunk_text) + + # Handle overlap + while current_length > self.chunk_overlap and len(current_chunk) > 1: + removed = current_chunk.pop(0) + current_length -= len(removed) + len(separator) + + current_chunk.append(split) + current_length += split_len + sep_len + + # Add remaining + if current_chunk: + chunk_text = separator.join(current_chunk).strip() + if chunk_text: + chunks.append(chunk_text) + + return chunks + + def _force_split(self, text: str) -> List[str]: + """Force split text by chunk_size when no separator works.""" + chunks: List[str] = [] + start = 0 + + while start < len(text): + end = start + self.chunk_size + chunk = text[start:end].strip() + if chunk: + chunks.append(chunk) + start = end - self.chunk_overlap if end < len(text) else len(text) + + return chunks + diff --git a/litellm/responses/streaming_iterator.py b/litellm/responses/streaming_iterator.py index 8eecc3e8211..0407776029d 100644 --- a/litellm/responses/streaming_iterator.py +++ b/litellm/responses/streaming_iterator.py @@ -8,7 +8,9 @@ import httpx import litellm from litellm.constants import STREAM_SSE_DONE_STRING from litellm.litellm_core_utils.asyncify import run_async_function +from litellm.litellm_core_utils.core_helpers import process_response_headers from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj +from litellm.litellm_core_utils.llm_response_utils.get_api_base import get_api_base from litellm.litellm_core_utils.thread_pool_executor import executor from litellm.llms.base_llm.responses.transformation import BaseResponsesAPIConfig from litellm.responses.utils import ResponsesAPIRequestUtils @@ -51,6 +53,23 @@ class BaseResponsesAPIStreamingIterator: self.litellm_metadata = litellm_metadata self.custom_llm_provider = custom_llm_provider + # set hidden params for response headers (e.g., x-litellm-model-id) + # This matches ths stream wrapper in litellm/litellm_core_utils/streaming_handler.py + _api_base = get_api_base( + model=model or "", + optional_params=self.logging_obj.model_call_details.get( + "litellm_params", {} + ), + ) + _model_info: Dict = litellm_metadata.get("model_info", {}) if litellm_metadata else {} + self._hidden_params = { + "model_id": _model_info.get("id", None), + "api_base": _api_base, + } + self._hidden_params["additional_headers"] = process_response_headers( + self.response.headers or {} + ) # GUARANTEE OPENAI HEADERS IN RESPONSE + def _process_chunk(self, chunk) -> Optional[ResponsesAPIStreamingResponse]: """Process a single chunk of data from the stream""" if not chunk: diff --git a/litellm/types/guardrails.py b/litellm/types/guardrails.py index 5f6295e151f..24a235def59 100644 --- a/litellm/types/guardrails.py +++ b/litellm/types/guardrails.py @@ -14,6 +14,9 @@ from litellm.types.proxy.guardrails.guardrail_hooks.grayswan import ( from litellm.types.proxy.guardrails.guardrail_hooks.ibm import ( IBMGuardrailsBaseConfigModel, ) +from litellm.types.proxy.guardrails.guardrail_hooks.tool_permission import ( + ToolPermissionGuardrailConfigModel, +) """ @@ -415,18 +418,6 @@ class NomaGuardrailConfigModel(BaseModel): ) -class ToolPermissionGuardrailConfigModel(BaseModel): - """Configuration parameters for the Tool Permission guardrail""" - - rules: Optional[List[Dict]] = Field( - default=None, description="List of permission rules for tool usage" - ) - default_action: Optional[str] = Field( - default="Deny", - description="Default action when no rule matches (Allow or Deny)", - ) - - class ZscalerAIGuardConfigModel(BaseModel): """Configuration parameters for the Zscaler AI Guard guardrail""" diff --git a/litellm/types/litellm_core_utils/streaming_chunk_builder_utils.py b/litellm/types/litellm_core_utils/streaming_chunk_builder_utils.py index aa879e14c34..a1f89dac5cf 100644 --- a/litellm/types/litellm_core_utils/streaming_chunk_builder_utils.py +++ b/litellm/types/litellm_core_utils/streaming_chunk_builder_utils.py @@ -2,7 +2,7 @@ from typing import TYPE_CHECKING, Optional from typing_extensions import TypedDict -from ..utils import CompletionTokensDetails, PromptTokensDetailsWrapper +from ..utils import CompletionTokensDetails, PromptTokensDetailsWrapper, ServerToolUse class UsagePerChunk(TypedDict): @@ -10,6 +10,7 @@ class UsagePerChunk(TypedDict): completion_tokens: int cache_creation_input_tokens: Optional[int] cache_read_input_tokens: Optional[int] + server_tool_use: Optional[ServerToolUse] web_search_requests: Optional[int] completion_tokens_details: Optional[CompletionTokensDetails] prompt_tokens_details: Optional[PromptTokensDetailsWrapper] diff --git a/litellm/types/llms/anthropic.py b/litellm/types/llms/anthropic.py index fc210a7d084..507b382f785 100644 --- a/litellm/types/llms/anthropic.py +++ b/litellm/types/llms/anthropic.py @@ -36,12 +36,20 @@ class AnthropicOutputSchema(TypedDict, total=False): schema: Required[dict] +class AnthropicOutputConfig(TypedDict, total=False): + """Configuration for controlling Claude's output behavior.""" + effort: Literal["high", "medium", "low"] + + class AnthropicMessagesTool(TypedDict, total=False): name: Required[str] description: str input_schema: Optional[AnthropicInputSchema] type: Literal["custom"] cache_control: Optional[Union[dict, ChatCompletionCachedContent]] + defer_loading: bool + allowed_callers: Optional[List[str]] + input_examples: Optional[List[Dict[str, Any]]] class AnthropicComputerTool(TypedDict, total=False): @@ -67,24 +75,78 @@ class AnthropicWebSearchTool(TypedDict, total=False): cache_control: Optional[Union[dict, ChatCompletionCachedContent]] max_uses: Optional[int] user_location: Optional[AnthropicWebSearchUserLocation] + defer_loading: Optional[bool] + allowed_callers: Optional[List[str]] + input_examples: Optional[List[Dict[str, Any]]] class AnthropicHostedTools(TypedDict, total=False): # for bash_tool and text_editor type: Required[str] name: Required[str] cache_control: Optional[Union[dict, ChatCompletionCachedContent]] + defer_loading: Optional[bool] + allowed_callers: Optional[List[str]] + input_examples: Optional[List[Dict[str, Any]]] class AnthropicCodeExecutionTool(TypedDict, total=False): type: Required[str] name: Required[Literal["code_execution"]] cache_control: Optional[Union[dict, ChatCompletionCachedContent]] + defer_loading: Optional[bool] + allowed_callers: Optional[List[str]] + input_examples: Optional[List[Dict[str, Any]]] class AnthropicMemoryTool(TypedDict, total=False): type: Required[str] name: Required[Literal["memory"]] cache_control: Optional[Union[dict, ChatCompletionCachedContent]] + defer_loading: Optional[bool] + allowed_callers: Optional[List[str]] + input_examples: Optional[List[Dict[str, Any]]] + + +class AnthropicToolSearchToolRegex(TypedDict, total=False): + """Tool search tool using regex patterns for tool discovery.""" + type: Required[Literal["tool_search_tool_regex_20251119"]] + name: Required[str] + + +class AnthropicToolSearchToolBM25(TypedDict, total=False): + """Tool search tool using BM25 algorithm for tool discovery.""" + type: Required[Literal["tool_search_tool_bm25_20251119"]] + name: Required[str] + cache_control: Optional[Union[dict, ChatCompletionCachedContent]] + defer_loading: Optional[bool] + allowed_callers: Optional[List[str]] + input_examples: Optional[List[Dict[str, Any]]] + + +class ToolReference(TypedDict, total=False): + """Reference to a tool that should be expanded from deferred tools.""" + type: Required[Literal["tool_reference"]] + tool_name: Required[str] + + +class DirectToolCaller(TypedDict, total=False): + """Indicates a tool was called directly by Claude.""" + type: Required[Literal["direct"]] + + +class CodeExecutionToolCaller(TypedDict, total=False): + """Indicates a tool was called programmatically from code execution.""" + type: Required[Literal["code_execution_20250825"]] + tool_id: Required[str] # ID of the code execution tool that made the call + + +ToolCaller = Union[DirectToolCaller, CodeExecutionToolCaller] + + +class AnthropicContainer(TypedDict, total=False): + """Container metadata for code execution.""" + id: Required[str] + expires_at: Optional[str] # ISO 8601 timestamp AllAnthropicToolsValues = Union[ @@ -94,6 +156,8 @@ AllAnthropicToolsValues = Union[ AnthropicWebSearchTool, AnthropicCodeExecutionTool, AnthropicMemoryTool, + AnthropicToolSearchToolRegex, + AnthropicToolSearchToolBM25, ] @@ -121,6 +185,7 @@ class AnthropicMessagesToolUseParam(TypedDict, total=False): name: str input: dict cache_control: Optional[Union[dict, ChatCompletionCachedContent]] + caller: Optional[ToolCaller] AnthropicMessagesAssistantMessageValues = Union[ @@ -372,6 +437,7 @@ class ToolUseBlock(TypedDict): name: str type: Literal["tool_use"] + caller: Optional[ToolCaller] class TextBlock(TypedDict): @@ -565,3 +631,11 @@ class ANTHROPIC_BETA_HEADER_VALUES(str, Enum): WEB_FETCH_2025_09_10 = "web-fetch-2025-09-10" CONTEXT_MANAGEMENT_2025_06_27 = "context-management-2025-06-27" STRUCTURED_OUTPUT_2025_09_25 = "structured-outputs-2025-11-13" + ADVANCED_TOOL_USE_2025_11_20 = "advanced-tool-use-2025-11-20" + + +# Tool search beta header constant +ANTHROPIC_TOOL_SEARCH_BETA_HEADER = "advanced-tool-use-2025-11-20" + +# Effort beta header constant +ANTHROPIC_EFFORT_BETA_HEADER = "effort-2025-11-24" diff --git a/litellm/types/llms/openai.py b/litellm/types/llms/openai.py index c29b2f32ea5..61d58e4c86d 100644 --- a/litellm/types/llms/openai.py +++ b/litellm/types/llms/openai.py @@ -1,6 +1,17 @@ from enum import Enum from os import PathLike -from typing import IO, Any, Dict, Iterable, List, Literal, Mapping, Optional, Tuple, Union +from typing import ( + IO, + Any, + Dict, + Iterable, + List, + Literal, + Mapping, + Optional, + Tuple, + Union, +) import httpx from openai._legacy_response import ( diff --git a/litellm/types/llms/vertex_ai.py b/litellm/types/llms/vertex_ai.py index 7a4a4723abd..5f00edc1ffa 100644 --- a/litellm/types/llms/vertex_ai.py +++ b/litellm/types/llms/vertex_ai.py @@ -183,8 +183,13 @@ GeminiResponseModalities = Literal["TEXT", "IMAGE", "AUDIO", "VIDEO"] GeminiImageAspectRatio = Literal["1:1", "2:3", "3:2", "3:4", "4:3", "9:16", "16:9", "21:9"] +GeminiImageSize = Literal["1K", "2K", "4K"] + + class GeminiImageConfig(TypedDict, total=False): aspectRatio: GeminiImageAspectRatio + imageSize: GeminiImageSize + class PrebuiltVoiceConfig(TypedDict): voiceName: str diff --git a/litellm/types/proxy/guardrails/guardrail_hooks/tool_permission.py b/litellm/types/proxy/guardrails/guardrail_hooks/tool_permission.py index b2248c51930..e78cfad8bdb 100644 --- a/litellm/types/proxy/guardrails/guardrail_hooks/tool_permission.py +++ b/litellm/types/proxy/guardrails/guardrail_hooks/tool_permission.py @@ -1,8 +1,10 @@ # Tool Permission Guardrail Type Definitions -from typing import Dict, Literal, Optional +from typing import Dict, List, Literal, Optional from pydantic import BaseModel, Field +from .base import GuardrailConfigModel + class ToolPermissionRule(BaseModel): """ @@ -43,3 +45,23 @@ class PermissionError(BaseModel): tool_name: str = Field(description="Name of the denied tool") rule_id: Optional[str] = Field(description="ID of the rule that caused denial") message: str = Field(description="Error message") + + +class ToolPermissionGuardrailConfigModel(GuardrailConfigModel): + """Configuration parameters exposed to the UI for the Tool Permission guardrail.""" + + rules: Optional[List[ToolPermissionRule]] = Field( + default=None, + description="Ordered allow/deny rules. Patterns support * wildcards and optional regex constraints on tool arguments.", + ) + default_action: Literal["allow", "deny"] = Field( + default="deny", description="Fallback decision when no rule matches" + ) + on_disallowed_action: Literal["block", "rewrite"] = Field( + default="block", + description="Choose whether disallowed tools block the request or get rewritten out of the payload", + ) + + @staticmethod + def ui_friendly_name() -> str: + return "LiteLLM Tool Permission Guardrail" diff --git a/litellm/types/rag.py b/litellm/types/rag.py new file mode 100644 index 00000000000..252530c7163 --- /dev/null +++ b/litellm/types/rag.py @@ -0,0 +1,146 @@ +""" +Type definitions for RAG (Retrieval Augmented Generation) Ingest API. +""" + +from typing import Any, Dict, List, Literal, Optional, Union + +from pydantic import BaseModel +from typing_extensions import TypedDict + + +class RAGChunkingStrategy(TypedDict, total=False): + """ + Chunking strategy config for RAG ingest using RecursiveCharacterTextSplitter. + + See: https://docs.langchain.com/oss/python/langchain/rag + """ + + chunk_size: int # Maximum size of chunks (default: 1000) + chunk_overlap: int # Overlap between chunks (default: 200) + separators: Optional[List[str]] # Custom separators for splitting + + +class RAGIngestOCROptions(TypedDict, total=False): + """OCR configuration for RAG ingest pipeline.""" + + model: str # e.g., "mistral/mistral-ocr-latest" + + +class RAGIngestEmbeddingOptions(TypedDict, total=False): + """Embedding configuration for RAG ingest pipeline.""" + + model: str # e.g., "text-embedding-3-small" + + +class OpenAIVectorStoreOptions(TypedDict, total=False): + """ + OpenAI vector store configuration. + + Example (auto-create): + {"custom_llm_provider": "openai"} + + Example (use existing): + {"custom_llm_provider": "openai", "vector_store_id": "vs_xxx"} + """ + + custom_llm_provider: Literal["openai"] + vector_store_id: Optional[str] # Existing VS ID (auto-creates if not provided) + ttl_days: Optional[int] # Time-to-live in days for indexed content + + +class BedrockVectorStoreOptions(TypedDict, total=False): + """ + Bedrock Knowledge Base configuration. + + Example (auto-create KB and all resources): + {"custom_llm_provider": "bedrock"} + + Example (use existing KB): + {"custom_llm_provider": "bedrock", "vector_store_id": "KB_ID"} + + Auto-creation creates: S3 bucket, OpenSearch Serverless collection, + IAM role, Knowledge Base, and Data Source. + """ + + custom_llm_provider: Literal["bedrock"] + vector_store_id: Optional[str] # Existing KB ID (auto-creates if not provided) + + # Bedrock-specific options + s3_bucket: Optional[str] # S3 bucket (auto-created if not provided) + s3_prefix: Optional[str] # S3 key prefix (default: "data/") + embedding_model: Optional[str] # Embedding model (default: amazon.titan-embed-text-v2:0) + data_source_id: Optional[str] # For existing KB: override auto-detected DS + wait_for_ingestion: Optional[bool] # Wait for completion (default: False - returns immediately) + ingestion_timeout: Optional[int] # Timeout in seconds if wait_for_ingestion=True (default: 300) + + # AWS auth (uses BaseAWSLLM) + aws_access_key_id: Optional[str] + aws_secret_access_key: Optional[str] + aws_session_token: Optional[str] + aws_region_name: Optional[str] # default: us-west-2 + aws_role_name: Optional[str] + aws_session_name: Optional[str] + aws_profile_name: Optional[str] + aws_web_identity_token: Optional[str] + aws_sts_endpoint: Optional[str] + aws_external_id: Optional[str] + + +# Union type for vector store options +RAGIngestVectorStoreOptions = Union[OpenAIVectorStoreOptions, BedrockVectorStoreOptions] + + +class RAGIngestOptions(TypedDict, total=False): + """ + Combined options for RAG ingest pipeline. + + Unified interface - just specify custom_llm_provider: + + Example (OpenAI): + from litellm.types.rag import RAGIngestOptions, OpenAIVectorStoreOptions + + options: RAGIngestOptions = { + "vector_store": OpenAIVectorStoreOptions( + custom_llm_provider="openai", + vector_store_id="vs_xxx", # optional + ) + } + + Example (Bedrock): + from litellm.types.rag import RAGIngestOptions, BedrockVectorStoreOptions + + options: RAGIngestOptions = { + "vector_store": BedrockVectorStoreOptions( + custom_llm_provider="bedrock", + vector_store_id="KB_ID", # optional - auto-creates if not provided + wait_for_ingestion=True, + ) + } + """ + + name: Optional[str] # Optional pipeline name for logging + ocr: Optional[RAGIngestOCROptions] # Optional OCR step + chunking_strategy: Optional[RAGChunkingStrategy] # RecursiveCharacterTextSplitter args + embedding: Optional[RAGIngestEmbeddingOptions] # Embedding model config + vector_store: RAGIngestVectorStoreOptions # OpenAI or Bedrock config + +class RAGIngestResponse(TypedDict, total=False): + """Response from RAG ingest API.""" + + id: str # Unique ingest job ID + status: Literal["completed", "in_progress", "failed"] + vector_store_id: str # The vector store ID (created or existing) + file_id: Optional[str] # The file ID in the vector store + + + +class RAGIngestRequest(BaseModel): + """Request body for RAG ingest API (for validation).""" + + file_url: Optional[str] = None # URL to fetch file from + file_id: Optional[str] = None # Existing file ID + ingest_options: Dict[str, Any] # RAGIngestOptions as dict for flexibility + + class Config: + extra = "allow" # Allow additional fields + diff --git a/litellm/types/router.py b/litellm/types/router.py index 2bf126211c3..002792d0490 100644 --- a/litellm/types/router.py +++ b/litellm/types/router.py @@ -159,6 +159,7 @@ class CredentialLiteLLMParams(BaseModel): aws_access_key_id: Optional[str] = None aws_secret_access_key: Optional[str] = None aws_region_name: Optional[str] = None + aws_bedrock_runtime_endpoint: Optional[str] = None ## IBM WATSONX ## watsonx_region_name: Optional[str] = None diff --git a/litellm/types/utils.py b/litellm/types/utils.py index c868af85d2a..b0d081d8f87 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -999,7 +999,8 @@ class PromptTokensDetailsWrapper( class ServerToolUse(BaseModel): - web_search_requests: Optional[int] + web_search_requests: Optional[int] = None + tool_search_requests: Optional[int] = None class Usage(CompletionUsage): @@ -2574,6 +2575,7 @@ class LlmProviders(str, Enum): AZURE = "azure" AZURE_TEXT = "azure_text" AZURE_AI = "azure_ai" + AZURE_ANTHROPIC = "azure_anthropic" SAGEMAKER = "sagemaker" SAGEMAKER_CHAT = "sagemaker_chat" BEDROCK = "bedrock" diff --git a/litellm/utils.py b/litellm/utils.py index f1f091b1719..1d8f40af412 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -2631,16 +2631,7 @@ def get_optional_params_image_gen( ): optional_params = non_default_params elif custom_llm_provider == "bedrock": - # use stability3 config class if model is a stability3 model - config_class = ( - litellm.AmazonStability3Config - if litellm.AmazonStability3Config._is_stability_3_model(model=model) - else ( - litellm.AmazonNovaCanvasConfig - if litellm.AmazonNovaCanvasConfig._is_nova_model(model=model) - else litellm.AmazonStabilityConfig - ) - ) + config_class = litellm.BedrockImageGeneration.get_config_class(model=model) supported_params = config_class.get_supported_openai_params(model=model) _check_valid_arg(supported_params=supported_params) optional_params = config_class.map_openai_params( @@ -3728,7 +3719,17 @@ def get_optional_params( # noqa: PLR0915 else False ), ) - + elif bedrock_route == "openai": + optional_params = litellm.AmazonBedrockOpenAIConfig().map_openai_params( + model=model, + non_default_params=non_default_params, + optional_params=optional_params, + drop_params=( + drop_params + if drop_params is not None and isinstance(drop_params, bool) + else False + ), + ) elif "anthropic" in bedrock_base_model and bedrock_route == "invoke": if bedrock_base_model.startswith("anthropic.claude-3"): optional_params = ( @@ -6867,7 +6868,7 @@ def convert_to_dict(message: Union[BaseModel, dict]) -> dict: dict: The converted message. """ if isinstance(message, BaseModel): - return message.model_dump(exclude_none=True) + return message.model_dump(exclude_none=True) # type: ignore elif isinstance(message, dict): return message else: @@ -7150,6 +7151,8 @@ class ProviderConfigManager: return litellm.AzureAIStudioConfig() elif litellm.LlmProviders.AZURE_TEXT == provider: return litellm.AzureOpenAITextConfig() + elif litellm.LlmProviders.AZURE_ANTHROPIC == provider: + return litellm.AzureAnthropicConfig() elif litellm.LlmProviders.HOSTED_VLLM == provider: return litellm.HostedVLLMChatConfig() elif litellm.LlmProviders.NLP_CLOUD == provider: @@ -7343,6 +7346,12 @@ class ProviderConfigManager: ) return VertexAIPartnerModelsAnthropicMessagesConfig() + elif litellm.LlmProviders.AZURE_ANTHROPIC == provider: + from litellm.llms.azure.anthropic.messages_transformation import ( + AzureAnthropicMessagesConfig, + ) + + return AzureAnthropicMessagesConfig() return None @staticmethod @@ -7604,6 +7613,12 @@ class ProviderConfigManager: ) return MilvusVectorStoreConfig() + elif litellm.LlmProviders.GEMINI == provider: + from litellm.llms.gemini.vector_stores.transformation import ( + GeminiVectorStoreConfig, + ) + + return GeminiVectorStoreConfig() return None @staticmethod @@ -7689,6 +7704,12 @@ class ProviderConfigManager: ) return get_runwayml_image_generation_config(model) + elif LlmProviders.VERTEX_AI == provider: + from litellm.llms.vertex_ai.image_generation import ( + get_vertex_ai_image_generation_config, + ) + + return get_vertex_ai_image_generation_config(model) return None @staticmethod @@ -7865,6 +7886,12 @@ class ProviderConfigManager: ) return AzureAVATextToSpeechConfig() + elif litellm.LlmProviders.ELEVENLABS == provider: + from litellm.llms.elevenlabs.text_to_speech.transformation import ( + ElevenLabsTextToSpeechConfig, + ) + + return ElevenLabsTextToSpeechConfig() elif litellm.LlmProviders.RUNWAYML == provider: from litellm.llms.runwayml.text_to_speech.transformation import ( RunwayMLTextToSpeechConfig, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 3b1a31d5018..1fe299815cc 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -1143,6 +1143,60 @@ "output_cost_per_token": 1.5e-05, "supports_function_calling": true }, + "azure/claude-haiku-4-5": { + "input_cost_per_token": 1e-06, + "litellm_provider": "azure_anthropic", + "max_input_tokens": 200000, + "max_output_tokens": 64000, + "max_tokens": 64000, + "mode": "chat", + "output_cost_per_token": 5e-06, + "supports_assistant_prefill": true, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "azure/claude-opus-4-1": { + "input_cost_per_token": 1.5e-05, + "litellm_provider": "azure_anthropic", + "max_input_tokens": 200000, + "max_output_tokens": 32000, + "max_tokens": 32000, + "mode": "chat", + "output_cost_per_token": 7.5e-05, + "supports_assistant_prefill": true, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "azure/claude-sonnet-4-5": { + "input_cost_per_token": 3e-06, + "litellm_provider": "azure_anthropic", + "max_input_tokens": 200000, + "max_output_tokens": 64000, + "max_tokens": 64000, + "mode": "chat", + "output_cost_per_token": 1.5e-05, + "supports_assistant_prefill": true, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, "azure/computer-use-preview": { "input_cost_per_token": 3e-06, "litellm_provider": "azure", @@ -9468,6 +9522,15 @@ "output_cost_per_token": 0.0, "supports_embedding_image_input": true }, + "embed-multilingual-light-v3.0": { + "input_cost_per_token": 1e-04, + "litellm_provider": "cohere", + "max_input_tokens": 1024, + "max_tokens": 1024, + "mode": "embedding", + "output_cost_per_token": 0.0, + "supports_embedding_image_input": true + }, "eu.amazon.nova-lite-v1:0": { "input_cost_per_token": 7.8e-08, "litellm_provider": "bedrock_converse", @@ -24551,6 +24614,58 @@ "supports_tool_choice": true, "supports_vision": true }, + "vertex_ai/claude-opus-4-5": { + "cache_creation_input_token_cost": 6.25e-06, + "cache_read_input_token_cost": 5e-07, + "input_cost_per_token": 5e-06, + "litellm_provider": "vertex_ai-anthropic_models", + "max_input_tokens": 200000, + "max_output_tokens": 64000, + "max_tokens": 64000, + "mode": "chat", + "output_cost_per_token": 2.5e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_assistant_prefill": true, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "tool_use_system_prompt_tokens": 159 + }, + "vertex_ai/claude-opus-4-5@20251101": { + "cache_creation_input_token_cost": 6.25e-06, + "cache_read_input_token_cost": 5e-07, + "input_cost_per_token": 5e-06, + "litellm_provider": "vertex_ai-anthropic_models", + "max_input_tokens": 200000, + "max_output_tokens": 64000, + "max_tokens": 64000, + "mode": "chat", + "output_cost_per_token": 2.5e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_assistant_prefill": true, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "tool_use_system_prompt_tokens": 159 + }, "vertex_ai/claude-sonnet-4-5": { "cache_creation_input_token_cost": 3.75e-06, "cache_read_input_token_cost": 3e-07, diff --git a/pyproject.toml b/pyproject.toml index d485772b36e..f32a2d4d28f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -67,6 +67,7 @@ polars = {version = "^1.31.0", optional = true, python = ">=3.10"} semantic-router = {version = ">=0.1.12", optional = true, python = ">=3.9,<3.14"} mlflow = {version = ">3.1.4", optional = true, python = ">=3.10"} soundfile = {version = "^0.12.1", optional = true} +grpcio = ">=1.62.3,<1.68.0" # Constrain to < 1.68.0 to avoid resource exhausted bug (https://github.com/grpc/grpc/issues/38290). Minimum 1.62.3 required by grpcio-status. [tool.poetry.extras] proxy = [ diff --git a/requirements.txt b/requirements.txt index 3a426d83e31..08129155d7f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -39,6 +39,7 @@ azure-storage-file-datalake==12.20.0 # for azure buck storage logging opentelemetry-api==1.25.0 opentelemetry-sdk==1.25.0 opentelemetry-exporter-otlp==1.25.0 +grpcio>=1.62.3,<1.68.0 # Constraint for opentelemetry-exporter-otlp-proto-grpc to avoid resource exhausted bug (https://github.com/grpc/grpc/issues/38290) sentry_sdk==2.21.0 # for sentry error handling detect-secrets==1.5.0 # Enterprise - secret detection / masking in LLM requests cryptography==44.0.1 diff --git a/tests/code_coverage_tests/recursive_detector.py b/tests/code_coverage_tests/recursive_detector.py index cac245f8d5b..c26d1669a81 100644 --- a/tests/code_coverage_tests/recursive_detector.py +++ b/tests/code_coverage_tests/recursive_detector.py @@ -32,6 +32,9 @@ IGNORE_FUNCTIONS = [ "_redact_base64", # max depth set. "_contains_vision_content", # max depth set. "_read_all_bytes", # max depth set. + "_fix_enum_types", # max depth set. + "_collect_argument_paths", # max depth set. + "_split_text", # max depth set. ] diff --git a/tests/image_gen_tests/test_bedrock_image_gen_unit_tests.py b/tests/image_gen_tests/test_bedrock_image_gen_unit_tests.py index d3a5ade1cef..5526f22cd5e 100644 --- a/tests/image_gen_tests/test_bedrock_image_gen_unit_tests.py +++ b/tests/image_gen_tests/test_bedrock_image_gen_unit_tests.py @@ -119,6 +119,20 @@ def test_transform_response_dict_to_openai_response(): assert [img.b64_json for img in result.data] == response_dict["images"] +def test_transform_response_dict_to_openai_response_from_stability_3_models_with_no_null_finish_reason(): + # Create a mock response + response_dict = {"finish_reasons": ["Filter reason: prompt"]} + model_response = ImageResponse() + + with pytest.raises(BedrockError) as exc_info: + AmazonStability3Config.transform_response_dict_to_openai_response( + model_response, response_dict + ) + + assert exc_info.value.status_code == 400 + assert exc_info.value.message == "Filter reason: prompt" + + def test_amazon_stability_get_supported_openai_params(): result = AmazonStabilityConfig.get_supported_openai_params() assert result == ["size"] @@ -168,7 +182,7 @@ def test_get_request_body_stability3(): model = "stability.sd3-large" result = handler._get_request_body( - model=model, bedrock_provider=None, prompt=prompt, optional_params=optional_params + model=model, prompt=prompt, optional_params=optional_params ) assert result["prompt"] == prompt @@ -181,7 +195,7 @@ def test_get_request_body_stability(): model = "stability.stable-diffusion-xl-v1" result = handler._get_request_body( - model=model, bedrock_provider=None, prompt=prompt, optional_params=optional_params + model=model, prompt=prompt, optional_params=optional_params ) assert result["text_prompts"][0]["text"] == prompt @@ -239,7 +253,7 @@ def test_get_request_body_nova_canvas_default(): model = "amazon.nova-canvas-v1" result = handler._get_request_body( - model=model, bedrock_provider=None, prompt=prompt, optional_params=optional_params + model=model, prompt=prompt, optional_params=optional_params ) assert result["taskType"] == "TEXT_IMAGE" @@ -254,7 +268,7 @@ def test_get_request_body_nova_canvas_text_image(): model = "amazon.nova-canvas-v1" result = handler._get_request_body( - model=model, bedrock_provider=None, prompt=prompt, optional_params=optional_params + model=model, prompt=prompt, optional_params=optional_params ) assert result["taskType"] == "TEXT_IMAGE" @@ -273,7 +287,7 @@ def test_get_request_body_nova_canvas_color_guided_generation(): model = "amazon.nova-canvas-v1" result = handler._get_request_body( - model=model, bedrock_provider=None, prompt=prompt, optional_params=optional_params + model=model, prompt=prompt, optional_params=optional_params ) assert result["taskType"] == "COLOR_GUIDED_GENERATION" @@ -437,7 +451,7 @@ def test_get_request_body_nova_canvas_inference_profile_arn(): bedrock_provider = handler.get_bedrock_invoke_provider(model=nova_model) result = handler._get_request_body( - model=nova_model, bedrock_provider=bedrock_provider, prompt=prompt, optional_params=optional_params + model=nova_model, prompt=prompt, optional_params=optional_params ) assert result["taskType"] == "TEXT_IMAGE" @@ -453,7 +467,7 @@ def test_get_request_body_nova_canvas_with_model_id_param(): model = "amazon.nova-canvas-v1" result = handler._get_request_body( - model=model, bedrock_provider=None, prompt=prompt, optional_params=optional_params + model=model, prompt=prompt, optional_params=optional_params ) # After fix, model_id should not appear in the result @@ -488,12 +502,9 @@ def test_get_request_body_cross_region_inference_profile(): # Cross-region inference profile format model = "us.amazon.nova-canvas-v1:0" - # Get the provider using the method from the handler - bedrock_provider = handler.get_bedrock_invoke_provider(model=model) - # This should work after the fix - cross-region format should be detected as 'nova' result = handler._get_request_body( - model=model, bedrock_provider=bedrock_provider, prompt=prompt, optional_params=optional_params + model=model, prompt=prompt, optional_params=optional_params ) assert result["taskType"] == "TEXT_IMAGE" @@ -508,7 +519,7 @@ def test_backward_compatibility_regular_nova_model(): model = "amazon.nova-canvas-v1" result = handler._get_request_body( - model=model, bedrock_provider=None, prompt=prompt, optional_params=optional_params + model=model, prompt=prompt, optional_params=optional_params ) assert result["taskType"] == "TEXT_IMAGE" diff --git a/tests/image_gen_tests/test_image_generation.py b/tests/image_gen_tests/test_image_generation.py index 24d5843293b..add60c755be 100644 --- a/tests/image_gen_tests/test_image_generation.py +++ b/tests/image_gen_tests/test_image_generation.py @@ -119,6 +119,38 @@ class TestVertexImageGeneration(BaseImageGenTest): } +class TestVertexAIGeminiImageGeneration(BaseImageGenTest): + """Test Gemini image generation models (Nano Banana)""" + def get_base_image_generation_call_args(self) -> dict: + # comment this when running locally + load_vertex_ai_credentials() + + litellm.in_memory_llm_clients_cache = InMemoryCache() + return { + "model": "vertex_ai/gemini-2.5-flash-image", + "vertex_ai_project": "pathrise-convert-1606954137718", + "vertex_ai_location": "us-central1", + "n": 1, + "size": "1024x1024", + } + + +class TestVertexAIGemini3ProImageGeneration(BaseImageGenTest): + """Test Gemini 3 Pro image generation model""" + def get_base_image_generation_call_args(self) -> dict: + # comment this when running locally + load_vertex_ai_credentials() + + litellm.in_memory_llm_clients_cache = InMemoryCache() + return { + "model": "vertex_ai/gemini-3-pro-image-preview", + "vertex_ai_project": "pathrise-convert-1606954137718", + "vertex_ai_location": "us-central1", + "n": 1, + "size": "1024x1024", + } + + class TestBedrockNovaCanvasTextToImage(BaseImageGenTest): def get_base_image_generation_call_args(self) -> dict: litellm.in_memory_llm_clients_cache = InMemoryCache() diff --git a/tests/litellm-proxy-extras/test_litellm_proxy_extras_utils.py b/tests/litellm-proxy-extras/test_litellm_proxy_extras_utils.py index 3bcb5b25da8..5714cd5c487 100644 --- a/tests/litellm-proxy-extras/test_litellm_proxy_extras_utils.py +++ b/tests/litellm-proxy-extras/test_litellm_proxy_extras_utils.py @@ -1,11 +1,5 @@ -import json import os import sys -import httpx -import pytest -import respx - -from fastapi.testclient import TestClient sys.path.insert( 0, os.path.abspath("../..") @@ -13,8 +7,10 @@ sys.path.insert( from litellm_proxy_extras.utils import ProxyExtrasDBManager + def test_custom_prisma_dir(monkeypatch): import tempfile + # create a temp directory temp_dir = tempfile.mkdtemp() monkeypatch.setenv("LITELLM_MIGRATION_DIR", temp_dir) @@ -30,3 +26,102 @@ def test_custom_prisma_dir(monkeypatch): migrations_dir = os.path.join(temp_dir, "migrations") assert os.path.exists(migrations_dir) + +class TestPermissionErrorDetection: + """Test cases for permission error detection in Prisma migrations""" + + def test_is_permission_error_postgres_42501(self): + """Test detection of PostgreSQL 42501 error code (insufficient privilege)""" + error_message = "Database error code: 42501 - permission denied for table users" + assert ProxyExtrasDBManager._is_permission_error(error_message) is True + + def test_is_permission_error_must_be_owner(self): + """Test detection of 'must be owner of table' error""" + error_message = "ERROR: must be owner of table my_table" + assert ProxyExtrasDBManager._is_permission_error(error_message) is True + + def test_is_permission_error_permission_denied_schema(self): + """Test detection of 'permission denied for schema' error""" + error_message = "permission denied for schema public" + assert ProxyExtrasDBManager._is_permission_error(error_message) is True + + def test_is_permission_error_permission_denied_table(self): + """Test detection of 'permission denied for table' error""" + error_message = "permission denied for table my_table" + assert ProxyExtrasDBManager._is_permission_error(error_message) is True + + def test_is_permission_error_must_be_owner_schema(self): + """Test detection of 'must be owner of schema' error""" + error_message = "must be owner of schema public" + assert ProxyExtrasDBManager._is_permission_error(error_message) is True + + def test_is_permission_error_case_insensitive(self): + """Test that permission error detection is case insensitive""" + error_message = "PERMISSION DENIED FOR TABLE my_table" + assert ProxyExtrasDBManager._is_permission_error(error_message) is True + + def test_is_permission_error_negative(self): + """Test that non-permission errors are not detected as permission errors""" + error_message = "column 'id' already exists" + assert ProxyExtrasDBManager._is_permission_error(error_message) is False + + +class TestIdempotentErrorDetection: + """Test cases for idempotent error detection in Prisma migrations""" + + def test_is_idempotent_error_already_exists(self): + """Test detection of generic 'already exists' error""" + error_message = "object already exists" + assert ProxyExtrasDBManager._is_idempotent_error(error_message) is True + + def test_is_idempotent_error_column_already_exists(self): + """Test detection of 'column already exists' error""" + error_message = "column 'email' already exists" + assert ProxyExtrasDBManager._is_idempotent_error(error_message) is True + + def test_is_idempotent_error_duplicate_key(self): + """Test detection of duplicate key violation error""" + error_message = "duplicate key value violates unique constraint" + assert ProxyExtrasDBManager._is_idempotent_error(error_message) is True + + def test_is_idempotent_error_relation_already_exists(self): + """Test detection of 'relation already exists' error""" + error_message = "relation 'users_pkey' already exists" + assert ProxyExtrasDBManager._is_idempotent_error(error_message) is True + + def test_is_idempotent_error_constraint_already_exists(self): + """Test detection of 'constraint already exists' error""" + error_message = "constraint 'fk_user_id' already exists" + assert ProxyExtrasDBManager._is_idempotent_error(error_message) is True + + def test_is_idempotent_error_case_insensitive(self): + """Test that idempotent error detection is case insensitive""" + error_message = "COLUMN 'ID' ALREADY EXISTS" + assert ProxyExtrasDBManager._is_idempotent_error(error_message) is True + + def test_is_idempotent_error_negative(self): + """Test that non-idempotent errors are not detected as idempotent errors""" + error_message = "Database error code: 42501 - permission denied" + assert ProxyExtrasDBManager._is_idempotent_error(error_message) is False + + +class TestErrorClassificationPriority: + """Test cases to ensure errors are correctly classified""" + + def test_permission_error_not_classified_as_idempotent(self): + """Ensure permission errors are not mistakenly classified as idempotent""" + error_message = "Database error code: 42501 - must be owner of table users" + assert ProxyExtrasDBManager._is_permission_error(error_message) is True + assert ProxyExtrasDBManager._is_idempotent_error(error_message) is False + + def test_idempotent_error_not_classified_as_permission(self): + """Ensure idempotent errors are not mistakenly classified as permission errors""" + error_message = "column 'created_at' already exists" + assert ProxyExtrasDBManager._is_idempotent_error(error_message) is True + assert ProxyExtrasDBManager._is_permission_error(error_message) is False + + def test_unknown_error_classified_as_neither(self): + """Ensure unknown errors are classified as neither permission nor idempotent""" + error_message = "connection timeout" + assert ProxyExtrasDBManager._is_permission_error(error_message) is False + assert ProxyExtrasDBManager._is_idempotent_error(error_message) is False diff --git a/tests/litellm/llms/vertex_ai/gemini/test_transformation.py b/tests/litellm/llms/vertex_ai/gemini/test_transformation.py index a0f24376614..6d005af28ac 100644 --- a/tests/litellm/llms/vertex_ai/gemini/test_transformation.py +++ b/tests/litellm/llms/vertex_ai/gemini/test_transformation.py @@ -191,4 +191,38 @@ async def test__transform_request_body_image_config_snake_case(): assert "generationConfig" in rb assert "image_config" in rb["generationConfig"] - assert rb["generationConfig"]["image_config"] == {"aspect_ratio": "16:9"} \ No newline at end of file + assert rb["generationConfig"]["image_config"] == {"aspect_ratio": "16:9"} + + +@pytest.mark.asyncio +async def test__transform_request_body_image_config_with_image_size(): + """Test imageSize parameter support in imageConfig""" + model = "gemini-3-pro-image-preview" + messages = [ + { + "role": "user", + "content": [ + {"type": "text", "text": "Generate a 4K image of Tokyo skyline"} + ] + } + ] + optional_params = { + "imageConfig": {"aspectRatio": "16:9", "imageSize": "4K"}, + "responseModalities": ["Image"] + } + litellm_params = {} + transform_request_params = { + "messages": messages, + "model": model, + "optional_params": optional_params, + "custom_llm_provider": "gemini", + "litellm_params": litellm_params, + "cached_content": None, + } + + rb: RequestBody = transformation._transform_request_body(**transform_request_params) + + assert "generationConfig" in rb + assert "imageConfig" in rb["generationConfig"] + assert rb["generationConfig"]["imageConfig"]["aspectRatio"] == "16:9" + assert rb["generationConfig"]["imageConfig"]["imageSize"] == "4K" \ No newline at end of file diff --git a/tests/llm_translation/test_bedrock_completion.py b/tests/llm_translation/test_bedrock_completion.py index 9242950daac..f43e939c681 100644 --- a/tests/llm_translation/test_bedrock_completion.py +++ b/tests/llm_translation/test_bedrock_completion.py @@ -3434,3 +3434,100 @@ async def test_bedrock_streaming_passthrough_test1(monkeypatch): print(mock_callback.call_args.kwargs.keys()) assert "standard_logging_object" in mock_callback.call_args.kwargs["kwargs"] assert "response_cost" in mock_callback.call_args.kwargs["kwargs"] + + +def test_bedrock_openai_imported_model(): + """ + Test that Bedrock imported models using OpenAI format work correctly. + + This test validates: + 1. The request body follows OpenAI Chat Completions format + 2. The URL is correctly constructed for Bedrock invoke endpoint + 3. Messages with system, user roles and image_url content are preserved + """ + from litellm.llms.custom_httpx.http_handler import HTTPHandler + + client = HTTPHandler() + + # Sample base64 image data (truncated for test) + sample_base64 = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==" + + messages = [ + { + "role": "system", + "content": "You are a helpful assistant that can analyze images.", + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "Spot the difference between the two images?", + }, + { + "type": "image_url", + "image_url": {"url": f"data:image/jpeg;base64,{sample_base64}"}, + }, + { + "type": "image_url", + "image_url": {"url": f"data:image/jpeg;base64,{sample_base64}"}, + }, + ], + }, + ] + + with patch.object(client, "post") as mock_post: + try: + response = completion( + model="bedrock/openai/arn:aws:bedrock:us-east-1:117159858402:imported-model/m4gc1mrfuddy", + messages=messages, + max_tokens=300, + temperature=0.5, + client=client, + ) + except Exception as e: + print(f"Exception (expected during mock): {e}") + + mock_post.assert_called_once() + + # Validate URL + url = mock_post.call_args.kwargs["url"] + print(f"URL: {url}") + assert "bedrock-runtime.us-east-1.amazonaws.com" in url + assert "arn:aws:bedrock:us-east-1:117159858402:imported-model/m4gc1mrfuddy" in url + assert "/invoke" in url + + # Validate request body follows OpenAI format + request_body = json.loads(mock_post.call_args.kwargs["data"]) + print(f"Request body: {json.dumps(request_body, indent=2)}") + + # Check messages structure + assert "messages" in request_body + assert len(request_body["messages"]) == 2 + + # Check system message + system_msg = request_body["messages"][0] + assert system_msg["role"] == "system" + assert "helpful assistant" in system_msg["content"] + + # Check user message with image content + user_msg = request_body["messages"][1] + assert user_msg["role"] == "user" + assert isinstance(user_msg["content"], list) + assert len(user_msg["content"]) == 3 + + # Check text content + assert user_msg["content"][0]["type"] == "text" + assert "Spot the difference" in user_msg["content"][0]["text"] + + # Check image_url content + assert user_msg["content"][1]["type"] == "image_url" + assert "image_url" in user_msg["content"][1] + assert user_msg["content"][1]["image_url"]["url"].startswith("data:image/jpeg;base64,") + + assert user_msg["content"][2]["type"] == "image_url" + assert "image_url" in user_msg["content"][2] + + # Check max_tokens and temperature + assert request_body["max_tokens"] == 300 + assert request_body["temperature"] == 0.5 diff --git a/tests/llm_translation/test_elevenlabs.py b/tests/llm_translation/test_elevenlabs.py index 4227c3f3c62..5128cd973e8 100644 --- a/tests/llm_translation/test_elevenlabs.py +++ b/tests/llm_translation/test_elevenlabs.py @@ -1,6 +1,8 @@ import os import sys +from typing import Any, Dict + import pytest from unittest.mock import patch, MagicMock import httpx @@ -11,6 +13,8 @@ sys.path.insert( import litellm from base_audio_transcription_unit_tests import BaseLLMAudioTranscriptionTest +os.environ.setdefault("ELEVENLABS_API_KEY", "test-elevenlabs-key") + class TestElevenLabsAudioTranscription(BaseLLMAudioTranscriptionTest): def get_base_audio_transcription_call_args(self) -> dict: @@ -108,4 +112,84 @@ class TestElevenLabsAudioTranscription(BaseLLMAudioTranscriptionTest): except Exception as e: print(f"❌ Test failed: {e}") print(f"Captured request data: {captured_request_data}") - raise \ No newline at end of file + raise + + +class TestElevenLabsTextToSpeechTransformation: + @pytest.fixture(scope="class") + def config(self): + from litellm.llms.elevenlabs.text_to_speech.transformation import ( + ElevenLabsTextToSpeechConfig, + ) + + return ElevenLabsTextToSpeechConfig() + + def test_map_openai_params_maps_voice_and_speed(self, config): + kwargs: Dict[str, Any] = {} + mapped_voice, mapped_params = config.map_openai_params( + model="eleven_multilingual_v2", + optional_params={ + "response_format": "mp3", + "speed": 1.25, + "model_id": "eleven_multilingual_v2", + }, + voice="alloy", + kwargs=kwargs, + ) + + assert mapped_voice == config.VOICE_MAPPINGS["alloy"] + assert mapped_params["voice_settings"]["speed"] == pytest.approx(1.25) + assert ( + kwargs[config.ELEVENLABS_QUERY_PARAMS_KEY]["output_format"] + == "mp3_44100_128" + ) + + def test_transform_request_and_url(self, config): + kwargs: Dict[str, Any] = {} + voice_id, optional_params = config.map_openai_params( + model="eleven_multilingual_v2", + optional_params={ + "response_format": "pcm", + "model_id": "eleven_multilingual_v2", + "pronunciation_dictionary_locators": [ + {"pronunciation_dictionary_id": "dict_1"} + ], + }, + voice="alloy", + kwargs=kwargs, + ) + + litellm_params: Dict[str, Any] = { + config.ELEVENLABS_VOICE_ID_KEY: voice_id, + config.ELEVENLABS_QUERY_PARAMS_KEY: kwargs[ + config.ELEVENLABS_QUERY_PARAMS_KEY + ], + } + + headers = config.validate_environment( + headers={}, model="eleven_multilingual_v2", api_key="test-key" + ) + + request_data = config.transform_text_to_speech_request( + model="eleven_multilingual_v2", + input="Hello world", + voice=voice_id, + optional_params=optional_params, + litellm_params=litellm_params, + headers=headers, + ) + + assert request_data["dict_body"]["text"] == "Hello world" + assert request_data["dict_body"]["model_id"] == "eleven_multilingual_v2" + assert request_data["dict_body"]["pronunciation_dictionary_locators"] == [ + {"pronunciation_dictionary_id": "dict_1"} + ] + + url = config.get_complete_url( + model="eleven_multilingual_v2", + api_base=None, + litellm_params=litellm_params, + ) + + assert voice_id in url + assert "output_format=pcm_44100" in url \ No newline at end of file diff --git a/tests/llm_translation/test_gemini.py b/tests/llm_translation/test_gemini.py index 1065509dd4e..57667085c00 100644 --- a/tests/llm_translation/test_gemini.py +++ b/tests/llm_translation/test_gemini.py @@ -295,6 +295,7 @@ def test_gemini_image_generation(): [ "gemini/gemini-2.5-flash-image-preview", "gemini/gemini-2.0-flash-preview-image-generation", + "gemini/gemini-3-pro-image-preview", ], ) def test_gemini_flash_image_preview_models(model_name: str): diff --git a/tests/proxy_unit_tests/test_proxy_token_counter.py b/tests/proxy_unit_tests/test_proxy_token_counter.py index 39606ef7ce9..eb364ebf5b8 100644 --- a/tests/proxy_unit_tests/test_proxy_token_counter.py +++ b/tests/proxy_unit_tests/test_proxy_token_counter.py @@ -819,3 +819,20 @@ async def test_vertex_ai_anthropic_token_counting(): assert response.original_response is not None assert "input_tokens" in response.original_response assert response.original_response["input_tokens"] == 15 + +@pytest.mark.parametrize("vertex_location", ["global", "us-central1"]) +def test_vertex_ai_gemini_token_counting_endpoint(vertex_location): + from litellm.llms.vertex_ai.vertex_ai_partner_models.count_tokens.handler import ( + VertexAIPartnerModelsTokenCounter, + ) + + endpoint = VertexAIPartnerModelsTokenCounter()._build_count_tokens_endpoint( + model="gemini-2.5-pro", + project_id="test-project", + vertex_location=vertex_location, + api_base=None, + ) + if vertex_location == "global": + assert endpoint == "https://aiplatform.googleapis.com" + else: + assert endpoint == f"https://{vertex_location}-aiplatform.googleapis.com" \ No newline at end of file diff --git a/tests/proxy_unit_tests/test_search_api_logging.py b/tests/proxy_unit_tests/test_search_api_logging.py new file mode 100644 index 00000000000..7ac22e51ef2 --- /dev/null +++ b/tests/proxy_unit_tests/test_search_api_logging.py @@ -0,0 +1,202 @@ +""" +Test search API logging and cost tracking in proxy. + +Tests that search API requests are properly logged to LiteLLM_SpendLogs +with correct fields populated (call_type, model, custom_llm_provider, +model_group, spend, etc.) +""" +import asyncio +import os +import sys +import time +from datetime import datetime +from unittest.mock import AsyncMock, patch + +import pytest + +sys.path.insert(0, os.path.abspath("../..")) +import litellm +from litellm import Router +from litellm.caching import DualCache +from litellm.proxy._types import UserAPIKeyAuth +from litellm.proxy.hooks.proxy_track_cost_callback import _ProxyDBLogger +from litellm.proxy.spend_tracking.spend_management_endpoints import view_spend_logs +from litellm.proxy.utils import ProxyLogging, hash_token, update_spend +from litellm.llms.base_llm.search.transformation import SearchResponse, SearchResult + + +@pytest.fixture +def prisma_client(): + from litellm.proxy import proxy_server + from litellm.proxy.proxy_cli import append_query_params + from litellm.proxy.utils import PrismaClient + + params = {"connection_limit": 100, "pool_timeout": 60} + database_url = os.getenv("DATABASE_URL") + if database_url is None: + pytest.skip("DATABASE_URL not set") + + modified_url = append_query_params(database_url, params) + os.environ["DATABASE_URL"] = modified_url + + user_api_key_cache = DualCache() + proxy_logging_obj = ProxyLogging(user_api_key_cache=user_api_key_cache) + + prisma_client = PrismaClient( + database_url=os.environ["DATABASE_URL"], proxy_logging_obj=proxy_logging_obj + ) + + proxy_server.litellm_proxy_budget_name = ( + f"litellm-proxy-budget-{time.time()}" + ) + proxy_server.user_custom_key_generate = None + + return prisma_client + + +@pytest.mark.asyncio +async def test_search_api_logging_and_cost_tracking(prisma_client): + """ + Test that search API requests are logged with correct fields and cost tracking. + + Verifies: + 1. Search request creates a spend log entry + 2. call_type is set to "asearch" + 3. model is set to search_tool_name + 4. custom_llm_provider is set correctly + 5. model_group is set to search_tool_name + 6. spend is calculated and logged + """ + setattr(litellm.proxy.proxy_server, "prisma_client", prisma_client) + setattr(litellm.proxy.proxy_server, "master_key", "sk-1234") + await litellm.proxy.proxy_server.prisma_client.connect() + + # Setup router with search tool + search_tool_name = "tavily-search" + search_provider = "tavily" + + router = Router(model_list=[]) + router.search_tools = [ + { + "search_tool_name": search_tool_name, + "litellm_params": { + "search_provider": search_provider, + }, + } + ] + + setattr(litellm.proxy.proxy_server, "llm_router", router) + + # Generate a test API key + from litellm.proxy.management_endpoints.key_management_endpoints import generate_key_fn + from litellm.proxy._types import GenerateKeyRequest + + from litellm.proxy._types import LitellmUserRoles + + user_api_key_dict = UserAPIKeyAuth( + user_role=LitellmUserRoles.PROXY_ADMIN, + api_key="sk-1234", + user_id="test_user", + ) + + key_request = GenerateKeyRequest(models=[], duration=None) + key_response = await generate_key_fn( + data=key_request, user_api_key_dict=user_api_key_dict + ) + generated_key = key_response.key + user_id = key_response.user_id + + # Create mock search response + mock_search_result = SearchResult( + title="Test Result", + url="https://example.com", + snippet="Test snippet", + ) + + mock_search_response = SearchResponse( + object="search", + results=[mock_search_result], + ) + + # Mock the search function to return our mock response + with patch("litellm.search.main.asearch", new_callable=AsyncMock) as mock_asearch: + mock_asearch.return_value = mock_search_response + + # Setup proxy logging + user_api_key_cache = DualCache() + proxy_logging_obj = ProxyLogging(user_api_key_cache=user_api_key_cache) + setattr(litellm.proxy.proxy_server, "proxy_logging_obj", proxy_logging_obj) + + # Call the track_cost_callback directly to simulate what happens after a search + proxy_db_logger = _ProxyDBLogger() + + # Simulate the kwargs that would be passed from the search endpoint + request_id = "search_test_123" + kwargs = { + "call_type": "asearch", + "model": search_tool_name, + "custom_llm_provider": search_provider, + "litellm_call_id": request_id, # Set request_id in kwargs + "litellm_params": { + "metadata": { + "user_api_key": hash_token(generated_key), + "user_api_key_user_id": user_id, + "model_group": search_tool_name, + } + }, + "metadata": { + "user_api_key": hash_token(generated_key), + "user_api_key_user_id": user_id, + "model_group": search_tool_name, + }, + "response_cost": 0.008, # Mock cost for tavily search + } + + # Set id on the response object + mock_search_response.id = request_id + + await proxy_db_logger._PROXY_track_cost_callback( + kwargs=kwargs, + completion_response=mock_search_response, + start_time=datetime.now(), + end_time=datetime.now(), + ) + + # Wait for async operations + await asyncio.sleep(2) + await update_spend( + prisma_client=prisma_client, + db_writer_client=None, + proxy_logging_obj=proxy_logging_obj, + ) + + # Query spend logs + spend_logs = await view_spend_logs( + request_id=request_id, + user_api_key_dict=UserAPIKeyAuth(api_key=generated_key), + ) + + # Verify spend log was created + assert len(spend_logs) == 1, f"Expected 1 spend log, got {len(spend_logs)}" + + spend_log = spend_logs[0] + + # Verify all fields are populated correctly + assert spend_log.request_id == request_id + assert spend_log.call_type == "asearch" + assert spend_log.model == search_tool_name + assert spend_log.custom_llm_provider == search_provider + assert spend_log.model_group == search_tool_name + assert spend_log.spend == 0.008 + # API key should be hashed (either the generated key or the one from metadata) + assert spend_log.api_key != "" # Should be populated + # Note: user field may be empty if not set in the request, but user_id should be in metadata + assert spend_log.metadata.get("user_api_key_user_id") == user_id or spend_log.user == user_id + + print(f"✅ Search API logging test passed!") + print(f" - call_type: {spend_log.call_type}") + print(f" - model: {spend_log.model}") + print(f" - custom_llm_provider: {spend_log.custom_llm_provider}") + print(f" - model_group: {spend_log.model_group}") + print(f" - spend: {spend_log.spend}") + diff --git a/tests/test_litellm/completion_extras/test_litellm_responses_transformation_transformation.py b/tests/test_litellm/completion_extras/test_litellm_responses_transformation_transformation.py new file mode 100644 index 00000000000..adbaf219079 --- /dev/null +++ b/tests/test_litellm/completion_extras/test_litellm_responses_transformation_transformation.py @@ -0,0 +1,136 @@ +""" +Test for response_format to text.format conversion in completion -> responses bridge +""" +import pytest +from litellm.completion_extras.litellm_responses_transformation.transformation import ( + LiteLLMResponsesTransformationHandler, +) + + +def test_transform_response_format_to_text_format_json_schema(): + """Test conversion of response_format with json_schema to text.format""" + handler = LiteLLMResponsesTransformationHandler() + + # Chat Completion format + response_format = { + "type": "json_schema", + "json_schema": { + "name": "person_schema", + "schema": { + "type": "object", + "properties": { + "name": {"type": "string"}, + "age": {"type": "integer"} + }, + "required": ["name", "age"], + "additionalProperties": False + }, + "strict": True + } + } + + # Convert to Responses API format + result = handler._transform_response_format_to_text_format(response_format) + + # Verify conversion + assert result is not None + assert "format" in result + assert result["format"]["type"] == "json_schema" + assert result["format"]["name"] == "person_schema" + assert result["format"]["strict"] is True + assert "schema" in result["format"] + assert result["format"]["schema"]["type"] == "object" + assert "properties" in result["format"]["schema"] + + +def test_transform_response_format_to_text_format_json_object(): + """Test conversion of response_format with json_object to text.format""" + handler = LiteLLMResponsesTransformationHandler() + + response_format = { + "type": "json_object" + } + + result = handler._transform_response_format_to_text_format(response_format) + + assert result is not None + assert "format" in result + assert result["format"]["type"] == "json_object" + + +def test_transform_response_format_to_text_format_text(): + """Test conversion of response_format with text to text.format""" + handler = LiteLLMResponsesTransformationHandler() + + response_format = { + "type": "text" + } + + result = handler._transform_response_format_to_text_format(response_format) + + assert result is not None + assert "format" in result + assert result["format"]["type"] == "text" + + +def test_transform_response_format_to_text_format_none(): + """Test that None input returns None""" + handler = LiteLLMResponsesTransformationHandler() + + result = handler._transform_response_format_to_text_format(None) + + assert result is None + + +def test_transform_request_with_response_format(): + """Test that transform_request correctly handles response_format parameter""" + handler = LiteLLMResponsesTransformationHandler() + + messages = [ + {"role": "user", "content": "Extract person info: John Doe, 30 years old"} + ] + + optional_params = { + "response_format": { + "type": "json_schema", + "json_schema": { + "name": "person_schema", + "schema": { + "type": "object", + "properties": { + "name": {"type": "string"}, + "age": {"type": "integer"} + }, + "required": ["name", "age"], + "additionalProperties": False + }, + "strict": True + } + } + } + + litellm_params = {} + headers = {} + + # Mock logging object + class MockLoggingObj: + pass + + litellm_logging_obj = MockLoggingObj() + + result = handler.transform_request( + model="o3-pro", + messages=messages, + optional_params=optional_params, + litellm_params=litellm_params, + headers=headers, + litellm_logging_obj=litellm_logging_obj, + ) + + # Verify that text parameter was set with converted format + assert "text" in result + assert result["text"] is not None + assert "format" in result["text"] + assert result["text"]["format"]["type"] == "json_schema" + assert result["text"]["format"]["name"] == "person_schema" + assert "schema" in result["text"]["format"] diff --git a/tests/test_litellm/litellm_core_utils/test_streaming_chunk_builder_utils.py b/tests/test_litellm/litellm_core_utils/test_streaming_chunk_builder_utils.py index f6636874336..2164a3b82e3 100644 --- a/tests/test_litellm/litellm_core_utils/test_streaming_chunk_builder_utils.py +++ b/tests/test_litellm/litellm_core_utils/test_streaming_chunk_builder_utils.py @@ -16,6 +16,7 @@ from litellm.types.utils import ( Function, ModelResponseStream, PromptTokensDetails, + ServerToolUse, StreamingChoices, Usage, ) @@ -325,3 +326,83 @@ def test_stream_chunk_builder_litellm_usage_chunks(): assert usage.prompt_tokens == 50 assert usage.completion_tokens == 27 assert usage.total_tokens == 77 + + +def test_stream_chunk_builder_anthropic_web_search(): + # Prepare two mocked streaming chunks with usage split across them + chunk1 = ModelResponseStream( + id="chatcmpl-mocked-usage-1", + created=1745513206, + model="claude-sonnet-4-5-20250929", + object="chat.completion.chunk", + system_fingerprint=None, + choices=[ + StreamingChoices( + finish_reason=None, + index=0, + delta=Delta( + provider_specific_fields=None, + content="", + role=None, + function_call=None, + tool_calls=None, + audio=None, + ), + logprobs=None, + ) + ], + provider_specific_fields=None, + stream_options={"include_usage": True}, + usage=Usage( + completion_tokens=0, + prompt_tokens=50, + total_tokens=50, + completion_tokens_details=None, + server_tool_use=ServerToolUse(web_search_requests=2), + prompt_tokens_details=None, + ), + ) + + chunk2 = ModelResponseStream( + id="chatcmpl-mocked-usage-1", + created=1745513207, + model="claude-sonnet-4-5-20250929", + object="chat.completion.chunk", + system_fingerprint=None, + choices=[ + StreamingChoices( + finish_reason="stop", + index=0, + delta=Delta( + provider_specific_fields=None, + content=None, + role=None, + function_call=None, + tool_calls=None, + audio=None, + ), + logprobs=None, + ) + ], + provider_specific_fields=None, + stream_options={"include_usage": True}, + usage=Usage( + completion_tokens=27, + prompt_tokens=0, + total_tokens=27, + completion_tokens_details=None, + prompt_tokens_details=None, + ), + ) + + chunks = [chunk1, chunk2] + processor = ChunkProcessor(chunks=chunks) + + usage = processor.calculate_usage( + chunks=chunks, model="claude-sonnet-4-5-20250929", completion_output="" + ) + + assert usage.prompt_tokens == 50 + assert usage.completion_tokens == 27 + assert usage.total_tokens == 77 + assert usage.server_tool_use['web_search_requests'] == 2 \ No newline at end of file diff --git a/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py b/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py index 8ff2f0a4474..09c16add77a 100644 --- a/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py +++ b/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py @@ -556,3 +556,744 @@ def test_anthropic_structured_output_beta_header(): "structured-outputs-2025-11-13" in response["raw_request_headers"]["anthropic-beta"] ) + + +# ============ Tool Search Tests ============ + + +def test_tool_search_regex_detection(): + """Test that tool search regex tools are properly detected""" + from litellm.llms.anthropic.common_utils import AnthropicModelInfo + + config = AnthropicModelInfo() + + # Test with tool search regex tool + tools = [ + { + "type": "tool_search_tool_regex_20251119", + "name": "tool_search_tool_regex" + } + ] + assert config.is_tool_search_used(tools) is True + + # Test without tool search + tools = [ + { + "type": "function", + "function": {"name": "get_weather"} + } + ] + assert config.is_tool_search_used(tools) is False + + +def test_tool_search_bm25_detection(): + """Test that tool search BM25 tools are properly detected""" + from litellm.llms.anthropic.common_utils import AnthropicModelInfo + + config = AnthropicModelInfo() + + # Test with tool search BM25 tool + tools = [ + { + "type": "tool_search_tool_bm25_20251119", + "name": "tool_search_tool_bm25" + } + ] + assert config.is_tool_search_used(tools) is True + + +def test_tool_search_beta_header(): + """Test that tool search beta header is automatically added""" + from litellm.llms.anthropic.common_utils import AnthropicModelInfo + + config = AnthropicModelInfo() + + headers = config.get_anthropic_headers( + api_key="test-key", + tool_search_used=True, + ) + + assert "anthropic-beta" in headers + assert "advanced-tool-use-2025-11-20" in headers["anthropic-beta"] + + +def test_tool_search_regex_mapping(): + """Test that tool search regex tools are properly mapped""" + config = AnthropicConfig() + + tool = { + "type": "tool_search_tool_regex_20251119", + "name": "tool_search_tool_regex" + } + + mapped_tool, mcp_server = config._map_tool_helper(tool) + + assert mapped_tool is not None + assert mapped_tool["type"] == "tool_search_tool_regex_20251119" + assert mapped_tool["name"] == "tool_search_tool_regex" + assert mcp_server is None + + +def test_tool_search_bm25_mapping(): + """Test that tool search BM25 tools are properly mapped""" + config = AnthropicConfig() + + tool = { + "type": "tool_search_tool_bm25_20251119", + "name": "tool_search_tool_bm25" + } + + mapped_tool, mcp_server = config._map_tool_helper(tool) + + assert mapped_tool is not None + assert mapped_tool["type"] == "tool_search_tool_bm25_20251119" + assert mapped_tool["name"] == "tool_search_tool_bm25" + assert mcp_server is None + + +def test_deferred_tools_separation(): + """Test that deferred and non-deferred tools are properly separated""" + config = AnthropicConfig() + + tools = [ + { + "type": "tool_search_tool_regex_20251119", + "name": "tool_search_tool_regex" + }, + { + "type": "function", + "function": {"name": "get_weather"}, + "defer_loading": True + }, + { + "type": "function", + "function": {"name": "search_files"}, + "defer_loading": False + } + ] + + non_deferred, deferred = config._separate_deferred_tools(tools) + + assert len(non_deferred) == 2 # tool_search and search_files + assert len(deferred) == 1 # get_weather + + +def test_server_tool_use_in_response(): + """Test that server_tool_use blocks are parsed correctly""" + config = AnthropicConfig() + + completion_response = { + "content": [ + { + "type": "server_tool_use", + "id": "srvtoolu_01ABC123", + "name": "tool_search_tool_regex", + "input": {"query": "weather"} + } + ] + } + + text, citations, thinking_blocks, reasoning_content, tool_calls = config.extract_response_content( + completion_response + ) + + assert len(tool_calls) == 1 + assert tool_calls[0]["id"] == "srvtoolu_01ABC123" + assert tool_calls[0]["function"]["name"] == "tool_search_tool_regex" + + +def test_tool_search_usage_tracking(): + """Test that tool_search_requests are tracked in usage""" + config = AnthropicConfig() + + usage_object = { + "input_tokens": 100, + "output_tokens": 50, + "server_tool_use": { + "tool_search_requests": 2 + } + } + + usage = config.calculate_usage(usage_object=usage_object, reasoning_content=None) + + assert usage.server_tool_use is not None + assert usage.server_tool_use.tool_search_requests == 2 + + +def test_tool_reference_expansion(): + """Test that tool_reference blocks are expanded correctly""" + config = AnthropicConfig() + + deferred_tools = [ + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get weather" + } + } + ] + + content = [ + {"type": "text", "text": "I'll search for tools"}, + {"type": "tool_reference", "tool_name": "get_weather"} + ] + + expanded = config._expand_tool_references(content, deferred_tools) + + assert len(expanded) == 2 + assert expanded[0]["type"] == "text" + assert expanded[1]["type"] == "function" + assert expanded[1]["function"]["name"] == "get_weather" + + +def test_defer_loading_preserved_in_transformation(): + """Test that defer_loading parameter is preserved when transforming tools""" + config = AnthropicConfig() + + tool = { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get weather information", + "parameters": { + "type": "object", + "properties": { + "location": {"type": "string"} + }, + "required": ["location"] + } + }, + "defer_loading": True + } + + mapped_tool, mcp_server = config._map_tool_helper(tool) + + assert mapped_tool is not None + assert mapped_tool.get("defer_loading") is True + assert mapped_tool["name"] == "get_weather" + assert mcp_server is None + + +def test_tool_search_complete_response_parsing(): + """Test parsing a complete tool search response with server_tool_use and tool_search_tool_result blocks""" + config = AnthropicConfig() + + # Simulating actual Anthropic API response with tool search + completion_response = { + "content": [ + { + "type": "text", + "text": "I'll search for weather-related tools that can help you." + }, + { + "type": "server_tool_use", + "id": "srvtoolu_015i6aVA2niwzv4RG4DtnxDJ", + "name": "tool_search_tool_regex", + "input": {"pattern": "weather", "limit": 5}, + "caller": {"type": "direct"} + }, + { + "type": "tool_search_tool_result", + "tool_use_id": "srvtoolu_015i6aVA2niwzv4RG4DtnxDJ", + "content": { + "type": "tool_search_tool_search_result", + "tool_references": [{"type": "tool_reference", "tool_name": "get_weather"}] + } + }, + { + "type": "text", + "text": "Great! I found a weather tool." + }, + { + "type": "tool_use", + "id": "toolu_01CrCNx4ntSaeeV9iArT4JfQ", + "name": "get_weather", + "input": {"location": "San Francisco"} + } + ], + "usage": { + "input_tokens": 1639, + "output_tokens": 170, + "server_tool_use": {"web_search_requests": 0} + } + } + + # Extract content + text, citations, thinking_blocks, reasoning_content, tool_calls = config.extract_response_content( + completion_response + ) + + # Verify text extraction (should concatenate both text blocks) + assert "I'll search for weather-related tools" in text + assert "Great! I found a weather tool" in text + + # Verify tool calls (should have both server_tool_use and tool_use) + assert len(tool_calls) == 2 + assert tool_calls[0]["function"]["name"] == "tool_search_tool_regex" + assert tool_calls[1]["function"]["name"] == "get_weather" + + # Verify usage calculation counts tool_search_requests from content + usage = config.calculate_usage( + usage_object=completion_response["usage"], + reasoning_content=None, + completion_response=completion_response + ) + + assert usage.server_tool_use is not None + assert usage.server_tool_use.web_search_requests == 0 + assert usage.server_tool_use.tool_search_requests == 1 # Counted from server_tool_use blocks + + +def test_allowed_callers_field_preservation(): + """Test that allowed_callers field is preserved during tool transformation.""" + config = AnthropicConfig() + + # Test with top-level allowed_callers + tool_with_allowed_callers = { + "type": "function", + "function": { + "name": "query_database", + "description": "Execute a SQL query", + "parameters": { + "type": "object", + "properties": { + "sql": {"type": "string"} + }, + "required": ["sql"] + } + }, + "allowed_callers": ["code_execution_20250825"] + } + + transformed_tool, _ = config._map_tool_helper(tool_with_allowed_callers) + assert transformed_tool is not None + assert "allowed_callers" in transformed_tool + assert transformed_tool["allowed_callers"] == ["code_execution_20250825"] + + +def test_programmatic_tool_calling_beta_header(): + """Test that beta header is automatically added when programmatic tool calling is detected.""" + from litellm.llms.anthropic.common_utils import AnthropicModelInfo + + model_info = AnthropicModelInfo() + + # Test detection with allowed_callers + tools = [ + { + "type": "code_execution_20250825", + "name": "code_execution" + }, + { + "type": "function", + "function": { + "name": "query_database", + "description": "Execute a SQL query", + "parameters": {"type": "object", "properties": {}} + }, + "allowed_callers": ["code_execution_20250825"] + } + ] + + is_programmatic = model_info.is_programmatic_tool_calling_used(tools) + assert is_programmatic is True + + # Test header generation + headers = model_info.get_anthropic_headers( + api_key="test-key", + programmatic_tool_calling_used=True + ) + + assert "anthropic-beta" in headers + assert "advanced-tool-use-2025-11-20" in headers["anthropic-beta"] + + +def test_caller_field_in_response(): + """Test that caller field is correctly parsed from tool_use blocks.""" + config = AnthropicConfig() + + # Mock response with programmatic tool call + completion_response = { + "id": "msg_test", + "type": "message", + "role": "assistant", + "content": [ + { + "type": "text", + "text": "I'll query the database." + }, + { + "type": "tool_use", + "id": "toolu_123", + "name": "query_database", + "input": {"sql": "SELECT * FROM users"}, + "caller": { + "type": "code_execution_20250825", + "tool_id": "srvtoolu_abc" + } + } + ], + "stop_reason": "tool_use", + "usage": {"input_tokens": 100, "output_tokens": 50} + } + + text, citations, thinking, reasoning, tool_calls = config.extract_response_content(completion_response) + + assert len(tool_calls) == 1 + assert tool_calls[0]["id"] == "toolu_123" + assert tool_calls[0]["function"]["name"] == "query_database" + assert "caller" in tool_calls[0] + assert tool_calls[0]["caller"]["type"] == "code_execution_20250825" + assert tool_calls[0]["caller"]["tool_id"] == "srvtoolu_abc" + + +def test_code_execution_20250825_tool_type(): + """Test that code_execution_20250825 tool type is handled correctly.""" + config = AnthropicConfig() + + tool = { + "type": "code_execution_20250825", + "name": "code_execution" + } + + transformed_tool, _ = config._map_tool_helper(tool) + assert transformed_tool is not None + assert transformed_tool["type"] == "code_execution_20250825" + assert transformed_tool["name"] == "code_execution" + + +def test_allowed_callers_in_function_field(): + """Test that allowed_callers in function field is also preserved.""" + config = AnthropicConfig() + + # Test with function.allowed_callers + tool = { + "type": "function", + "function": { + "name": "query_database", + "description": "Execute a SQL query", + "parameters": { + "type": "object", + "properties": { + "sql": {"type": "string"} + }, + "required": ["sql"] + }, + "allowed_callers": ["code_execution_20250825"] + } + } + + transformed_tool, _ = config._map_tool_helper(tool) + assert transformed_tool is not None + assert "allowed_callers" in transformed_tool + assert transformed_tool["allowed_callers"] == ["code_execution_20250825"] + + +def test_input_examples_field_preservation(): + """Test that input_examples field is preserved during tool transformation.""" + config = AnthropicConfig() + + # Test with top-level input_examples + tool_with_examples = { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get the current weather in a given location", + "parameters": { + "type": "object", + "properties": { + "location": {"type": "string"}, + "unit": {"type": "string", "enum": ["celsius", "fahrenheit"]} + }, + "required": ["location"] + } + }, + "input_examples": [ + {"location": "San Francisco, CA", "unit": "fahrenheit"}, + {"location": "Tokyo, Japan", "unit": "celsius"} + ] + } + + transformed_tool, _ = config._map_tool_helper(tool_with_examples) + assert transformed_tool is not None + assert "input_examples" in transformed_tool + assert len(transformed_tool["input_examples"]) == 2 + assert transformed_tool["input_examples"][0]["location"] == "San Francisco, CA" + + +def test_input_examples_beta_header(): + """Test that beta header is automatically added when input_examples is detected.""" + from litellm.llms.anthropic.common_utils import AnthropicModelInfo + + model_info = AnthropicModelInfo() + + # Test detection with input_examples + tools = [ + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get weather information", + "parameters": {"type": "object", "properties": {}} + }, + "input_examples": [ + {"location": "San Francisco, CA"} + ] + } + ] + + is_examples_used = model_info.is_input_examples_used(tools) + assert is_examples_used is True + + # Test header generation + headers = model_info.get_anthropic_headers( + api_key="test-key", + input_examples_used=True + ) + + assert "anthropic-beta" in headers + assert "advanced-tool-use-2025-11-20" in headers["anthropic-beta"] + + +def test_input_examples_in_function_field(): + """Test that input_examples in function field is also preserved.""" + config = AnthropicConfig() + + # Test with function.input_examples + tool = { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get weather information", + "parameters": { + "type": "object", + "properties": { + "location": {"type": "string"} + }, + "required": ["location"] + }, + "input_examples": [ + {"location": "Paris, France"}, + {"location": "London, UK"} + ] + } + } + + transformed_tool, _ = config._map_tool_helper(tool) + assert transformed_tool is not None + assert "input_examples" in transformed_tool + assert len(transformed_tool["input_examples"]) == 2 + + +def test_input_examples_with_other_features(): + """Test that input_examples works alongside other tool features.""" + config = AnthropicConfig() + + # Tool with input_examples, defer_loading, and allowed_callers + tool = { + "type": "function", + "function": { + "name": "query_database", + "description": "Execute a SQL query", + "parameters": { + "type": "object", + "properties": { + "sql": {"type": "string"} + }, + "required": ["sql"] + } + }, + "input_examples": [ + {"sql": "SELECT * FROM users WHERE id = 1"} + ], + "defer_loading": True, + "allowed_callers": ["code_execution_20250825"] + } + + transformed_tool, _ = config._map_tool_helper(tool) + assert transformed_tool is not None + assert "input_examples" in transformed_tool + assert "defer_loading" in transformed_tool + assert "allowed_callers" in transformed_tool + assert transformed_tool["defer_loading"] is True + assert transformed_tool["allowed_callers"] == ["code_execution_20250825"] + + +def test_input_examples_empty_list_not_added(): + """Test that empty input_examples list is not added to transformed tool.""" + config = AnthropicConfig() + + # Tool with empty input_examples + tool = { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get weather information", + "parameters": { + "type": "object", + "properties": { + "location": {"type": "string"} + }, + "required": ["location"] + } + }, + "input_examples": [] + } + + transformed_tool, _ = config._map_tool_helper(tool) + assert transformed_tool is not None + # Empty list should not be added + assert "input_examples" not in transformed_tool or len(transformed_tool.get("input_examples", [])) == 0 + + +# ============ Effort Parameter Tests ============ + + +def test_effort_output_config_preservation(): + """Test that output_config with effort is preserved in transformation.""" + config = AnthropicConfig() + + messages = [{"role": "user", "content": "Analyze this code"}] + optional_params = { + "output_config": { + "effort": "medium" + } + } + + result = config.transform_request( + model="claude-opus-4-5-20251101", + messages=messages, + optional_params=optional_params, + litellm_params={}, + headers={} + ) + + assert "output_config" in result + assert result["output_config"]["effort"] == "medium" + + +def test_effort_beta_header_injection(): + """Test that effort beta header is automatically added when output_config is detected.""" + from litellm.llms.anthropic.common_utils import AnthropicModelInfo + + model_info = AnthropicModelInfo() + + # Test with effort parameter + optional_params = { + "output_config": { + "effort": "low" + } + } + + effort_used = model_info.is_effort_used(optional_params=optional_params) + assert effort_used is True + + headers = model_info.get_anthropic_headers( + api_key="test-key", + effort_used=effort_used + ) + + assert "anthropic-beta" in headers + assert "effort-2025-11-24" in headers["anthropic-beta"] + + +def test_effort_validation(): + """Test that only valid effort values are accepted.""" + config = AnthropicConfig() + + messages = [{"role": "user", "content": "Test"}] + + # Valid values should work + for effort in ["high", "medium", "low"]: + optional_params = {"output_config": {"effort": effort}} + result = config.transform_request( + model="claude-opus-4-5-20251101", + messages=messages, + optional_params=optional_params, + litellm_params={}, + headers={} + ) + assert result["output_config"]["effort"] == effort + + # Invalid value should raise error + with pytest.raises(ValueError, match="Invalid effort value"): + optional_params = {"output_config": {"effort": "invalid"}} + config.transform_request( + model="claude-opus-4-5-20251101", + messages=messages, + optional_params=optional_params, + litellm_params={}, + headers={} + ) + + +def test_effort_with_claude_opus_45(): + """Test effort parameter works with Claude Opus 4.5 model.""" + config = AnthropicConfig() + + messages = [{"role": "user", "content": "Complex analysis task"}] + optional_params = { + "output_config": { + "effort": "high" + } + } + + result = config.transform_request( + model="claude-opus-4-5-20251101", + messages=messages, + optional_params=optional_params, + litellm_params={}, + headers={} + ) + + assert "output_config" in result + assert result["output_config"]["effort"] == "high" + assert result["model"] == "claude-opus-4-5-20251101" + + +def test_effort_with_other_features(): + """Test effort works alongside other features (thinking, tools).""" + config = AnthropicConfig() + + messages = [{"role": "user", "content": "Use tools efficiently"}] + tools = [ + { + "type": "function", + "function": { + "name": "get_data", + "description": "Get data", + "parameters": { + "type": "object", + "properties": { + "query": {"type": "string"} + }, + "required": ["query"] + } + } + } + ] + optional_params = { + "output_config": { + "effort": "low" + }, + "tools": tools, + "thinking": { + "type": "enabled", + "budget_tokens": 1000 + } + } + + result = config.transform_request( + model="claude-opus-4-5-20251101", + messages=messages, + optional_params=optional_params, + litellm_params={}, + headers={} + ) + + # Verify all features are present + assert "output_config" in result + assert result["output_config"]["effort"] == "low" + assert "tools" in result + assert len(result["tools"]) > 0 + assert "thinking" in result diff --git a/tests/test_litellm/llms/azure/anthropic/test_azure_anthropic_handler.py b/tests/test_litellm/llms/azure/anthropic/test_azure_anthropic_handler.py new file mode 100644 index 00000000000..bb5d1f9933d --- /dev/null +++ b/tests/test_litellm/llms/azure/anthropic/test_azure_anthropic_handler.py @@ -0,0 +1,216 @@ +import os +import sys + +sys.path.insert( + 0, os.path.abspath(os.path.join(os.path.dirname(__file__), "../../../../..")) +) + +import json +from unittest.mock import MagicMock, patch + +import pytest + +from litellm.llms.azure.anthropic.handler import AzureAnthropicChatCompletion +from litellm.types.utils import ModelResponse + + +class TestAzureAnthropicChatCompletion: + def test_inherits_from_anthropic_chat_completion(self): + """Test that AzureAnthropicChatCompletion inherits from AnthropicChatCompletion""" + handler = AzureAnthropicChatCompletion() + assert isinstance(handler, AzureAnthropicChatCompletion) + # Check that it has methods from parent class + assert hasattr(handler, "acompletion_function") + assert hasattr(handler, "acompletion_stream_function") + + @patch("litellm.utils.ProviderConfigManager") + @patch("litellm.llms.azure.anthropic.handler.AzureAnthropicConfig") + def test_completion_uses_azure_anthropic_config(self, mock_azure_config, mock_provider_manager): + """Test that completion method uses AzureAnthropicConfig""" + handler = AzureAnthropicChatCompletion() + mock_config = MagicMock() + mock_config.transform_request.return_value = {"model": "claude-sonnet-4-5", "messages": []} + mock_config.transform_response.return_value = ModelResponse() + mock_config_instance = MagicMock() + mock_config_instance.validate_environment.return_value = {"x-api-key": "test-api-key", "anthropic-version": "2023-06-01"} + mock_azure_config.return_value = mock_config_instance + mock_provider_manager.get_provider_chat_config.return_value = mock_config + + model = "claude-sonnet-4-5" + messages = [{"role": "user", "content": "Hello"}] + api_base = "https://test.services.ai.azure.com/anthropic/v1/messages" + custom_llm_provider = "azure_anthropic" + custom_prompt_dict = {} + model_response = ModelResponse() + print_verbose = MagicMock() + encoding = MagicMock() + api_key = "test-api-key" + logging_obj = MagicMock() + optional_params = {} + timeout = 60.0 + litellm_params = {"api_key": "test-api-key"} + headers = {} + + with patch.object( + handler, "acompletion_function", return_value=ModelResponse() + ) as mock_acompletion: + handler.completion( + model=model, + messages=messages, + api_base=api_base, + custom_llm_provider=custom_llm_provider, + custom_prompt_dict=custom_prompt_dict, + model_response=model_response, + print_verbose=print_verbose, + encoding=encoding, + api_key=api_key, + logging_obj=logging_obj, + optional_params=optional_params, + timeout=timeout, + litellm_params=litellm_params, + headers=headers, + acompletion=True, + ) + + # Verify AzureAnthropicConfig was used + mock_azure_config.assert_called_once() + mock_config_instance.validate_environment.assert_called_once() + + @patch("litellm.llms.anthropic.chat.handler.make_sync_call") + @patch("litellm.utils.ProviderConfigManager") + @patch("litellm.llms.azure.anthropic.handler.AzureAnthropicConfig") + def test_completion_streaming(self, mock_azure_config, mock_provider_manager, mock_make_sync_call): + # Note: decorators are applied in reverse order + """Test completion with streaming""" + handler = AzureAnthropicChatCompletion() + mock_config = MagicMock() + mock_config.transform_request.return_value = { + "model": "claude-sonnet-4-5", + "messages": [], + "stream": True, + } + mock_config_instance = MagicMock() + mock_config_instance.validate_environment.return_value = {"x-api-key": "test-api-key", "anthropic-version": "2023-06-01"} + mock_azure_config.return_value = mock_config_instance + mock_provider_manager.get_provider_chat_config.return_value = mock_config + + # Mock streaming response + mock_stream = MagicMock() + mock_headers = MagicMock() + mock_make_sync_call.return_value = (mock_stream, mock_headers) + + model = "claude-sonnet-4-5" + messages = [{"role": "user", "content": "Hello"}] + api_base = "https://test.services.ai.azure.com/anthropic/v1/messages" + custom_llm_provider = "azure_anthropic" + custom_prompt_dict = {} + model_response = ModelResponse() + print_verbose = MagicMock() + encoding = MagicMock() + api_key = "test-api-key" + logging_obj = MagicMock() + optional_params = {"stream": True} + timeout = 60.0 + litellm_params = {"api_key": "test-api-key"} + headers = {} + + result = handler.completion( + model=model, + messages=messages, + api_base=api_base, + custom_llm_provider=custom_llm_provider, + custom_prompt_dict=custom_prompt_dict, + model_response=model_response, + print_verbose=print_verbose, + encoding=encoding, + api_key=api_key, + logging_obj=logging_obj, + optional_params=optional_params, + timeout=timeout, + litellm_params=litellm_params, + headers=headers, + acompletion=False, + ) + + # Verify streaming was handled + mock_make_sync_call.assert_called_once() + assert result is not None + + @patch("litellm.llms.custom_httpx.http_handler._get_httpx_client") + @patch("litellm.utils.ProviderConfigManager") + @patch("litellm.llms.azure.anthropic.handler.AzureAnthropicConfig") + def test_completion_non_streaming(self, mock_azure_config, mock_provider_manager, mock_get_client): + # Note: decorators are applied in reverse order + """Test completion without streaming""" + handler = AzureAnthropicChatCompletion() + mock_config = MagicMock() + mock_config.transform_request.return_value = { + "model": "claude-sonnet-4-5", + "messages": [], + } + mock_response = ModelResponse() + mock_config.transform_response.return_value = mock_response + mock_config_instance = MagicMock() + mock_config_instance.validate_environment.return_value = {"x-api-key": "test-api-key", "anthropic-version": "2023-06-01"} + mock_azure_config.return_value = mock_config_instance + mock_provider_manager.get_provider_chat_config.return_value = mock_config + + model = "claude-sonnet-4-5" + messages = [{"role": "user", "content": "Hello"}] + api_base = "https://test.services.ai.azure.com/anthropic/v1/messages" + custom_llm_provider = "azure_anthropic" + custom_prompt_dict = {} + model_response = ModelResponse() + print_verbose = MagicMock() + encoding = MagicMock() + api_key = "test-api-key" + logging_obj = MagicMock() + optional_params = {} + timeout = 60.0 + litellm_params = {"api_key": "test-api-key"} + headers = {} + + # Mock HTTP client + mock_client = MagicMock() + mock_response_obj = MagicMock() + mock_response_obj.status_code = 200 + mock_response_obj.text = json.dumps({ + "id": "test-id", + "model": "claude-sonnet-4-5", + "content": [{"type": "text", "text": "Hello!"}], + "stop_reason": "end_turn", + "usage": {"input_tokens": 10, "output_tokens": 5}, + }) + mock_response_obj.json.return_value = { + "id": "test-id", + "model": "claude-sonnet-4-5", + "content": [{"type": "text", "text": "Hello!"}], + "stop_reason": "end_turn", + "usage": {"input_tokens": 10, "output_tokens": 5}, + } + mock_client.post.return_value = mock_response_obj + mock_get_client.return_value = mock_client + + result = handler.completion( + model=model, + messages=messages, + api_base=api_base, + custom_llm_provider=custom_llm_provider, + custom_prompt_dict=custom_prompt_dict, + model_response=model_response, + print_verbose=print_verbose, + encoding=encoding, + api_key=api_key, + logging_obj=logging_obj, + optional_params=optional_params, + timeout=timeout, + litellm_params=litellm_params, + headers=headers, + client=None, # Let it create the client + acompletion=False, + ) + + # Verify non-streaming was handled + mock_client.post.assert_called_once() + assert result is not None + diff --git a/tests/test_litellm/llms/azure/anthropic/test_azure_anthropic_messages_transformation.py b/tests/test_litellm/llms/azure/anthropic/test_azure_anthropic_messages_transformation.py new file mode 100644 index 00000000000..abed1a7852e --- /dev/null +++ b/tests/test_litellm/llms/azure/anthropic/test_azure_anthropic_messages_transformation.py @@ -0,0 +1,241 @@ +import os +import sys + +sys.path.insert( + 0, os.path.abspath(os.path.join(os.path.dirname(__file__), "../../../../..")) +) + +from unittest.mock import MagicMock, patch + +import pytest + +from litellm.llms.azure.anthropic.messages_transformation import ( + AzureAnthropicMessagesConfig, +) +from litellm.types.router import GenericLiteLLMParams + + +class TestAzureAnthropicMessagesConfig: + def test_inherits_from_anthropic_messages_config(self): + """Test that AzureAnthropicMessagesConfig inherits from AnthropicMessagesConfig""" + config = AzureAnthropicMessagesConfig() + assert isinstance(config, AzureAnthropicMessagesConfig) + # Check that it has methods from parent class + assert hasattr(config, "get_supported_anthropic_messages_params") + assert hasattr(config, "get_complete_url") + assert hasattr(config, "validate_anthropic_messages_environment") + assert hasattr(config, "transform_anthropic_messages_request") + assert hasattr(config, "transform_anthropic_messages_response") + + def test_validate_anthropic_messages_environment_with_dict_litellm_params(self): + """Test validate_anthropic_messages_environment with dict litellm_params""" + config = AzureAnthropicMessagesConfig() + headers = {} + model = "claude-sonnet-4-5" + messages = [{"role": "user", "content": "Hello"}] + optional_params = {} + litellm_params = {"api_key": "test-api-key"} + api_key = "test-api-key" + + with patch( + "litellm.llms.azure.common_utils.BaseAzureLLM._base_validate_azure_environment" + ) as mock_validate: + mock_validate.return_value = {"api-key": "test-api-key"} + result, api_base = config.validate_anthropic_messages_environment( + headers=headers, + model=model, + messages=messages, + optional_params=optional_params, + litellm_params=litellm_params, + api_key=api_key, + ) + + # Verify that dict was converted to GenericLiteLLMParams + call_args = mock_validate.call_args + assert isinstance(call_args[1]["litellm_params"], GenericLiteLLMParams) + assert call_args[1]["litellm_params"].api_key == "test-api-key" + assert "anthropic-version" in result + assert "x-api-key" in result + assert result["x-api-key"] == "test-api-key" + assert "api-key" not in result + + def test_validate_anthropic_messages_environment_converts_api_key_to_x_api_key(self): + """Test that api-key header is converted to x-api-key""" + config = AzureAnthropicMessagesConfig() + headers = {} + model = "claude-sonnet-4-5" + messages = [{"role": "user", "content": "Hello"}] + optional_params = {} + litellm_params = {"api_key": "test-api-key"} + + with patch( + "litellm.llms.azure.common_utils.BaseAzureLLM._base_validate_azure_environment" + ) as mock_validate: + mock_validate.return_value = {"api-key": "test-api-key"} + result, api_base = config.validate_anthropic_messages_environment( + headers=headers, + model=model, + messages=messages, + optional_params=optional_params, + litellm_params=litellm_params, + ) + + # Verify api-key was converted to x-api-key + assert "x-api-key" in result + assert result["x-api-key"] == "test-api-key" + assert "api-key" not in result + + def test_validate_anthropic_messages_environment_sets_headers(self): + """Test that required headers are set""" + config = AzureAnthropicMessagesConfig() + headers = {} + model = "claude-sonnet-4-5" + messages = [{"role": "user", "content": "Hello"}] + optional_params = {} + litellm_params = {"api_key": "test-api-key"} + + with patch( + "litellm.llms.azure.common_utils.BaseAzureLLM._base_validate_azure_environment" + ) as mock_validate: + mock_validate.return_value = {"api-key": "test-api-key"} + result, api_base = config.validate_anthropic_messages_environment( + headers=headers, + model=model, + messages=messages, + optional_params=optional_params, + litellm_params=litellm_params, + ) + + assert "anthropic-version" in result + assert result["anthropic-version"] == "2023-06-01" + assert "content-type" in result + assert result["content-type"] == "application/json" + assert "x-api-key" in result + + def test_get_complete_url_with_base_url(self): + """Test get_complete_url with base URL""" + config = AzureAnthropicMessagesConfig() + api_base = "https://test.services.ai.azure.com/anthropic" + api_key = "test-api-key" + model = "claude-sonnet-4-5" + optional_params = {} + litellm_params = {} + + url = config.get_complete_url( + api_base=api_base, + api_key=api_key, + model=model, + optional_params=optional_params, + litellm_params=litellm_params, + ) + + assert url == "https://test.services.ai.azure.com/anthropic/v1/messages" + + def test_get_complete_url_with_base_url_ending_with_slash(self): + """Test get_complete_url with base URL ending with slash""" + config = AzureAnthropicMessagesConfig() + api_base = "https://test.services.ai.azure.com/anthropic/" + api_key = "test-api-key" + model = "claude-sonnet-4-5" + optional_params = {} + litellm_params = {} + + url = config.get_complete_url( + api_base=api_base, + api_key=api_key, + model=model, + optional_params=optional_params, + litellm_params=litellm_params, + ) + + assert url == "https://test.services.ai.azure.com/anthropic/v1/messages" + + def test_get_complete_url_with_base_url_already_containing_v1_messages(self): + """Test get_complete_url with base URL already containing /v1/messages""" + config = AzureAnthropicMessagesConfig() + api_base = "https://test.services.ai.azure.com/anthropic/v1/messages" + api_key = "test-api-key" + model = "claude-sonnet-4-5" + optional_params = {} + litellm_params = {} + + url = config.get_complete_url( + api_base=api_base, + api_key=api_key, + model=model, + optional_params=optional_params, + litellm_params=litellm_params, + ) + + assert url == "https://test.services.ai.azure.com/anthropic/v1/messages" + + def test_get_complete_url_with_base_url_containing_anthropic(self): + """Test get_complete_url with base URL already containing /anthropic""" + config = AzureAnthropicMessagesConfig() + api_base = "https://test.services.ai.azure.com/anthropic" + api_key = "test-api-key" + model = "claude-sonnet-4-5" + optional_params = {} + litellm_params = {} + + url = config.get_complete_url( + api_base=api_base, + api_key=api_key, + model=model, + optional_params=optional_params, + litellm_params=litellm_params, + ) + + assert url == "https://test.services.ai.azure.com/anthropic/v1/messages" + + def test_get_complete_url_with_base_url_without_anthropic(self): + """Test get_complete_url with base URL without /anthropic""" + config = AzureAnthropicMessagesConfig() + api_base = "https://test.services.ai.azure.com" + api_key = "test-api-key" + model = "claude-sonnet-4-5" + optional_params = {} + litellm_params = {} + + url = config.get_complete_url( + api_base=api_base, + api_key=api_key, + model=model, + optional_params=optional_params, + litellm_params=litellm_params, + ) + + assert url == "https://test.services.ai.azure.com/anthropic/v1/messages" + + def test_get_complete_url_raises_error_when_api_base_missing(self): + """Test get_complete_url raises error when api_base is None""" + config = AzureAnthropicMessagesConfig() + api_base = None + api_key = "test-api-key" + model = "claude-sonnet-4-5" + optional_params = {} + litellm_params = {} + + with patch("litellm.secret_managers.main.get_secret_str", return_value=None): + with pytest.raises(ValueError, match="Missing Azure API Base"): + config.get_complete_url( + api_base=api_base, + api_key=api_key, + model=model, + optional_params=optional_params, + litellm_params=litellm_params, + ) + + def test_get_supported_anthropic_messages_params(self): + """Test get_supported_anthropic_messages_params returns correct params""" + config = AzureAnthropicMessagesConfig() + model = "claude-sonnet-4-5" + params = config.get_supported_anthropic_messages_params(model) + + assert "messages" in params + assert "model" in params + assert "max_tokens" in params + assert "temperature" in params + assert "tools" in params + assert "tool_choice" in params + diff --git a/tests/test_litellm/llms/azure/anthropic/test_azure_anthropic_provider_config.py b/tests/test_litellm/llms/azure/anthropic/test_azure_anthropic_provider_config.py new file mode 100644 index 00000000000..db118154eee --- /dev/null +++ b/tests/test_litellm/llms/azure/anthropic/test_azure_anthropic_provider_config.py @@ -0,0 +1,59 @@ +import os +import sys + +sys.path.insert( + 0, os.path.abspath(os.path.join(os.path.dirname(__file__), "../../../../..")) +) + +from unittest.mock import patch + +import pytest + +import litellm +from litellm.types.utils import LlmProviders +from litellm.utils import ProviderConfigManager + + +class TestAzureAnthropicProviderConfig: + def test_get_provider_anthropic_messages_config_returns_azure_config(self): + """Test that get_provider_anthropic_messages_config returns AzureAnthropicMessagesConfig for azure_anthropic provider""" + from litellm.llms.azure.anthropic.messages_transformation import ( + AzureAnthropicMessagesConfig, + ) + + config = ProviderConfigManager.get_provider_anthropic_messages_config( + model="claude-sonnet-4-5", + provider=LlmProviders.AZURE_ANTHROPIC, + ) + + assert config is not None + assert isinstance(config, AzureAnthropicMessagesConfig) + + def test_get_provider_anthropic_messages_config_returns_anthropic_config_for_anthropic_provider(self): + """Test that get_provider_anthropic_messages_config returns AnthropicMessagesConfig for anthropic provider""" + from litellm.llms.azure.anthropic.messages_transformation import ( + AzureAnthropicMessagesConfig, + ) + + config = ProviderConfigManager.get_provider_anthropic_messages_config( + model="claude-sonnet-4-5", + provider=LlmProviders.ANTHROPIC, + ) + + # Should return AnthropicMessagesConfig, not AzureAnthropicMessagesConfig + assert config is not None + assert not isinstance(config, AzureAnthropicMessagesConfig) + assert isinstance(config, litellm.AnthropicMessagesConfig) + + def test_get_provider_chat_config_returns_azure_anthropic_config(self): + """Test that get_provider_chat_config returns AzureAnthropicConfig for azure_anthropic provider""" + from litellm.llms.azure.anthropic.transformation import AzureAnthropicConfig + + config = ProviderConfigManager.get_provider_chat_config( + model="claude-sonnet-4-5", + provider=LlmProviders.AZURE_ANTHROPIC, + ) + + assert config is not None + assert isinstance(config, AzureAnthropicConfig) + diff --git a/tests/test_litellm/llms/azure/anthropic/test_azure_anthropic_provider_routing.py b/tests/test_litellm/llms/azure/anthropic/test_azure_anthropic_provider_routing.py new file mode 100644 index 00000000000..a7aa2983175 --- /dev/null +++ b/tests/test_litellm/llms/azure/anthropic/test_azure_anthropic_provider_routing.py @@ -0,0 +1,82 @@ +import os +import sys + +sys.path.insert( + 0, os.path.abspath(os.path.join(os.path.dirname(__file__), "../../../../..")) +) + +import pytest + +from litellm.litellm_core_utils.get_llm_provider_logic import _is_azure_anthropic_model, get_llm_provider + + +class TestAzureAnthropicProviderRouting: + def test_is_azure_anthropic_model_with_claude(self): + """Test _is_azure_anthropic_model detects Claude models""" + # Test various Claude model names + assert _is_azure_anthropic_model("azure/claude-sonnet-4-5") == "claude-sonnet-4-5" + assert _is_azure_anthropic_model("azure/claude-opus-4-1") == "claude-opus-4-1" + assert _is_azure_anthropic_model("azure/claude-haiku-4-5") == "claude-haiku-4-5" + assert _is_azure_anthropic_model("azure/claude-3-5-sonnet") == "claude-3-5-sonnet" + assert _is_azure_anthropic_model("azure/claude-3-opus") == "claude-3-opus" + + def test_is_azure_anthropic_model_case_insensitive(self): + """Test _is_azure_anthropic_model is case insensitive""" + assert _is_azure_anthropic_model("azure/CLAUDE-sonnet-4-5") == "CLAUDE-sonnet-4-5" + assert _is_azure_anthropic_model("azure/Claude-Sonnet-4-5") == "Claude-Sonnet-4-5" + + def test_is_azure_anthropic_model_with_non_claude(self): + """Test _is_azure_anthropic_model returns None for non-Claude models""" + assert _is_azure_anthropic_model("azure/gpt-4") is None + assert _is_azure_anthropic_model("azure/gpt-35-turbo") is None + assert _is_azure_anthropic_model("azure/command-r-plus") is None + + def test_is_azure_anthropic_model_with_invalid_format(self): + """Test _is_azure_anthropic_model handles invalid formats""" + assert _is_azure_anthropic_model("azure") is None + assert _is_azure_anthropic_model("claude-sonnet-4-5") is None + assert _is_azure_anthropic_model("") is None + + def test_get_llm_provider_routes_azure_claude_to_azure_anthropic(self): + """Test that get_llm_provider routes azure/claude-* models to azure_anthropic""" + model, provider, dynamic_api_key, api_base = get_llm_provider( + model="azure/claude-sonnet-4-5" + ) + assert provider == "azure_anthropic" + assert model == "claude-sonnet-4-5" # Should strip "azure/" prefix + + def test_get_llm_provider_routes_azure_claude_opus(self): + """Test routing for Claude Opus models""" + model, provider, dynamic_api_key, api_base = get_llm_provider( + model="azure/claude-opus-4-1" + ) + assert provider == "azure_anthropic" + assert model == "claude-opus-4-1" + + def test_get_llm_provider_routes_azure_claude_haiku(self): + """Test routing for Claude Haiku models""" + model, provider, dynamic_api_key, api_base = get_llm_provider( + model="azure/claude-haiku-4-5" + ) + assert provider == "azure_anthropic" + assert model == "claude-haiku-4-5" + + def test_get_llm_provider_does_not_route_non_claude_azure_models(self): + """Test that non-Claude Azure models are not routed to azure_anthropic""" + model, provider, dynamic_api_key, api_base = get_llm_provider( + model="azure/gpt-4" + ) + assert provider != "azure_anthropic" + # Should be routed to regular azure provider + assert provider == "azure" or provider == "openai" + + def test_get_llm_provider_with_custom_llm_provider_override(self): + """Test that custom_llm_provider parameter can override routing""" + model, provider, dynamic_api_key, api_base = get_llm_provider( + model="azure/claude-sonnet-4-5", custom_llm_provider="azure" + ) + # When custom_llm_provider is explicitly set, it should be respected + # But the routing logic should still detect it as azure_anthropic + # This depends on the order of checks in get_llm_provider + assert provider in ["azure_anthropic", "azure"] + diff --git a/tests/test_litellm/llms/azure/anthropic/test_azure_anthropic_transformation.py b/tests/test_litellm/llms/azure/anthropic/test_azure_anthropic_transformation.py new file mode 100644 index 00000000000..f26831e9195 --- /dev/null +++ b/tests/test_litellm/llms/azure/anthropic/test_azure_anthropic_transformation.py @@ -0,0 +1,193 @@ +import os +import sys + +sys.path.insert( + 0, os.path.abspath(os.path.join(os.path.dirname(__file__), "../../../../..")) +) + +from unittest.mock import MagicMock, patch + +import pytest + +from litellm.llms.azure.anthropic.transformation import AzureAnthropicConfig +from litellm.types.router import GenericLiteLLMParams + + +class TestAzureAnthropicConfig: + def test_custom_llm_provider(self): + """Test that custom_llm_provider returns 'azure_anthropic'""" + config = AzureAnthropicConfig() + assert config.custom_llm_provider == "azure_anthropic" + + def test_validate_environment_with_dict_litellm_params(self): + """Test validate_environment with dict litellm_params""" + config = AzureAnthropicConfig() + headers = {} + model = "claude-sonnet-4-5" + messages = [{"role": "user", "content": "Hello"}] + optional_params = {} + litellm_params = {"api_key": "test-api-key"} + api_key = "test-api-key" + + with patch( + "litellm.llms.azure.common_utils.BaseAzureLLM._base_validate_azure_environment" + ) as mock_validate: + mock_validate.return_value = {"api-key": "test-api-key"} + result = config.validate_environment( + headers=headers, + model=model, + messages=messages, + optional_params=optional_params, + litellm_params=litellm_params, + api_key=api_key, + ) + + # Verify that dict was converted to GenericLiteLLMParams + call_args = mock_validate.call_args + assert isinstance(call_args[1]["litellm_params"], GenericLiteLLMParams) + assert call_args[1]["litellm_params"].api_key == "test-api-key" + assert "anthropic-version" in result + + def test_validate_environment_with_generic_litellm_params(self): + """Test validate_environment with GenericLiteLLMParams object""" + config = AzureAnthropicConfig() + headers = {} + model = "claude-sonnet-4-5" + messages = [{"role": "user", "content": "Hello"}] + optional_params = {} + litellm_params = GenericLiteLLMParams(api_key="test-api-key") + api_key = "test-api-key" + + with patch( + "litellm.llms.azure.common_utils.BaseAzureLLM._base_validate_azure_environment" + ) as mock_validate: + mock_validate.return_value = {"api-key": "test-api-key"} + result = config.validate_environment( + headers=headers, + model=model, + messages=messages, + optional_params=optional_params, + litellm_params=litellm_params, + api_key=api_key, + ) + + # Verify that GenericLiteLLMParams was passed through + call_args = mock_validate.call_args + assert isinstance(call_args[1]["litellm_params"], GenericLiteLLMParams) + assert "anthropic-version" in result + + def test_validate_environment_sets_api_key_in_litellm_params(self): + """Test that api_key parameter is set in litellm_params if provided""" + config = AzureAnthropicConfig() + headers = {} + model = "claude-sonnet-4-5" + messages = [{"role": "user", "content": "Hello"}] + optional_params = {} + litellm_params = {} # Empty dict, no api_key + api_key = "provided-api-key" + + with patch( + "litellm.llms.azure.common_utils.BaseAzureLLM._base_validate_azure_environment" + ) as mock_validate: + mock_validate.return_value = {"api-key": "provided-api-key"} + config.validate_environment( + headers=headers, + model=model, + messages=messages, + optional_params=optional_params, + litellm_params=litellm_params, + api_key=api_key, + ) + + # Verify that api_key was set in litellm_params + call_args = mock_validate.call_args + assert call_args[1]["litellm_params"].api_key == "provided-api-key" + + def test_validate_environment_converts_api_key_to_x_api_key(self): + """Test that api-key header is converted to x-api-key (Azure Anthropic uses x-api-key)""" + config = AzureAnthropicConfig() + headers = {} + model = "claude-sonnet-4-5" + messages = [{"role": "user", "content": "Hello"}] + optional_params = {} + litellm_params = {"api_key": "test-api-key"} + + with patch( + "litellm.llms.azure.common_utils.BaseAzureLLM._base_validate_azure_environment" + ) as mock_validate: + mock_validate.return_value = {"api-key": "test-api-key"} + with patch.object( + config, "get_anthropic_headers", return_value={} + ): + result = config.validate_environment( + headers=headers, + model=model, + messages=messages, + optional_params=optional_params, + litellm_params=litellm_params, + ) + + # Verify api-key was converted to x-api-key + assert "x-api-key" in result + assert result["x-api-key"] == "test-api-key" + assert "api-key" not in result + + def test_validate_environment_sets_anthropic_version(self): + """Test that anthropic-version header is set""" + config = AzureAnthropicConfig() + headers = {} + model = "claude-sonnet-4-5" + messages = [{"role": "user", "content": "Hello"}] + optional_params = {} + litellm_params = {"api_key": "test-api-key"} + + with patch( + "litellm.llms.azure.common_utils.BaseAzureLLM._base_validate_azure_environment" + ) as mock_validate: + mock_validate.return_value = {"api-key": "test-api-key"} + with patch.object(config, "get_anthropic_headers", return_value={}): + result = config.validate_environment( + headers=headers, + model=model, + messages=messages, + optional_params=optional_params, + litellm_params=litellm_params, + ) + + assert result["anthropic-version"] == "2023-06-01" + + def test_validate_environment_preserves_existing_anthropic_version(self): + """Test that existing anthropic-version header is preserved""" + config = AzureAnthropicConfig() + headers = {"anthropic-version": "2024-01-01"} + model = "claude-sonnet-4-5" + messages = [{"role": "user", "content": "Hello"}] + optional_params = {} + litellm_params = {"api_key": "test-api-key"} + + with patch( + "litellm.llms.azure.common_utils.BaseAzureLLM._base_validate_azure_environment" + ) as mock_validate: + mock_validate.return_value = {"api-key": "test-api-key", "anthropic-version": "2024-01-01"} + with patch.object(config, "get_anthropic_headers", return_value={"anthropic-version": "2024-01-01"}): + result = config.validate_environment( + headers=headers, + model=model, + messages=messages, + optional_params=optional_params, + litellm_params=litellm_params, + ) + + assert result["anthropic-version"] == "2024-01-01" + + def test_inherits_anthropic_config_methods(self): + """Test that AzureAnthropicConfig inherits methods from AnthropicConfig""" + config = AzureAnthropicConfig() + + # Test that it has AnthropicConfig methods + assert hasattr(config, "get_anthropic_headers") + assert hasattr(config, "is_cache_control_set") + assert hasattr(config, "is_computer_tool_used") + assert hasattr(config, "transform_request") + assert hasattr(config, "transform_response") + diff --git a/tests/test_litellm/llms/azure/chat/test_azure_gpt5_transformation.py b/tests/test_litellm/llms/azure/chat/test_azure_gpt5_transformation.py index 2ef2020b09a..3095ff87f5a 100644 --- a/tests/test_litellm/llms/azure/chat/test_azure_gpt5_transformation.py +++ b/tests/test_litellm/llms/azure/chat/test_azure_gpt5_transformation.py @@ -101,3 +101,83 @@ def test_azure_gpt5_codex_series_transform_request(config: AzureOpenAIGPT5Config ) assert request["model"] == "gpt-5-codex" + +# GPT-5.1 temperature handling tests for Azure +def test_azure_gpt5_1_temperature_with_reasoning_effort_none(config: AzureOpenAIGPT5Config): + """Test that Azure GPT-5.1 supports any temperature when reasoning_effort='none' and drop_params=True. + + Note: Azure OpenAI doesn't support reasoning_effort='none', so it's dropped from the params + when drop_params=True. The temperature logic still works correctly because the parent treats + missing reasoning_effort the same as 'none' for gpt-5.1. + """ + params = config.map_openai_params( + non_default_params={"temperature": 0.5, "reasoning_effort": "none"}, + optional_params={}, + model="azure/gpt-5.1", + drop_params=True, + api_version="2024-05-01-preview", + ) + assert params["temperature"] == 0.5 + # Azure doesn't support reasoning_effort="none", so it should be dropped + assert "reasoning_effort" not in params or params.get("reasoning_effort") != "none" + + +def test_azure_gpt5_1_reasoning_effort_none_error_when_drop_params_false(config: AzureOpenAIGPT5Config): + """Test that Azure GPT-5.1 raises error for reasoning_effort='none' when drop_params=False.""" + with pytest.raises(litellm.utils.UnsupportedParamsError): + config.map_openai_params( + non_default_params={"reasoning_effort": "none"}, + optional_params={}, + model="azure/gpt-5.1", + drop_params=False, + api_version="2024-05-01-preview", + ) + + +def test_azure_gpt5_1_temperature_without_reasoning_effort(config: AzureOpenAIGPT5Config): + """Test that Azure GPT-5.1 supports any temperature when reasoning_effort is not specified.""" + params = config.map_openai_params( + non_default_params={"temperature": 0.7}, + optional_params={}, + model="azure/gpt-5.1", + drop_params=False, + api_version="2024-05-01-preview", + ) + assert params["temperature"] == 0.7 + + +def test_azure_gpt5_1_temperature_with_reasoning_effort_other_values(config: AzureOpenAIGPT5Config): + """Test that Azure GPT-5.1 only allows temperature=1 when reasoning_effort is not 'none'.""" + # Test that temperature != 1 raises error when reasoning_effort is set to other values + with pytest.raises(litellm.utils.UnsupportedParamsError): + config.map_openai_params( + non_default_params={"temperature": 0.7, "reasoning_effort": "low"}, + optional_params={}, + model="azure/gpt-5.1", + drop_params=False, + api_version="2024-05-01-preview", + ) + + # Test that temperature=1 is allowed with other reasoning_effort values + params = config.map_openai_params( + non_default_params={"temperature": 1.0, "reasoning_effort": "medium"}, + optional_params={}, + model="azure/gpt-5.1", + drop_params=False, + api_version="2024-05-01-preview", + ) + assert params["temperature"] == 1.0 + assert params["reasoning_effort"] == "medium" + + +def test_azure_gpt5_1_series_temperature_handling(config: AzureOpenAIGPT5Config): + """Test that Azure GPT-5.1 with gpt5_series prefix supports temperature with reasoning_effort='none'.""" + params = config.map_openai_params( + non_default_params={"temperature": 0.6}, + optional_params={}, + model="gpt5_series/gpt-5.1", + drop_params=False, + api_version="2024-05-01-preview", + ) + assert params["temperature"] == 0.6 + diff --git a/tests/test_litellm/llms/azure/videos/test_azure_video_transformation.py b/tests/test_litellm/llms/azure/videos/test_azure_video_transformation.py index 640933179a6..b3d7945db39 100644 --- a/tests/test_litellm/llms/azure/videos/test_azure_video_transformation.py +++ b/tests/test_litellm/llms/azure/videos/test_azure_video_transformation.py @@ -65,8 +65,13 @@ class TestAzureVideoConfig: assert result["size"] == "1280x720" assert result["user"] == "test_user" - def test_validate_environment_with_api_key(self): - """Test environment validation with provided API key.""" + @patch('litellm.llms.azure.common_utils.litellm') + def test_validate_environment_with_api_key(self, mock_litellm): + """Test environment validation with provided API key - should use api-key header for Azure.""" + # Since validate_environment passes litellm_params=None, it relies on litellm.api_key or litellm.azure_key + mock_litellm.api_key = self.api_key + mock_litellm.azure_key = None + headers = {"Content-Type": "application/json"} result_headers = self.config.validate_environment( @@ -75,14 +80,15 @@ class TestAzureVideoConfig: api_key=self.api_key ) - assert "Authorization" in result_headers - assert result_headers["Authorization"] == f"Bearer {self.api_key}" + # Azure uses "api-key" header, not "Authorization: Bearer" + assert "api-key" in result_headers + assert result_headers["api-key"] == self.api_key assert result_headers["Content-Type"] == "application/json" - @patch('litellm.llms.azure.videos.transformation.get_secret_str') - @patch('litellm.llms.azure.videos.transformation.litellm') + @patch('litellm.llms.azure.common_utils.get_secret_str') + @patch('litellm.llms.azure.common_utils.litellm') def test_validate_environment_without_api_key(self, mock_litellm, mock_get_secret): - """Test environment validation without provided API key.""" + """Test environment validation without provided API key - should fallback to secret manager.""" mock_litellm.api_key = None mock_litellm.azure_key = None mock_get_secret.return_value = "secret-api-key" @@ -95,8 +101,8 @@ class TestAzureVideoConfig: api_key=None ) - assert "Authorization" in result_headers - assert result_headers["Authorization"] == "Bearer secret-api-key" + assert "api-key" in result_headers + assert result_headers["api-key"] == "secret-api-key" def test_get_complete_url(self): """Test URL construction for Azure video API.""" @@ -320,23 +326,24 @@ class TestAzureVideoConfig: logging_obj=logging_obj ) - def test_azure_specific_environment_validation(self): + @patch('litellm.llms.azure.common_utils.litellm') + def test_azure_specific_environment_validation(self, mock_litellm): """Test Azure-specific environment validation with different key sources.""" + # Test with azure_key + mock_litellm.api_key = None + mock_litellm.azure_key = "azure-test-key" + mock_litellm.openai_key = None + headers = {"Content-Type": "application/json"} - # Test with azure_key - with patch('litellm.llms.azure.videos.transformation.litellm') as mock_litellm: - mock_litellm.api_key = None - mock_litellm.azure_key = "azure-test-key" - mock_litellm.openai_key = None - - result_headers = self.config.validate_environment( - headers=headers, - model=self.model, - api_key=None - ) - - assert result_headers["Authorization"] == "Bearer azure-test-key" + result_headers = self.config.validate_environment( + headers=headers, + model=self.model, + api_key=None + ) + + assert "api-key" in result_headers + assert result_headers["api-key"] == "azure-test-key" def test_usage_data_creation_in_video_create(self): """Test that usage data is created correctly in video create response.""" diff --git a/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py b/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py index c09d3b8d841..37c95be72ce 100644 --- a/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py +++ b/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py @@ -238,6 +238,30 @@ def test_transform_tool_call_with_cache_control(): assert "cachePoint" in transformed_cache_msg assert transformed_cache_msg["cachePoint"]["type"] == "default" + +def test_reasoning_with_forced_tool_choice_switches_to_auto(): + config = AmazonConverseConfig() + + non_default_params = { + "tools": [ + { + "type": "function", + "function": {"name": "get_current_weather", "parameters": {}}, + } + ], + "tool_choice": "required", + "reasoning_effort": "low", + } + + optional_params = config.map_openai_params( + model="bedrock/us.anthropic.claude-3-7-sonnet-20250219-v1:0", + non_default_params=non_default_params, + optional_params={}, + drop_params=False, + ) + + assert optional_params["tool_choice"] == {"auto": {}} + def test_get_supported_openai_params(): config = AmazonConverseConfig() supported_params = config.get_supported_openai_params( @@ -2592,8 +2616,10 @@ def test_empty_assistant_message_handling(): empty or whitespace-only content with a placeholder to prevent AWS Bedrock Converse API 400 Bad Request errors. """ - from litellm.litellm_core_utils.prompt_templates.factory import _bedrock_converse_messages_pt - + from litellm.litellm_core_utils.prompt_templates.factory import ( + _bedrock_converse_messages_pt, + ) + # Test case 1: Empty string content - test with modify_params=True to prevent merging messages = [ {"role": "user", "content": "Hello"}, diff --git a/tests/test_litellm/llms/bedrock/embed/test_bedrock_embedding.py b/tests/test_litellm/llms/bedrock/embed/test_bedrock_embedding.py index f436c66f203..a266bea3513 100644 --- a/tests/test_litellm/llms/bedrock/embed/test_bedrock_embedding.py +++ b/tests/test_litellm/llms/bedrock/embed/test_bedrock_embedding.py @@ -404,4 +404,154 @@ def test_twelvelabs_missing_input_type_error(): ) # Should succeed without input_type - assert isinstance(response, litellm.EmbeddingResponse) \ No newline at end of file + assert isinstance(response, litellm.EmbeddingResponse) + + +@pytest.mark.parametrize( + "model,embed_response", + [ + ("bedrock/amazon.titan-embed-text-v1", titan_embedding_response), + ("bedrock/amazon.titan-embed-text-v2:0", titan_embedding_response), + ("bedrock/cohere.embed-english-v3", cohere_embedding_response), + ], +) +def test_bedrock_embedding_header_forwarding(model, embed_response): + """ + Test that custom headers are correctly forwarded to Bedrock embedding API calls. + + This test verifies the fix for the issue where headers configured via + forward_client_headers_to_llm_api were not being passed to Bedrock embedding provider. + + Relevant Issue: https://github.com/BerriAI/litellm/pull/16042 + """ + litellm.set_verbose = True + client = HTTPHandler() + test_api_key = "test-bearer-token-12345" + + # Headers that would be set by the proxy when forwarding client headers + custom_headers = { + "X-Custom-Header": "CustomValue", + "X-BYOK-Token": "secret-token", + "Extra-Header": "foobar", + } + + with patch.object(client, "post") as mock_post: + mock_response = Mock() + mock_response.status_code = 200 + mock_response.text = json.dumps(embed_response) + mock_response.json = lambda: json.loads(mock_response.text) + mock_post.return_value = mock_response + + try: + # Call embedding with custom headers via kwargs + # This simulates what the proxy does when forward_client_headers_to_llm_api is set + response = litellm.embedding( + model=model, + input=test_input, + client=client, + headers=custom_headers, # This is how proxy passes forwarded headers + aws_region_name="us-east-1", + aws_bedrock_runtime_endpoint="https://bedrock-runtime.us-east-1.amazonaws.com", + api_key=test_api_key, + ) + + assert isinstance(response, litellm.EmbeddingResponse) + + # Verify that the request was made + assert mock_post.called, "HTTP client post should be called" + + # Get the actual call arguments + call_kwargs = mock_post.call_args.kwargs + headers = call_kwargs.get("headers", {}) + + # Verify our custom headers are present in the request headers + # Note: AWS SigV4 signing may modify header names to lowercase + for header_key, header_value in custom_headers.items(): + header_found = ( + header_key in headers + or header_key.lower() in headers + or any(k.lower() == header_key.lower() for k in headers.keys()) + ) + assert header_found, ( + f"Header {header_key} should be in request headers. " + f"Found headers: {list(headers.keys())}" + ) + + print(f"✓ Test passed for {model}") + print(f" Headers correctly forwarded: {list(headers.keys())}") + + except Exception as e: + pytest.fail(f"Failed to forward headers to {model}: {str(e)}") + + +def test_bedrock_embedding_extra_headers_and_headers_merge(): + """ + Test that both extra_headers and headers parameters are correctly merged for Bedrock embeddings. + + This ensures that headers from kwargs (forwarded by proxy) and extra_headers + (passed explicitly) are both included in the final headers sent to the provider. + """ + litellm.set_verbose = True + client = HTTPHandler() + test_api_key = "test-bearer-token-12345" + model = "bedrock/amazon.titan-embed-text-v1" + + # Headers from proxy (via kwargs["headers"]) + proxy_headers = {"X-Forwarded-Header": "ProxyValue"} + + # Explicit extra_headers + explicit_headers = {"X-Explicit-Header": "ExplicitValue"} + + # Mock response + embed_response = { + "embedding": [0.1, 0.2, 0.3], + "inputTextTokenCount": 10 + } + + with patch.object(client, "post") as mock_post: + mock_response = Mock() + mock_response.status_code = 200 + mock_response.text = json.dumps(embed_response) + mock_response.json = lambda: json.loads(mock_response.text) + mock_post.return_value = mock_response + + try: + response = litellm.embedding( + model=model, + input=test_input, + client=client, + headers=proxy_headers, # From proxy forwarding + extra_headers=explicit_headers, # Explicitly passed + aws_region_name="us-east-1", + aws_bedrock_runtime_endpoint="https://bedrock-runtime.us-east-1.amazonaws.com", + api_key=test_api_key, + ) + + assert isinstance(response, litellm.EmbeddingResponse) + + call_kwargs = mock_post.call_args.kwargs + headers = call_kwargs.get("headers", {}) + + # Both sets of headers should be present + # Note: AWS SigV4 signing may modify header names to lowercase + proxy_header_found = any( + k.lower() == "x-forwarded-header" for k in headers.keys() + ) + assert proxy_header_found, ( + "Proxy forwarded header should be present. " + f"Found headers: {list(headers.keys())}" + ) + + explicit_header_found = any( + k.lower() == "x-explicit-header" for k in headers.keys() + ) + assert explicit_header_found, ( + "Explicitly passed header should be present. " + f"Found headers: {list(headers.keys())}" + ) + + print("✓ Both header sources correctly merged and forwarded") + print(f" Final headers: {list(headers.keys())}") + + except Exception as e: + pytest.fail(f"Failed to merge and forward headers: {str(e)}") \ No newline at end of file diff --git a/tests/test_litellm/llms/openai/test_gpt5_transformation.py b/tests/test_litellm/llms/openai/test_gpt5_transformation.py index 2e1eacce532..5080a7a7c59 100644 --- a/tests/test_litellm/llms/openai/test_gpt5_transformation.py +++ b/tests/test_litellm/llms/openai/test_gpt5_transformation.py @@ -209,3 +209,122 @@ def test_gpt5_1_reasoning_effort_none(config: OpenAIConfig): drop_params=False, ) assert params["reasoning_effort"] == effort + + +# GPT-5.1 temperature handling tests +def test_gpt5_1_model_detection(gpt5_config: OpenAIGPT5Config): + """Test that GPT-5.1 models are correctly detected.""" + assert gpt5_config.is_model_gpt_5_1_model("gpt-5.1") + assert gpt5_config.is_model_gpt_5_1_model("gpt-5.1-codex") + assert gpt5_config.is_model_gpt_5_1_model("gpt-5.1-chat") + assert not gpt5_config.is_model_gpt_5_1_model("gpt-5") + assert not gpt5_config.is_model_gpt_5_1_model("gpt-5-mini") + assert not gpt5_config.is_model_gpt_5_1_model("gpt-5-codex") + + +def test_gpt5_1_temperature_with_reasoning_effort_none(config: OpenAIConfig): + """Test that GPT-5.1 supports any temperature when reasoning_effort='none'.""" + # Test various temperature values with reasoning_effort="none" + for temp in [0.0, 0.2, 0.5, 0.7, 0.9, 1.0, 1.5, 2.0]: + params = config.map_openai_params( + non_default_params={"temperature": temp, "reasoning_effort": "none"}, + optional_params={}, + model="gpt-5.1", + drop_params=False, + ) + assert params["temperature"] == temp + assert params["reasoning_effort"] == "none" + + +def test_gpt5_1_temperature_without_reasoning_effort(config: OpenAIConfig): + """Test that GPT-5.1 supports any temperature when reasoning_effort is not specified. + + When reasoning_effort is not provided, it defaults to "none" for gpt-5.1, + so temperature should be allowed. + """ + # Test various temperature values without reasoning_effort (defaults to "none") + for temp in [0.0, 0.2, 0.5, 0.7, 0.9, 1.0, 1.5, 2.0]: + params = config.map_openai_params( + non_default_params={"temperature": temp}, + optional_params={}, + model="gpt-5.1", + drop_params=False, + ) + assert params["temperature"] == temp + + +def test_gpt5_1_temperature_with_reasoning_effort_other_values(config: OpenAIConfig): + """Test that GPT-5.1 only allows temperature=1 when reasoning_effort is not 'none'.""" + # Test that temperature != 1 raises error when reasoning_effort is set to other values + for effort in ["low", "medium", "high"]: + with pytest.raises(litellm.utils.UnsupportedParamsError): + config.map_openai_params( + non_default_params={"temperature": 0.7, "reasoning_effort": effort}, + optional_params={}, + model="gpt-5.1", + drop_params=False, + ) + + # Test that temperature=1 is allowed with other reasoning_effort values + for effort in ["low", "medium", "high"]: + params = config.map_openai_params( + non_default_params={"temperature": 1.0, "reasoning_effort": effort}, + optional_params={}, + model="gpt-5.1", + drop_params=False, + ) + assert params["temperature"] == 1.0 + assert params["reasoning_effort"] == effort + + +def test_gpt5_1_temperature_with_reasoning_effort_in_optional_params(config: OpenAIConfig): + """Test that reasoning_effort can be in optional_params and still work correctly.""" + # Test with reasoning_effort="none" in optional_params + params = config.map_openai_params( + non_default_params={"temperature": 0.5}, + optional_params={"reasoning_effort": "none"}, + model="gpt-5.1", + drop_params=False, + ) + assert params["temperature"] == 0.5 + + # Test with reasoning_effort="low" in optional_params (should only allow temp=1) + with pytest.raises(litellm.utils.UnsupportedParamsError): + config.map_openai_params( + non_default_params={"temperature": 0.5}, + optional_params={"reasoning_effort": "low"}, + model="gpt-5.1", + drop_params=False, + ) + +def test_gpt5_1_temperature_drop_when_not_none(config: OpenAIConfig): + """Test that GPT-5.1 drops temperature when reasoning_effort != 'none' and drop_params=True.""" + params = config.map_openai_params( + non_default_params={"temperature": 0.7, "reasoning_effort": "low"}, + optional_params={}, + model="gpt-5.1", + drop_params=True, + ) + assert "temperature" not in params + assert params["reasoning_effort"] == "low" + + +def test_gpt5_temperature_still_restricted(config: OpenAIConfig): + """Test that regular gpt-5 (not 5.1) still only allows temperature=1.""" + # Regular gpt-5 should still only allow temperature=1 + with pytest.raises(litellm.utils.UnsupportedParamsError): + config.map_openai_params( + non_default_params={"temperature": 0.7}, + optional_params={}, + model="gpt-5", + drop_params=False, + ) + + # temperature=1 should still work for gpt-5 + params = config.map_openai_params( + non_default_params={"temperature": 1.0}, + optional_params={}, + model="gpt-5", + drop_params=False, + ) + assert params["temperature"] == 1.0 diff --git a/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py b/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py index 8942239bb21..2b305dbade1 100644 --- a/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py +++ b/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py @@ -1967,3 +1967,98 @@ def test_media_resolution_per_part(): assert "inline_data" in image2_part assert image2_part["inline_data"]["mediaResolution"] == "high" + +def test_gemini_3_image_models_no_thinking_config(): + """ + Test that Gemini 3 image models do NOT receive automatic thinkingConfig. + + Related issue: https://github.com/BerriAI/litellm/issues/17013 + gemini-3-pro-image-preview does not support thinking_level parameter + and returns BadRequestError: "Thinking level is not supported for this model" + """ + from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import ( + VertexGeminiConfig, + ) + + v = VertexGeminiConfig() + + # Test gemini-3-pro-image-preview (the specific model from the bug report) + model = "gemini-3-pro-image-preview" + optional_params = {} + non_default_params = {} + + result = v.map_openai_params( + non_default_params=non_default_params, + optional_params=optional_params, + model=model, + drop_params=False, + ) + + # Should NOT have thinkingConfig automatically added + assert "thinkingConfig" not in result + # But should still get temperature=1.0 for Gemini 3 + assert result["temperature"] == 1.0 + + +def test_gemini_3_text_models_get_thinking_config(): + """ + Test that Gemini 3 text models DO receive automatic thinkingConfig. + This ensures we didn't break the existing behavior for non-image models. + """ + from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import ( + VertexGeminiConfig, + ) + + v = VertexGeminiConfig() + + # Test gemini-3-pro-preview (text model, should get thinking) + model = "gemini-3-pro-preview" + optional_params = {} + non_default_params = {} + + result = v.map_openai_params( + non_default_params=non_default_params, + optional_params=optional_params, + model=model, + drop_params=False, + ) + + # Should have thinkingConfig automatically added + assert "thinkingConfig" in result + assert result["thinkingConfig"]["thinkingLevel"] == "low" + assert result["temperature"] == 1.0 + + +def test_gemini_image_models_excluded_from_thinking(): + """ + Test that any Gemini model with 'image' in the name is excluded from thinking config. + This covers current and future image models. + """ + from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import ( + VertexGeminiConfig, + ) + + v = VertexGeminiConfig() + + # Test various image model patterns + image_models = [ + "gemini-3-pro-image-preview", + "gemini-3-pro-image-generation", + "gemini-3-flash-image-preview", + "gemini/gemini-3-image-edit", + ] + + for model in image_models: + optional_params = {} + non_default_params = {} + + result = v.map_openai_params( + non_default_params=non_default_params, + optional_params=optional_params, + model=model, + drop_params=False, + ) + + # None of these should have thinkingConfig + assert "thinkingConfig" not in result, f"Model {model} should not have thinkingConfig" + diff --git a/tests/test_litellm/llms/vertex_ai/image_generation/test_vertex_ai_image_generation_transformation.py b/tests/test_litellm/llms/vertex_ai/image_generation/test_vertex_ai_image_generation_transformation.py new file mode 100644 index 00000000000..9f33400594b --- /dev/null +++ b/tests/test_litellm/llms/vertex_ai/image_generation/test_vertex_ai_image_generation_transformation.py @@ -0,0 +1,457 @@ +import os +import sys +from unittest.mock import MagicMock, patch + +import httpx +import pytest + +sys.path.insert(0, os.path.abspath("../..")) + +from litellm.llms.vertex_ai.image_generation import ( + get_vertex_ai_image_generation_config, +) +from litellm.llms.vertex_ai.image_generation.vertex_gemini_transformation import ( + VertexAIGeminiImageGenerationConfig, +) +from litellm.llms.vertex_ai.image_generation.vertex_imagen_transformation import ( + VertexAIImagenImageGenerationConfig, +) + + +class TestVertexAIGeminiImageGenerationConfig: + def setup_method(self): + """Set up test fixtures""" + self.config = VertexAIGeminiImageGenerationConfig() + + def test_get_supported_openai_params(self): + """Test get_supported_openai_params returns correct params""" + supported = self.config.get_supported_openai_params("gemini-2.5-flash-image") + assert "n" in supported + assert "size" in supported + + def test_map_openai_params_n(self): + """Test mapping n parameter to candidate_count""" + non_default_params = {"n": 3} + optional_params = {} + result = self.config.map_openai_params( + non_default_params, optional_params, "gemini-2.5-flash-image", False + ) + assert result.get("candidate_count") == 3 + + def test_map_openai_params_size(self): + """Test mapping size parameter to aspectRatio""" + non_default_params = {"size": "1024x1024"} + optional_params = {} + result = self.config.map_openai_params( + non_default_params, optional_params, "gemini-2.5-flash-image", False + ) + assert result.get("aspectRatio") == "1:1" + + def test_map_openai_params_size_16_9(self): + """Test mapping 16:9 size""" + non_default_params = {"size": "1792x1024"} + optional_params = {} + result = self.config.map_openai_params( + non_default_params, optional_params, "gemini-2.5-flash-image", False + ) + assert result.get("aspectRatio") == "16:9" + + def test_map_size_to_aspect_ratio(self): + """Test size to aspect ratio mapping""" + assert self.config._map_size_to_aspect_ratio("1024x1024") == "1:1" + assert self.config._map_size_to_aspect_ratio("1792x1024") == "16:9" + assert self.config._map_size_to_aspect_ratio("1024x1792") == "9:16" + assert self.config._map_size_to_aspect_ratio("1280x896") == "4:3" + assert self.config._map_size_to_aspect_ratio("896x1280") == "3:4" + assert self.config._map_size_to_aspect_ratio("unknown") == "1:1" # default + + def test_transform_image_generation_request_basic(self): + """Test basic request transformation""" + request = self.config.transform_image_generation_request( + model="gemini-2.5-flash-image", + prompt="A nano banana", + optional_params={}, + litellm_params={}, + headers={}, + ) + assert "contents" in request + assert "generationConfig" in request + assert request["generationConfig"]["responseModalities"] == ["IMAGE"] + assert request["contents"][0]["parts"][0]["text"] == "A nano banana" + + def test_transform_image_generation_request_with_aspect_ratio(self): + """Test request transformation with aspectRatio""" + request = self.config.transform_image_generation_request( + model="gemini-2.5-flash-image", + prompt="A nano banana", + optional_params={"aspectRatio": "16:9"}, + litellm_params={}, + headers={}, + ) + assert request["generationConfig"]["imageConfig"]["aspectRatio"] == "16:9" + + def test_transform_image_generation_request_with_image_size(self): + """Test request transformation with imageSize (Gemini 3 Pro)""" + request = self.config.transform_image_generation_request( + model="gemini-3-pro-image-preview", + prompt="A nano banana", + optional_params={"imageSize": "4K"}, + litellm_params={}, + headers={}, + ) + assert request["generationConfig"]["imageConfig"]["imageSize"] == "4K" + + def test_transform_image_generation_request_with_candidate_count(self): + """Test request transformation with candidate_count""" + request = self.config.transform_image_generation_request( + model="gemini-2.5-flash-image", + prompt="A nano banana", + optional_params={"candidate_count": 2}, + litellm_params={}, + headers={}, + ) + assert request["generationConfig"]["candidateCount"] == 2 + + def test_transform_image_generation_request_with_n(self): + """Test request transformation with n parameter""" + request = self.config.transform_image_generation_request( + model="gemini-2.5-flash-image", + prompt="A nano banana", + optional_params={"n": 2}, + litellm_params={}, + headers={}, + ) + assert request["generationConfig"]["candidateCount"] == 2 + + def test_transform_image_generation_response(self): + """Test response transformation""" + mock_response = MagicMock(spec=httpx.Response) + mock_response.status_code = 200 + mock_response.json.return_value = { + "candidates": [ + { + "content": { + "parts": [ + { + "inlineData": { + "mimeType": "image/png", + "data": "base64_encoded_image_data", + } + } + ] + } + } + ] + } + mock_response.headers = {} + + from litellm.types.utils import ImageResponse + + model_response = ImageResponse() + result = self.config.transform_image_generation_response( + model="gemini-2.5-flash-image", + raw_response=mock_response, + model_response=model_response, + logging_obj=MagicMock(), + request_data={}, + optional_params={}, + litellm_params={}, + encoding=None, + ) + + assert len(result.data) == 1 + assert result.data[0].b64_json == "base64_encoded_image_data" + assert result.data[0].url is None + + def test_transform_image_generation_response_multiple_images(self): + """Test response transformation with multiple images""" + mock_response = MagicMock(spec=httpx.Response) + mock_response.status_code = 200 + mock_response.json.return_value = { + "candidates": [ + { + "content": { + "parts": [ + { + "inlineData": { + "mimeType": "image/png", + "data": "image1", + } + }, + { + "inlineData": { + "mimeType": "image/png", + "data": "image2", + } + }, + ] + } + } + ] + } + mock_response.headers = {} + + from litellm.types.utils import ImageResponse + + model_response = ImageResponse() + result = self.config.transform_image_generation_response( + model="gemini-2.5-flash-image", + raw_response=mock_response, + model_response=model_response, + logging_obj=MagicMock(), + request_data={}, + optional_params={}, + litellm_params={}, + encoding=None, + ) + + assert len(result.data) == 2 + assert result.data[0].b64_json == "image1" + assert result.data[1].b64_json == "image2" + + +class TestVertexAIImagenImageGenerationConfig: + def setup_method(self): + """Set up test fixtures""" + self.config = VertexAIImagenImageGenerationConfig() + + def test_get_supported_openai_params(self): + """Test get_supported_openai_params returns correct params""" + supported = self.config.get_supported_openai_params("imagegeneration@006") + assert "n" in supported + assert "size" in supported + + def test_map_openai_params_n(self): + """Test mapping n parameter to sampleCount""" + non_default_params = {"n": 3} + optional_params = {} + result = self.config.map_openai_params( + non_default_params, optional_params, "imagegeneration@006", False + ) + assert result.get("sampleCount") == 3 + + def test_map_openai_params_size(self): + """Test mapping size parameter to aspectRatio""" + non_default_params = {"size": "1024x1024"} + optional_params = {} + result = self.config.map_openai_params( + non_default_params, optional_params, "imagegeneration@006", False + ) + assert result.get("aspectRatio") == "1:1" + + def test_map_size_to_aspect_ratio(self): + """Test size to aspect ratio mapping""" + assert self.config._map_size_to_aspect_ratio("1024x1024") == "1:1" + assert self.config._map_size_to_aspect_ratio("1792x1024") == "16:9" + assert self.config._map_size_to_aspect_ratio("unknown") == "1:1" # default + + def test_transform_image_generation_request_basic(self): + """Test basic request transformation""" + request = self.config.transform_image_generation_request( + model="imagegeneration@006", + prompt="A cat", + optional_params={}, + litellm_params={}, + headers={}, + ) + assert "instances" in request + assert "parameters" in request + assert request["instances"][0]["prompt"] == "A cat" + assert request["parameters"]["sampleCount"] == 1 + + def test_transform_image_generation_request_with_params(self): + """Test request transformation with parameters""" + request = self.config.transform_image_generation_request( + model="imagegeneration@006", + prompt="A cat", + optional_params={"sampleCount": 2, "aspectRatio": "16:9"}, + litellm_params={}, + headers={}, + ) + assert request["parameters"]["sampleCount"] == 2 + assert request["parameters"]["aspectRatio"] == "16:9" + + def test_transform_image_generation_response(self): + """Test response transformation""" + mock_response = MagicMock(spec=httpx.Response) + mock_response.status_code = 200 + mock_response.json.return_value = { + "predictions": [ + {"bytesBase64Encoded": "base64_encoded_image_data"} + ] + } + mock_response.headers = {} + + from litellm.types.utils import ImageResponse + + model_response = ImageResponse() + result = self.config.transform_image_generation_response( + model="imagegeneration@006", + raw_response=mock_response, + model_response=model_response, + logging_obj=MagicMock(), + request_data={}, + optional_params={}, + litellm_params={}, + encoding=None, + ) + + assert len(result.data) == 1 + assert result.data[0].b64_json == "base64_encoded_image_data" + assert result.data[0].url is None + + def test_transform_image_generation_response_multiple_images(self): + """Test response transformation with multiple images""" + mock_response = MagicMock(spec=httpx.Response) + mock_response.status_code = 200 + mock_response.json.return_value = { + "predictions": [ + {"bytesBase64Encoded": "image1"}, + {"bytesBase64Encoded": "image2"}, + ] + } + mock_response.headers = {} + + from litellm.types.utils import ImageResponse + + model_response = ImageResponse() + result = self.config.transform_image_generation_response( + model="imagegeneration@006", + raw_response=mock_response, + model_response=model_response, + logging_obj=MagicMock(), + request_data={}, + optional_params={}, + litellm_params={}, + encoding=None, + ) + + assert len(result.data) == 2 + assert result.data[0].b64_json == "image1" + assert result.data[1].b64_json == "image2" + + +class TestGetVertexAIImageGenerationConfig: + """Test the router function that selects the correct config""" + + def test_get_gemini_model_config(self): + """Test that Gemini models return Gemini config""" + config = get_vertex_ai_image_generation_config("gemini-2.5-flash-image") + assert isinstance(config, VertexAIGeminiImageGenerationConfig) + + config = get_vertex_ai_image_generation_config("gemini-3-pro-image-preview") + assert isinstance(config, VertexAIGeminiImageGenerationConfig) + + config = get_vertex_ai_image_generation_config( + "vertex_ai/gemini-2.5-flash-image" + ) + assert isinstance(config, VertexAIGeminiImageGenerationConfig) + + def test_get_imagen_model_config(self): + """Test that Imagen models return Imagen config""" + config = get_vertex_ai_image_generation_config("imagegeneration@006") + assert isinstance(config, VertexAIImagenImageGenerationConfig) + + config = get_vertex_ai_image_generation_config("imagen-4.0-generate-001") + assert isinstance(config, VertexAIImagenImageGenerationConfig) + + config = get_vertex_ai_image_generation_config( + "vertex_ai/imagegeneration@006" + ) + assert isinstance(config, VertexAIImagenImageGenerationConfig) + + def test_get_non_gemini_model_config(self): + """Test that non-Gemini models default to Imagen config""" + config = get_vertex_ai_image_generation_config("some-other-model") + assert isinstance(config, VertexAIImagenImageGenerationConfig) + + +class TestVertexAIImageGenerationIntegration: + """Integration tests for Vertex AI image generation""" + + @pytest.mark.skipif( + not os.getenv("VERTEXAI_PROJECT"), + reason="Vertex AI credentials not set", + ) + def test_gemini_image_generation_config_validation(self): + """Test that Gemini config can validate environment""" + config = VertexAIGeminiImageGenerationConfig() + with patch.object( + config, "_resolve_vertex_project", return_value="test-project" + ), patch.object( + config, "_resolve_vertex_location", return_value="us-central1" + ), patch.object( + config, "_ensure_access_token", return_value=("token", None) + ): + headers = config.validate_environment( + headers={}, + model="gemini-2.5-flash-image", + messages=[], + optional_params={}, + litellm_params={}, + ) + assert "Authorization" in headers + + @pytest.mark.skipif( + not os.getenv("VERTEXAI_PROJECT"), + reason="Vertex AI credentials not set", + ) + def test_imagen_image_generation_config_validation(self): + """Test that Imagen config can validate environment""" + config = VertexAIImagenImageGenerationConfig() + with patch.object( + config, "_resolve_vertex_project", return_value="test-project" + ), patch.object( + config, "_resolve_vertex_location", return_value="us-central1" + ), patch.object( + config, "_ensure_access_token", return_value=("token", None) + ): + headers = config.validate_environment( + headers={}, + model="imagegeneration@006", + messages=[], + optional_params={}, + litellm_params={}, + ) + assert "Authorization" in headers + + def test_gemini_get_complete_url(self): + """Test Gemini config URL generation""" + config = VertexAIGeminiImageGenerationConfig() + with patch.object( + config, "_resolve_vertex_project", return_value="test-project" + ), patch.object( + config, "_resolve_vertex_location", return_value="us-central1" + ): + url = config.get_complete_url( + api_base=None, + api_key=None, + model="gemini-2.5-flash-image", + optional_params={}, + litellm_params={}, + ) + assert "test-project" in url + assert "us-central1" in url + assert "gemini-2.5-flash-image" in url + assert "generateContent" in url + + def test_imagen_get_complete_url(self): + """Test Imagen config URL generation""" + config = VertexAIImagenImageGenerationConfig() + with patch.object( + config, "_resolve_vertex_project", return_value="test-project" + ), patch.object( + config, "_resolve_vertex_location", return_value="us-central1" + ): + url = config.get_complete_url( + api_base=None, + api_key=None, + model="imagegeneration@006", + optional_params={}, + litellm_params={}, + ) + assert "test-project" in url + assert "us-central1" in url + assert "imagegeneration@006" in url + assert "predict" in url + diff --git a/tests/test_litellm/llms/vertex_ai/test_vertex_ai_common_utils.py b/tests/test_litellm/llms/vertex_ai/test_vertex_ai_common_utils.py index 4ea1d81c266..a5eee9e37b1 100644 --- a/tests/test_litellm/llms/vertex_ai/test_vertex_ai_common_utils.py +++ b/tests/test_litellm/llms/vertex_ai/test_vertex_ai_common_utils.py @@ -803,6 +803,124 @@ def test_fix_enum_empty_strings(): assert input_schema["properties"]["user_agent_type"]["description"] == "Device type for user agent" +def test_fix_enum_types(): + """ + Test _fix_enum_types function removes enum fields when type is not string. + + This test verifies the fix for the issue where Gemini rejects cached content + with function parameter enums on non-string types, causing API failures. + + Relevant issue: Gemini only allows enums for string-typed fields + """ + from litellm.llms.vertex_ai.common_utils import _fix_enum_types + + # Input: Schema with enum on non-string type (the problematic case) + input_schema = { + "type": "object", + "properties": { + "truncateMode": { + "enum": ["auto", "none", "start", "end"], + "type": "string", # This should keep the enum + "description": "How to truncate content" + }, + "maxLength": { + "enum": [100, 200, 500], # This should be removed + "type": "integer", + "description": "Maximum length" + }, + "enabled": { + "enum": [True, False], # This should be removed + "type": "boolean", + "description": "Whether feature is enabled" + }, + "nested": { + "type": "object", + "properties": { + "innerEnum": { + "enum": ["a", "b", "c"], # This should be kept + "type": "string" + }, + "innerNonStringEnum": { + "enum": [1, 2, 3], # This should be removed + "type": "integer" + } + } + }, + "anyOfField": { + "anyOf": [ + {"type": "string", "enum": ["option1", "option2"]}, # This should be kept + {"type": "integer", "enum": [1, 2, 3]} # This should be removed + ] + } + } + } + + # Expected output: Non-string enums removed, string enums kept + expected_output = { + "type": "object", + "properties": { + "truncateMode": { + "enum": ["auto", "none", "start", "end"], # Kept - string type + "type": "string", + "description": "How to truncate content" + }, + "maxLength": { # enum removed + "type": "integer", + "description": "Maximum length" + }, + "enabled": { # enum removed + "type": "boolean", + "description": "Whether feature is enabled" + }, + "nested": { + "type": "object", + "properties": { + "innerEnum": { + "enum": ["a", "b", "c"], # Kept - string type + "type": "string" + }, + "innerNonStringEnum": { # enum removed + "type": "integer" + } + } + }, + "anyOfField": { + "anyOf": [ + {"type": "string", "enum": ["option1", "option2"]}, # Kept - has string type + {"type": "integer"} # enum removed + ] + } + } + } + + # Apply the transformation + _fix_enum_types(input_schema) + + # Verify the transformation + assert input_schema == expected_output + + # Verify specific transformations: + # 1. String enums are preserved + assert "enum" in input_schema["properties"]["truncateMode"] + assert input_schema["properties"]["truncateMode"]["enum"] == ["auto", "none", "start", "end"] + + assert "enum" in input_schema["properties"]["nested"]["properties"]["innerEnum"] + assert input_schema["properties"]["nested"]["properties"]["innerEnum"]["enum"] == ["a", "b", "c"] + + # 2. Non-string enums are removed + assert "enum" not in input_schema["properties"]["maxLength"] + assert "enum" not in input_schema["properties"]["enabled"] + assert "enum" not in input_schema["properties"]["nested"]["properties"]["innerNonStringEnum"] + + # 3. anyOf with string type keeps enum, non-string removes it + assert "enum" in input_schema["properties"]["anyOfField"]["anyOf"][0] + assert "enum" not in input_schema["properties"]["anyOfField"]["anyOf"][1] + + # 4. Other properties preserved + assert input_schema["properties"]["maxLength"]["type"] == "integer" + assert input_schema["properties"]["enabled"]["type"] == "boolean" + + def test_get_token_url(): from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import ( VertexLLM, diff --git a/tests/test_litellm/proxy/auth/test_auth_checks.py b/tests/test_litellm/proxy/auth/test_auth_checks.py index 057b56ce317..6cd4bec3f18 100644 --- a/tests/test_litellm/proxy/auth/test_auth_checks.py +++ b/tests/test_litellm/proxy/auth/test_auth_checks.py @@ -732,3 +732,182 @@ async def test_get_team_object_raises_404_when_not_found(): assert exc_info.value.status_code == 404 assert "Team doesn't exist in db" in str(exc_info.value.detail) + + +# Reject Client-Side Metadata Tags Tests + + +@pytest.mark.asyncio +async def test_reject_clientside_metadata_tags_enabled_with_tags(): + """Test that common_checks rejects request when reject_clientside_metadata_tags is True and metadata.tags is present""" + from litellm.proxy.auth.auth_checks import common_checks + from fastapi import Request + + request_body = { + "model": "gpt-3.5-turbo", + "messages": [{"role": "user", "content": "test"}], + "metadata": {"tags": ["custom-tag"]}, + } + + general_settings = {"reject_clientside_metadata_tags": True} + + # Create a mock request object + mock_request = MagicMock(spec=Request) + + with pytest.raises(ProxyException) as exc_info: + await common_checks( + request_body=request_body, + team_object=None, + user_object=None, + end_user_object=None, + global_proxy_spend=None, + general_settings=general_settings, + route="/chat/completions", + llm_router=None, + proxy_logging_obj=MagicMock(), + valid_token=None, + request=mock_request, + ) + + assert exc_info.value.type == ProxyErrorTypes.bad_request_error + assert "metadata.tags" in exc_info.value.message + assert exc_info.value.param == "metadata.tags" + assert exc_info.value.code == 400 + + +@pytest.mark.asyncio +async def test_reject_clientside_metadata_tags_enabled_without_tags(): + """Test that common_checks allows request when reject_clientside_metadata_tags is True but no metadata.tags is present""" + from litellm.proxy.auth.auth_checks import common_checks + from fastapi import Request + + request_body = { + "model": "gpt-3.5-turbo", + "messages": [{"role": "user", "content": "test"}], + "metadata": {"custom_field": "value"}, # No tags field + } + + general_settings = {"reject_clientside_metadata_tags": True} + + # Create a mock request object + mock_request = MagicMock(spec=Request) + + # Should not raise an exception + result = await common_checks( + request_body=request_body, + team_object=None, + user_object=None, + end_user_object=None, + global_proxy_spend=None, + general_settings=general_settings, + route="/chat/completions", + llm_router=None, + proxy_logging_obj=MagicMock(), + valid_token=None, + request=mock_request, + ) + + assert result is True + + +@pytest.mark.asyncio +async def test_reject_clientside_metadata_tags_disabled_with_tags(): + """Test that common_checks allows request with metadata.tags when reject_clientside_metadata_tags is False""" + from litellm.proxy.auth.auth_checks import common_checks + from fastapi import Request + + request_body = { + "model": "gpt-3.5-turbo", + "messages": [{"role": "user", "content": "test"}], + "metadata": {"tags": ["custom-tag"]}, + } + + general_settings = {"reject_clientside_metadata_tags": False} + + # Create a mock request object + mock_request = MagicMock(spec=Request) + + # Should not raise an exception + result = await common_checks( + request_body=request_body, + team_object=None, + user_object=None, + end_user_object=None, + global_proxy_spend=None, + general_settings=general_settings, + route="/chat/completions", + llm_router=None, + proxy_logging_obj=MagicMock(), + valid_token=None, + request=mock_request, + ) + + assert result is True + + +@pytest.mark.asyncio +async def test_reject_clientside_metadata_tags_not_set_with_tags(): + """Test that common_checks allows request with metadata.tags when reject_clientside_metadata_tags is not set""" + from litellm.proxy.auth.auth_checks import common_checks + from fastapi import Request + + request_body = { + "model": "gpt-3.5-turbo", + "messages": [{"role": "user", "content": "test"}], + "metadata": {"tags": ["custom-tag"]}, + } + + general_settings = {} # No reject_clientside_metadata_tags setting + + # Create a mock request object + mock_request = MagicMock(spec=Request) + + # Should not raise an exception + result = await common_checks( + request_body=request_body, + team_object=None, + user_object=None, + end_user_object=None, + global_proxy_spend=None, + general_settings=general_settings, + route="/chat/completions", + llm_router=None, + proxy_logging_obj=MagicMock(), + valid_token=None, + request=mock_request, + ) + + assert result is True + + +@pytest.mark.asyncio +async def test_reject_clientside_metadata_tags_non_llm_route(): + """Test that reject_clientside_metadata_tags check only applies to LLM API routes""" + from litellm.proxy.auth.auth_checks import common_checks + from fastapi import Request + + request_body = { + "metadata": {"tags": ["custom-tag"]}, + } + + general_settings = {"reject_clientside_metadata_tags": True} + + # Create a mock request object + mock_request = MagicMock(spec=Request) + + # Should not raise an exception for non-LLM route + result = await common_checks( + request_body=request_body, + team_object=None, + user_object=None, + end_user_object=None, + global_proxy_spend=None, + general_settings=general_settings, + route="/key/generate", # Management route, not LLM route + llm_router=None, + proxy_logging_obj=MagicMock(), + valid_token=None, + request=mock_request, + ) + + assert result is True diff --git a/tests/test_litellm/proxy/common_utils/test_http_parsing_utils.py b/tests/test_litellm/proxy/common_utils/test_http_parsing_utils.py index a8df4273765..85858866dda 100644 --- a/tests/test_litellm/proxy/common_utils/test_http_parsing_utils.py +++ b/tests/test_litellm/proxy/common_utils/test_http_parsing_utils.py @@ -93,6 +93,208 @@ async def test_form_data_parsing(): assert not hasattr(mock_request, "body") or not mock_request.body.called +@pytest.mark.asyncio +async def test_form_data_with_json_metadata(): + """ + Test that form data with a JSON-encoded metadata field is correctly parsed. + + When form data includes a 'metadata' field, it comes as a JSON string that needs + to be parsed into a Python dictionary (lines 42-43 of http_parsing_utils.py). + """ + # Create a mock request with form data containing JSON metadata + mock_request = MagicMock() + + # Metadata is sent as a JSON string in form data + metadata_json_string = json.dumps({ + "user_id": "12345", + "request_type": "audio_transcription", + "tags": ["urgent", "production"], + "custom_field": {"nested": "value"} + }) + + test_data = { + "model": "whisper-1", + "file": "audio.mp3", + "metadata": metadata_json_string # This is a JSON string, not a dict + } + + # Mock the form method to return the test data as an awaitable + mock_request.form = AsyncMock(return_value=test_data) + mock_request.headers = {"content-type": "multipart/form-data"} + mock_request.scope = {} + + # Parse the form data + result = await _read_request_body(mock_request) + + # Verify the metadata was parsed from JSON string to dict + assert "metadata" in result + assert isinstance(result["metadata"], dict) + assert result["metadata"]["user_id"] == "12345" + assert result["metadata"]["request_type"] == "audio_transcription" + assert result["metadata"]["tags"] == ["urgent", "production"] + assert result["metadata"]["custom_field"] == {"nested": "value"} + + # Verify other fields remain unchanged + assert result["model"] == "whisper-1" + assert result["file"] == "audio.mp3" + + # Verify form() was called + mock_request.form.assert_called_once() + + +@pytest.mark.asyncio +async def test_form_data_with_invalid_json_metadata(): + """ + Test that form data with invalid JSON in metadata field raises an exception. + + This tests error handling when the metadata field contains malformed JSON. + """ + # Create a mock request with form data containing invalid JSON metadata + mock_request = MagicMock() + + test_data = { + "model": "whisper-1", + "file": "audio.mp3", + "metadata": '{"invalid": json}' # Invalid JSON - unquoted value + } + + # Mock the form method to return the test data + mock_request.form = AsyncMock(return_value=test_data) + mock_request.headers = {"content-type": "multipart/form-data"} + mock_request.scope = {} + + # Should raise JSONDecodeError when trying to parse invalid JSON metadata + with pytest.raises(json.JSONDecodeError): + await _read_request_body(mock_request) + + +@pytest.mark.asyncio +async def test_form_data_without_metadata(): + """ + Test that form data without metadata field works correctly. + + Ensures the metadata parsing logic doesn't break when metadata is absent. + """ + # Create a mock request with form data without metadata + mock_request = MagicMock() + + test_data = { + "model": "whisper-1", + "file": "audio.mp3", + "language": "en" + } + + # Mock the form method to return the test data + mock_request.form = AsyncMock(return_value=test_data) + mock_request.headers = {"content-type": "application/x-www-form-urlencoded"} + mock_request.scope = {} + + # Parse the form data + result = await _read_request_body(mock_request) + + # Verify all fields are preserved as-is + assert result == test_data + assert "metadata" not in result + assert result["model"] == "whisper-1" + assert result["file"] == "audio.mp3" + assert result["language"] == "en" + + +@pytest.mark.asyncio +async def test_form_data_with_empty_metadata(): + """ + Test that form data with empty JSON object in metadata field is parsed correctly. + """ + # Create a mock request with form data containing empty metadata + mock_request = MagicMock() + + test_data = { + "model": "whisper-1", + "file": "audio.mp3", + "metadata": "{}" # Empty JSON object as string + } + + # Mock the form method to return the test data + mock_request.form = AsyncMock(return_value=test_data) + mock_request.headers = {"content-type": "multipart/form-data"} + mock_request.scope = {} + + # Parse the form data + result = await _read_request_body(mock_request) + + # Verify the metadata was parsed to an empty dict + assert "metadata" in result + assert isinstance(result["metadata"], dict) + assert result["metadata"] == {} + assert result["model"] == "whisper-1" + + +@pytest.mark.asyncio +async def test_form_data_with_dict_metadata(): + """ + Test that form data with metadata already as a dict is not parsed again. + + This handles edge cases where metadata might already be a dictionary + (shouldn't happen in normal form data, but defensive coding). + """ + # Create a mock request with form data where metadata is already a dict + mock_request = MagicMock() + + metadata_dict = { + "user_id": "12345", + "tags": ["test"] + } + + test_data = { + "model": "whisper-1", + "file": "audio.mp3", + "metadata": metadata_dict # Already a dict, not a string + } + + # Mock the form method to return the test data + mock_request.form = AsyncMock(return_value=test_data) + mock_request.headers = {"content-type": "multipart/form-data"} + mock_request.scope = {} + + # Parse the form data + result = await _read_request_body(mock_request) + + # Verify the metadata remains as a dict and is not parsed + assert "metadata" in result + assert isinstance(result["metadata"], dict) + assert result["metadata"] == metadata_dict + assert result["metadata"]["user_id"] == "12345" + assert result["model"] == "whisper-1" + + +@pytest.mark.asyncio +async def test_form_data_with_none_metadata(): + """ + Test that form data with None metadata value is handled gracefully. + """ + # Create a mock request with form data where metadata is None + mock_request = MagicMock() + + test_data = { + "model": "whisper-1", + "file": "audio.mp3", + "metadata": None # None value + } + + # Mock the form method to return the test data + mock_request.form = AsyncMock(return_value=test_data) + mock_request.headers = {"content-type": "multipart/form-data"} + mock_request.scope = {} + + # Parse the form data + result = await _read_request_body(mock_request) + + # Verify the metadata remains None (not parsed) + assert "metadata" in result + assert result["metadata"] is None + assert result["model"] == "whisper-1" + + @pytest.mark.asyncio async def test_empty_request_body(): """ diff --git a/tests/test_litellm/proxy/guardrails/test_pillar_guardrails.py b/tests/test_litellm/proxy/guardrails/test_pillar_guardrails.py index 9aecd01dc56..99a51d20a7d 100644 --- a/tests/test_litellm/proxy/guardrails/test_pillar_guardrails.py +++ b/tests/test_litellm/proxy/guardrails/test_pillar_guardrails.py @@ -234,6 +234,22 @@ def pillar_async_response(): ) +@pytest.fixture +def user_api_key_dict_with_context(): + """Fixture providing UserAPIKeyAuth with complete context.""" + return UserAPIKeyAuth( + token="hashed-test-token", + key_name="production-api-key", + key_alias="prod-key", + user_id="user-123", + user_email="test@example.com", + team_id="team-456", + team_alias="engineering-team", + org_id="org-789", + metadata={"environment": "production", "region": "us-east-1"}, + ) + + @pytest.fixture def mock_llm_response_with_tools(): """Fixture providing a mock LLM response with tool calls.""" @@ -502,6 +518,217 @@ async def test_pre_call_hook_custom_header_overrides( assert captured_headers.get("plr_evidence") == "false" +# ========================================================================= +# LITELLM KEY CONTEXT HEADER TESTS +# ========================================================================= + + +@pytest.mark.asyncio +async def test_litellm_context_headers_automatically_added( + sample_request_data, + user_api_key_dict_with_context, + dual_cache, + pillar_clean_response, +): + """Test that LiteLLM context headers are automatically added (always enabled).""" + guardrail = PillarGuardrail( + guardrail_name="pillar-context-enabled", + api_key="test-pillar-key", + api_base="https://api.pillar.security", + ) + + captured_headers: Dict[str, str] = {} + + async def _mock_post(*args, **kwargs): + captured_headers.update(kwargs.get("headers", {})) + return pillar_clean_response + + with patch( + "litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.post", + new=_mock_post, + ): + await guardrail.async_pre_call_hook( + data=sample_request_data, + cache=dual_cache, + user_api_key_dict=user_api_key_dict_with_context, + call_type="completion", + ) + + # Verify LiteLLM context headers are present + assert "X-LiteLLM-Key-Name" in captured_headers + assert captured_headers["X-LiteLLM-Key-Name"] == "production-api-key" + assert "X-LiteLLM-Key-Alias" in captured_headers + assert captured_headers["X-LiteLLM-Key-Alias"] == "prod-key" + assert "X-LiteLLM-User-Id" in captured_headers + assert captured_headers["X-LiteLLM-User-Id"] == "user-123" + assert "X-LiteLLM-User-Email" in captured_headers + assert captured_headers["X-LiteLLM-User-Email"] == "test@example.com" + assert "X-LiteLLM-Team-Id" in captured_headers + assert captured_headers["X-LiteLLM-Team-Id"] == "team-456" + assert "X-LiteLLM-Team-Name" in captured_headers + assert captured_headers["X-LiteLLM-Team-Name"] == "engineering-team" + assert "X-LiteLLM-Org-Id" in captured_headers + assert captured_headers["X-LiteLLM-Org-Id"] == "org-789" + + # Metadata is NOT sent (may contain sensitive information) + assert "X-LiteLLM-Metadata" not in captured_headers + + +@pytest.mark.asyncio +async def test_litellm_context_with_partial_fields( + sample_request_data, + dual_cache, + pillar_clean_response, +): + """Test that partial LiteLLM context (only some fields present) is handled correctly.""" + # Create UserAPIKeyAuth with only some fields populated + partial_context = UserAPIKeyAuth( + user_id="user-only", + team_id="team-only", + ) + + guardrail = PillarGuardrail( + guardrail_name="pillar-partial-context", + api_key="test-pillar-key", + api_base="https://api.pillar.security", + pass_litellm_key_header=True, + ) + + captured_headers: Dict[str, str] = {} + + async def _mock_post(*args, **kwargs): + captured_headers.update(kwargs.get("headers", {})) + return pillar_clean_response + + with patch( + "litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.post", + new=_mock_post, + ): + await guardrail.async_pre_call_hook( + data=sample_request_data, + cache=dual_cache, + user_api_key_dict=partial_context, + call_type="completion", + ) + + # Verify only populated fields are present + assert "X-LiteLLM-User-Id" in captured_headers + assert captured_headers["X-LiteLLM-User-Id"] == "user-only" + assert "X-LiteLLM-Team-Id" in captured_headers + assert captured_headers["X-LiteLLM-Team-Id"] == "team-only" + + # Verify empty fields are not present + assert "X-LiteLLM-Key-Name" not in captured_headers + assert "X-LiteLLM-User-Email" not in captured_headers + + +# ========================================================================= +# MULTI-MODAL CONTENT TESTS +# ========================================================================= + + +@pytest.mark.asyncio +async def test_multimodal_image_url_support( + user_api_key_dict, + dual_cache, + pillar_clean_response, +): + """Test that messages with image URLs are properly handled.""" + multimodal_data = { + "model": "gpt-4-vision-preview", + "messages": [ + { + "role": "user", + "content": [ + {"type": "text", "text": "What's in this image?"}, + { + "type": "image_url", + "image_url": { + "url": "https://example.com/image.jpg", + "detail": "high", + }, + }, + ], + } + ], + } + + guardrail = PillarGuardrail( + guardrail_name="pillar-multimodal", + api_key="test-pillar-key", + api_base="https://api.pillar.security", + ) + + captured_payload: Dict[str, Any] = {} + + async def _mock_post(*args, **kwargs): + captured_payload.update(kwargs.get("json", {})) + return pillar_clean_response + + with patch( + "litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.post", + new=_mock_post, + ): + result = await guardrail.async_pre_call_hook( + data=multimodal_data, + cache=dual_cache, + user_api_key_dict=user_api_key_dict, + call_type="completion", + ) + + # Verify multimodal message structure is preserved + assert result == multimodal_data + assert "messages" in captured_payload + assert len(captured_payload["messages"]) == 1 + assert isinstance(captured_payload["messages"][0]["content"], list) + assert captured_payload["messages"][0]["content"][1]["type"] == "image_url" + + +@pytest.mark.asyncio +async def test_multimodal_with_attachments( + user_api_key_dict, + dual_cache, + pillar_clean_response, +): + """Test that messages with file attachments are properly handled.""" + multimodal_data = { + "model": "gpt-4", + "messages": [ + { + "role": "user", + "content": "Analyze this document", + "attachments": [ + { + "file_id": "file-abc123", + "tools": [{"type": "code_interpreter"}], + } + ], + } + ], + } + + guardrail = PillarGuardrail( + guardrail_name="pillar-attachments", + api_key="test-pillar-key", + api_base="https://api.pillar.security", + ) + + with patch( + "litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.post", + return_value=pillar_clean_response, + ): + result = await guardrail.async_pre_call_hook( + data=multimodal_data, + cache=dual_cache, + user_api_key_dict=user_api_key_dict, + call_type="completion", + ) + + # Verify attachment structure is preserved + assert result == multimodal_data + assert result["messages"][0]["attachments"] is not None + + # ============================================================================ # EDGE CASE TESTS # ============================================================================ diff --git a/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py index c9b7e057904..86b23c98ba5 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py @@ -1783,6 +1783,10 @@ async def test_team_member_delete_cleans_membership(mock_db_client, mock_admin_a mock_db_client.db.litellm_teammembership = MagicMock() mock_db_client.db.litellm_teammembership.delete_many = AsyncMock(return_value=MagicMock()) + # Verification token deletion should be called + mock_db_client.db.litellm_verificationtoken = MagicMock() + mock_db_client.db.litellm_verificationtoken.delete_many = AsyncMock(return_value=MagicMock()) + # Execute await team_member_delete( data=TeamMemberDeleteRequest(team_id=test_team_id, user_id=test_user_id), @@ -1795,6 +1799,54 @@ async def test_team_member_delete_cleans_membership(mock_db_client, mock_admin_a ) +@pytest.mark.asyncio +async def test_team_member_delete_cleans_verification_tokens(mock_db_client, mock_admin_auth): + from litellm.proxy._types import TeamMemberDeleteRequest + from litellm.proxy.management_endpoints.team_endpoints import team_member_delete + + test_team_id = "team-del-tokens-123" + test_user_id = "user-tokens@example.com" + + mock_team_row = MagicMock() + mock_team_row.model_dump.return_value = { + "team_id": test_team_id, + "members_with_roles": [ + {"user_id": test_user_id, "user_email": None, "role": "user"} + ], + "team_member_permissions": [], + "metadata": {}, + "models": [], + "spend": 0.0, + } + + mock_db_client.db.litellm_teamtable.find_unique = AsyncMock(return_value=mock_team_row) + mock_db_client.db.litellm_teamtable.update = AsyncMock(return_value=mock_team_row) + + mock_user_row = MagicMock() + mock_user_row.user_id = test_user_id + mock_user_row.teams = [test_team_id] + mock_db_client.db.litellm_usertable.find_many = AsyncMock(return_value=[mock_user_row]) + mock_db_client.db.litellm_usertable.update = AsyncMock(return_value=MagicMock()) + + mock_db_client.db.litellm_teammembership = MagicMock() + mock_db_client.db.litellm_teammembership.delete_many = AsyncMock(return_value=MagicMock()) + + mock_db_client.db.litellm_verificationtoken = MagicMock() + mock_db_client.db.litellm_verificationtoken.delete_many = AsyncMock(return_value=MagicMock()) + + await team_member_delete( + data=TeamMemberDeleteRequest(team_id=test_team_id, user_id=test_user_id), + user_api_key_dict=mock_admin_auth, + ) + + mock_db_client.db.litellm_verificationtoken.delete_many.assert_awaited_once_with( + where={ + "user_id": {"in": [test_user_id]}, + "team_id": test_team_id, + } + ) + + @pytest.mark.asyncio async def test_new_team_max_budget_exceeds_user_max_budget(): """ diff --git a/tests/test_litellm/proxy/pass_through_endpoints/llm_provider_handlers/test_cohere_passthrough_logging_handler.py b/tests/test_litellm/proxy/pass_through_endpoints/llm_provider_handlers/test_cohere_passthrough_logging_handler.py new file mode 100644 index 00000000000..0b6d3fdeced --- /dev/null +++ b/tests/test_litellm/proxy/pass_through_endpoints/llm_provider_handlers/test_cohere_passthrough_logging_handler.py @@ -0,0 +1,154 @@ +import json +import os +import sys +from datetime import datetime +from unittest.mock import MagicMock, patch + +import httpx +import pytest + +sys.path.insert( + 0, os.path.abspath("../../..") +) # Adds the parent directory to the system path + +from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj +from litellm.proxy.pass_through_endpoints.llm_provider_handlers.cohere_passthrough_logging_handler import ( + CoherePassthroughLoggingHandler, +) +from litellm.types.passthrough_endpoints.pass_through_endpoints import ( + PassthroughStandardLoggingPayload, +) + + +class TestCoherePassthroughLoggingHandler: + """Test the Cohere passthrough logging handler for embed cost tracking.""" + + def setup_method(self): + """Set up test fixtures""" + self.start_time = datetime.now() + self.end_time = datetime.now() + self.handler = CoherePassthroughLoggingHandler() + + # Mock Cohere embed response + self.mock_cohere_embed_response = { + "embeddings": [ + [0.1, 0.2, 0.3, 0.4, 0.5], + [0.6, 0.7, 0.8, 0.9, 1.0], + ], + "meta": { + "billed_units": { + "input_tokens": 3, + } + }, + } + + def _create_mock_logging_obj(self) -> LiteLLMLoggingObj: + """Create a mock logging object""" + mock_logging_obj = MagicMock() + mock_logging_obj.model_call_details = {} + return mock_logging_obj + + def _create_mock_httpx_response(self, response_data: dict = None) -> httpx.Response: + """Create a mock httpx response""" + if response_data is None: + response_data = self.mock_cohere_embed_response + + mock_response = MagicMock(spec=httpx.Response) + mock_response.status_code = 200 + mock_response.text = json.dumps(response_data) + mock_response.json.return_value = response_data + mock_response.headers = {"content-type": "application/json"} + return mock_response + + def _create_passthrough_logging_payload(self) -> PassthroughStandardLoggingPayload: + """Create a mock passthrough logging payload""" + return PassthroughStandardLoggingPayload( + url="https://api.cohere.com/v1/embed", + request_body={"model": "embed-english-v3.0", "texts": ["test passthrough"]}, + request_method="POST", + ) + + @patch("litellm.completion_cost") + @patch( + "litellm.litellm_core_utils.litellm_logging.get_standard_logging_object_payload" + ) + @patch("litellm.llms.cohere.embed.v1_transformation.CohereEmbeddingConfig._transform_response") + def test_cohere_embed_passthrough_cost_tracking( + self, mock_transform_response, mock_get_standard_logging, mock_completion_cost + ): + """Test successful cost tracking for Cohere embed passthrough""" + # Arrange + from litellm.types.utils import EmbeddingResponse + + # Create a mock embedding response + mock_embedding_response = EmbeddingResponse() + mock_embedding_response.data = [ + {"object": "embedding", "index": 0, "embedding": [0.1, 0.2, 0.3]}, + {"object": "embedding", "index": 1, "embedding": [0.4, 0.5, 0.6]}, + ] + mock_embedding_response.model = "embed-english-v3.0" + mock_embedding_response.object = "list" + from litellm.types.utils import Usage + mock_embedding_response.usage = Usage( + prompt_tokens=3, completion_tokens=0, total_tokens=3 + ) + + mock_transform_response.return_value = mock_embedding_response + mock_completion_cost.return_value = 3.6e-07 # Expected cost for embed-v4.0 + mock_get_standard_logging.return_value = {"test": "logging_payload"} + + mock_httpx_response = self._create_mock_httpx_response() + mock_logging_obj = self._create_mock_logging_obj() + passthrough_payload = self._create_passthrough_logging_payload() + + kwargs = { + "passthrough_logging_payload": passthrough_payload, + } + + request_body = { + "model": "embed-english-v3.0", + "texts": ["test passthrough"], + } + + # Act + result = self.handler.cohere_passthrough_handler( + httpx_response=mock_httpx_response, + response_body=self.mock_cohere_embed_response, + logging_obj=mock_logging_obj, + url_route="https://api.cohere.com/v1/embed", + result="", + start_time=self.start_time, + end_time=self.end_time, + cache_hit=False, + request_body=request_body, + **kwargs, + ) + + # Assert + assert result is not None + assert "result" in result + assert "kwargs" in result + assert result["kwargs"]["model"] == "embed-english-v3.0" + assert result["kwargs"]["custom_llm_provider"] == "cohere" + + # Verify cost calculation was called with correct parameters + mock_completion_cost.assert_called_once() + call_args = mock_completion_cost.call_args + assert call_args.kwargs["model"] == "embed-english-v3.0" + assert call_args.kwargs["custom_llm_provider"] == "cohere" + assert call_args.kwargs["call_type"] == "aembedding" + + # Verify logging object was updated + assert mock_logging_obj.model_call_details["response_cost"] == 3.6e-07 + assert mock_logging_obj.model_call_details["model"] == "embed-english-v3.0" + assert mock_logging_obj.model_call_details["custom_llm_provider"] == "cohere" + + # Verify result is an EmbeddingResponse + assert hasattr(result["result"], "data") + assert hasattr(result["result"], "model") + assert result["result"].model == "embed-english-v3.0" + + +if __name__ == "__main__": + pytest.main([__file__]) + diff --git a/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py b/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py index ea1017e1d5a..b0e198d5e7e 100644 --- a/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py +++ b/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py @@ -1179,6 +1179,161 @@ class TestBedrockLLMProxyRoute: in str(exc_info.value.detail) ) + @pytest.mark.asyncio + async def test_bedrock_passthrough_uses_model_specific_credentials(self): + """ + Test that Bedrock passthrough endpoints use credentials from model configuration + instead of environment variables when a router model is used. + + This test verifies the fix for the bug where passthrough endpoints were using + environment variables instead of model-specific credentials from config.yaml. + """ + from litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints import ( + handle_bedrock_passthrough_router_model, + ) + from litellm import Router + from litellm.litellm_core_utils.get_litellm_params import get_litellm_params + + # Model-specific credentials (different from env vars) + model_access_key = "MODEL_SPECIFIC_ACCESS_KEY" + model_secret_key = "MODEL_SPECIFIC_SECRET_KEY" + model_region = "us-west-2" + model_session_token = "MODEL_SESSION_TOKEN" + + # Environment variables (should NOT be used) + env_access_key = "ENV_ACCESS_KEY" + env_secret_key = "ENV_SECRET_KEY" + env_region = "us-east-1" + + # Set environment variables to different values + with patch.dict( + os.environ, + { + "AWS_ACCESS_KEY_ID": env_access_key, + "AWS_SECRET_ACCESS_KEY": env_secret_key, + "AWS_REGION_NAME": env_region, + }, + ): + # Test 1: Verify get_litellm_params extracts AWS credentials from kwargs + kwargs_with_creds = { + "aws_access_key_id": model_access_key, + "aws_secret_access_key": model_secret_key, + "aws_region_name": model_region, + "aws_session_token": model_session_token, + "model": "bedrock/test-model", + } + litellm_params = get_litellm_params(**kwargs_with_creds) + + # Verify credentials are extracted + assert litellm_params.get("aws_access_key_id") == model_access_key + assert litellm_params.get("aws_secret_access_key") == model_secret_key + assert litellm_params.get("aws_region_name") == model_region + assert litellm_params.get("aws_session_token") == model_session_token + + # Test 2: Verify router passes model credentials to passthrough + router = Router( + model_list=[ + { + "model_name": "claude-opus-4-1", + "litellm_params": { + "model": "bedrock/us.anthropic.claude-opus-4-20250514-v1:0", + "aws_access_key_id": model_access_key, + "aws_secret_access_key": model_secret_key, + "aws_region_name": model_region, + "aws_session_token": model_session_token, + "custom_llm_provider": "bedrock", + }, + } + ] + ) + + # Verify router has model-specific credentials + deployments = router.get_model_list(model_name="claude-opus-4-1") + assert len(deployments) > 0 + deployment = deployments[0] + deployment_litellm_params = deployment.get("litellm_params", {}) + + # Verify model-specific credentials are in the deployment + assert deployment_litellm_params.get("aws_access_key_id") == model_access_key + assert deployment_litellm_params.get("aws_secret_access_key") == model_secret_key + assert deployment_litellm_params.get("aws_region_name") == model_region + assert deployment_litellm_params.get("aws_session_token") == model_session_token + + # Verify environment variables are NOT in the deployment + assert deployment_litellm_params.get("aws_access_key_id") != env_access_key + assert deployment_litellm_params.get("aws_secret_access_key") != env_secret_key + assert deployment_litellm_params.get("aws_region_name") != env_region + + # Test 3: Verify credentials are passed through the passthrough route + # Mock the passthrough route to capture what credentials are used + captured_kwargs = {} + + async def mock_llm_passthrough_route(**kwargs): + captured_kwargs.update(kwargs) + mock_response = MagicMock() + mock_response.status_code = 200 + mock_response.aread = AsyncMock( + return_value=b'{"content": [{"text": "Hello"}]}' + ) + return mock_response + + mock_request = MagicMock(spec=Request) + mock_request.method = "POST" + mock_request.headers = {"content-type": "application/json"} + mock_request.query_params = {} + mock_request.url = MagicMock() + mock_request.url.path = "/bedrock/model/claude-opus-4-1/converse" + + mock_request_body = { + "messages": [{"role": "user", "content": [{"text": "Hello"}]}] + } + + mock_user_api_key_dict = Mock() + mock_user_api_key_dict.api_key = "test-key" + mock_proxy_logging_obj = Mock() + mock_proxy_logging_obj.post_call_failure_hook = AsyncMock() + + with patch( + "litellm.passthrough.main.llm_passthrough_route", + new_callable=AsyncMock, + side_effect=mock_llm_passthrough_route, + ), patch( + "litellm.proxy.common_request_processing.ProxyBaseLLMRequestProcessing.base_passthrough_process_llm_request", + new_callable=AsyncMock, + ) as mock_process: + # Setup mock response + mock_response = MagicMock() + mock_response.status_code = 200 + mock_response.aread = AsyncMock( + return_value=b'{"content": [{"text": "Hello"}]}' + ) + mock_process.return_value = mock_response + + # Call the handler + await handle_bedrock_passthrough_router_model( + model="claude-opus-4-1", + endpoint="model/claude-opus-4-1/converse", + request=mock_request, + request_body=mock_request_body, + llm_router=router, + user_api_key_dict=mock_user_api_key_dict, + proxy_logging_obj=mock_proxy_logging_obj, + general_settings={}, + proxy_config=None, + select_data_generator=None, + user_model=None, + user_temperature=None, + user_request_timeout=None, + user_max_tokens=None, + user_api_base=None, + version=None, + ) + + # Verify that the router was called (which means credentials flow through) + # The key verification is that get_litellm_params extracts the credentials + # and they're available in the router's deployment + assert mock_process.called + class TestLLMPassthroughFactoryProxyRoute: @pytest.mark.asyncio diff --git a/tests/test_litellm/proxy/test_model_id_header_propagation.py b/tests/test_litellm/proxy/test_model_id_header_propagation.py new file mode 100644 index 00000000000..cc4e7c084d6 --- /dev/null +++ b/tests/test_litellm/proxy/test_model_id_header_propagation.py @@ -0,0 +1,250 @@ +""" +Test that x-litellm-model-id header is propagated correctly on error responses. + +This test suite verifies the `maybe_get_model_id` method +which is responsible for extracting model_id from different locations +depending on the request lifecycle stage. +""" + +import pytest +from unittest.mock import MagicMock + +from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing +from litellm.proxy._types import UserAPIKeyAuth + + +def test_maybe_get_model_id_from_litellm_params(): + """ + Test extraction of model_id from logging_obj.litellm_params (used by /v1/chat/completions). + """ + # Create a ProxyBaseLLMRequestProcessing instance + processor = ProxyBaseLLMRequestProcessing(data={}) + + # Create a mock logging object with model_info in litellm_params + mock_logging_obj = MagicMock() + mock_logging_obj.litellm_params = { + "model_info": { + "id": "test-model-id-from-litellm-params" + } + } + + # Test extraction + model_id = processor.maybe_get_model_id(mock_logging_obj) + + assert model_id == "test-model-id-from-litellm-params" + + +def test_maybe_get_model_id_from_litellm_params_nested(): + """ + Test extraction of model_id from nested metadata in logging_obj.litellm_params. + """ + processor = ProxyBaseLLMRequestProcessing(data={}) + + # Create a mock logging object with model_info nested in metadata + mock_logging_obj = MagicMock() + mock_logging_obj.litellm_params = { + "metadata": { + "model_info": { + "id": "test-model-id-nested" + } + } + } + + # Test extraction + model_id = processor.maybe_get_model_id(mock_logging_obj) + + assert model_id == "test-model-id-nested" + + +def test_maybe_get_model_id_from_kwargs(): + """ + Test extraction of model_id from logging_obj.kwargs (fallback path). + """ + processor = ProxyBaseLLMRequestProcessing(data={}) + + # Create a mock logging object with model_info in kwargs + mock_logging_obj = MagicMock() + mock_logging_obj.litellm_params = None + mock_logging_obj.kwargs = { + "litellm_params": { + "model_info": { + "id": "test-model-id-from-kwargs" + } + } + } + + # Test extraction + model_id = processor.maybe_get_model_id(mock_logging_obj) + + assert model_id == "test-model-id-from-kwargs" + + +def test_maybe_get_model_id_from_data(): + """ + Test extraction of model_id from self.data (used by /v1/messages and /v1/responses). + """ + # Create a processor with model_info in data + processor = ProxyBaseLLMRequestProcessing(data={ + "litellm_metadata": { + "model_info": { + "id": "test-model-id-from-data" + } + } + }) + + # Create a mock logging object without model_info + mock_logging_obj = MagicMock() + mock_logging_obj.litellm_params = {} + mock_logging_obj.kwargs = {} + + # Test extraction - should fall back to self.data + model_id = processor.maybe_get_model_id(mock_logging_obj) + + assert model_id == "test-model-id-from-data" + + +def test_maybe_get_model_id_no_logging_obj(): + """ + Test extraction of model_id when logging_obj is None (should use self.data). + """ + # Create a processor with model_info in data + processor = ProxyBaseLLMRequestProcessing(data={ + "litellm_metadata": { + "model_info": { + "id": "test-model-id-no-logging-obj" + } + } + }) + + # Test extraction with None logging_obj + model_id = processor.maybe_get_model_id(None) + + assert model_id == "test-model-id-no-logging-obj" + + +def test_maybe_get_model_id_not_found(): + """ + Test extraction of model_id when it's not available anywhere (should return None). + """ + processor = ProxyBaseLLMRequestProcessing(data={}) + + # Create a mock logging object without model_info anywhere + mock_logging_obj = MagicMock() + mock_logging_obj.litellm_params = {} + mock_logging_obj.kwargs = {} + + # Test extraction - should return None + model_id = processor.maybe_get_model_id(mock_logging_obj) + + assert model_id is None + + +def test_maybe_get_model_id_priority_litellm_params_over_data(): + """ + Test that model_id from logging_obj.litellm_params takes priority over self.data. + """ + # Create a processor with model_info in both places + processor = ProxyBaseLLMRequestProcessing(data={ + "litellm_metadata": { + "model_info": { + "id": "model-id-from-data" + } + } + }) + + # Create a mock logging object with model_info + mock_logging_obj = MagicMock() + mock_logging_obj.litellm_params = { + "model_info": { + "id": "model-id-from-litellm-params" + } + } + + # Test extraction - should prefer litellm_params + model_id = processor.maybe_get_model_id(mock_logging_obj) + + assert model_id == "model-id-from-litellm-params" + + +def test_get_custom_headers_includes_model_id(): + """ + Test that get_custom_headers includes x-litellm-model-id when model_id is provided. + """ + # Create mock user_api_key_dict with all required attributes + mock_user_api_key_dict = MagicMock() + mock_user_api_key_dict.user_id = "test-user" + mock_user_api_key_dict.team_id = "test-team" + mock_user_api_key_dict.tpm_limit = 1000 + mock_user_api_key_dict.rpm_limit = 100 + + # Call get_custom_headers with a model_id + headers = ProxyBaseLLMRequestProcessing.get_custom_headers( + user_api_key_dict=mock_user_api_key_dict, + model_id="test-model-123", + cache_key="test-cache-key", + api_base="https://api.example.com", + version="1.0.0", + response_cost=0.001, + request_data={}, + hidden_params={} + ) + + # Verify model_id is in headers + assert "x-litellm-model-id" in headers + assert headers["x-litellm-model-id"] == "test-model-123" + + +def test_get_custom_headers_without_model_id(): + """ + Test that get_custom_headers works correctly when model_id is None or empty. + """ + # Create mock user_api_key_dict with all required attributes + mock_user_api_key_dict = MagicMock() + mock_user_api_key_dict.user_id = "test-user" + mock_user_api_key_dict.team_id = "test-team" + mock_user_api_key_dict.tpm_limit = 1000 + mock_user_api_key_dict.rpm_limit = 100 + + # Call get_custom_headers without a model_id + headers = ProxyBaseLLMRequestProcessing.get_custom_headers( + user_api_key_dict=mock_user_api_key_dict, + model_id=None, + cache_key="test-cache-key", + api_base="https://api.example.com", + version="1.0.0", + response_cost=0.001, + request_data={}, + hidden_params={} + ) + + # x-litellm-model-id should not be in headers (or should be empty/None) + if "x-litellm-model-id" in headers: + assert headers["x-litellm-model-id"] in [None, ""] + + +def test_get_custom_headers_with_empty_string_model_id(): + """ + Test that get_custom_headers handles empty string model_id correctly. + """ + # Create mock user_api_key_dict with all required attributes + mock_user_api_key_dict = MagicMock() + mock_user_api_key_dict.user_id = "test-user" + mock_user_api_key_dict.team_id = "test-team" + mock_user_api_key_dict.tpm_limit = 1000 + mock_user_api_key_dict.rpm_limit = 100 + + # Call get_custom_headers with empty string model_id + headers = ProxyBaseLLMRequestProcessing.get_custom_headers( + user_api_key_dict=mock_user_api_key_dict, + model_id="", + cache_key="test-cache-key", + api_base="https://api.example.com", + version="1.0.0", + response_cost=0.001, + request_data={}, + hidden_params={} + ) + + # x-litellm-model-id should not be in headers (or should be empty) + if "x-litellm-model-id" in headers: + assert headers["x-litellm-model-id"] == "" diff --git a/tests/test_litellm/test_router.py b/tests/test_litellm/test_router.py index 8851264db07..032616849bd 100644 --- a/tests/test_litellm/test_router.py +++ b/tests/test_litellm/test_router.py @@ -1692,3 +1692,35 @@ async def test_router_acompletion_with_unknown_model_and_no_fallback(): # Check that the error message is correct. # The router returns 'no healthy deployments' because get_model_list returns [] not None. assert "no healthy deployments for this model" in str(excinfo.value) + + +def test_get_deployment_credentials_with_provider_aws_bedrock_runtime_endpoint(): + """ + Test that get_deployment_credentials_with_provider correctly copies + aws_bedrock_runtime_endpoint from deployment litellm_params to credentials. + """ + router = litellm.Router( + model_list=[ + { + "model_name": "bedrock-claude-model", + "litellm_params": { + "model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", + "aws_access_key_id": "test-access-key", + "aws_secret_access_key": "test-secret-key", + "aws_region_name": "us-east-1", + "aws_bedrock_runtime_endpoint": "https://bedrock-runtime.us-east-1.amazonaws.com", + }, + } + ], + ) + + credentials = router.get_deployment_credentials_with_provider( + model_id="bedrock-claude-model" + ) + + assert credentials is not None + assert credentials["aws_bedrock_runtime_endpoint"] == "https://bedrock-runtime.us-east-1.amazonaws.com" + assert credentials["aws_access_key_id"] == "test-access-key" + assert credentials["aws_secret_access_key"] == "test-secret-key" + assert credentials["aws_region_name"] == "us-east-1" + assert credentials["custom_llm_provider"] == "bedrock" diff --git a/tests/vector_store_tests/rag/base_rag_tests.py b/tests/vector_store_tests/rag/base_rag_tests.py new file mode 100644 index 00000000000..55b23e4e897 --- /dev/null +++ b/tests/vector_store_tests/rag/base_rag_tests.py @@ -0,0 +1,173 @@ +""" +Base RAG test class that enforces common tests across all providers. + +Providers should inherit from BaseRAGTest and implement the abstract methods. +""" + +import os +import sys +import uuid +from abc import ABC, abstractmethod +from typing import Any, Dict, Optional + +import pytest + +sys.path.insert(0, os.path.abspath("../../..")) + +import litellm +from litellm.types.rag import ( + RAGIngestOptions, + OpenAIVectorStoreOptions, + BedrockVectorStoreOptions, +) + + +class BaseRAGTest(ABC): + """ + Abstract base test class for RAG ingestion tests. + + Providers should inherit from this class and implement: + - get_base_ingest_options(): Returns provider-specific ingest options + - query_vector_store(): Queries the vector store after ingestion + """ + + @abstractmethod + def get_base_ingest_options(self) -> RAGIngestOptions: + """ + Must return the base ingest options for the provider. + + Example for OpenAI: + return { + "vector_store": OpenAIVectorStoreOptions( + custom_llm_provider="openai", + ) + } + + Example for Bedrock: + return { + "vector_store": BedrockVectorStoreOptions( + custom_llm_provider="bedrock", + ) + } + """ + pass + + @abstractmethod + async def query_vector_store( + self, + vector_store_id: str, + query: str, + ) -> Optional[Dict[str, Any]]: + """ + Query the vector store to verify ingestion. + + Args: + vector_store_id: The ID of the vector store to query + query: The search query + + Returns: + Search results dict or None if no results found + """ + pass + + def get_unique_filename(self, prefix: str = "test") -> str: + """Generate a unique filename for test documents.""" + unique_id = uuid.uuid4().hex[:8] + return f"{prefix}_{unique_id}.txt", unique_id + + @pytest.mark.asyncio + async def test_basic_ingest(self): + """ + Test basic text file ingestion to vector store. + """ + litellm._turn_on_debug() + + filename, unique_id = self.get_unique_filename("basic_ingest") + text_content = f"Test document {unique_id} for RAG ingestion.".encode("utf-8") + file_data = (filename, text_content, "text/plain") + + ingest_options = self.get_base_ingest_options() + ingest_options["name"] = f"test-basic-ingest-{unique_id}" + + try: + response = await litellm.rag.aingest( + ingest_options=ingest_options, + file_data=file_data, + ) + + print(f"RAG Ingest Response: {response}") + + assert "id" in response + assert response["id"].startswith("ingest_") + assert "status" in response + assert response["status"] in ["completed", "failed"] + assert "vector_store_id" in response + + if response["status"] == "completed": + assert response["vector_store_id"] + print(f"Vector store ID: {response['vector_store_id']}") + + except litellm.InternalServerError: + pytest.skip("Skipping test due to litellm.InternalServerError") + + @pytest.mark.asyncio + async def test_ingest_and_query(self): + """ + Test full RAG flow: ingest a document and then query it. + """ + import asyncio + + litellm._turn_on_debug() + + filename, unique_id = self.get_unique_filename("ingest_query") + text_content = f""" + Test document {unique_id} for RAG ingestion and query. + LiteLLM provides a unified interface for 100+ LLMs. + This content should be retrievable via semantic search. + """.encode("utf-8") + file_data = (filename, text_content, "text/plain") + + ingest_options = self.get_base_ingest_options() + ingest_options["name"] = f"test-ingest-query-{unique_id}" + + try: + # Step 1: Ingest + ingest_response = await litellm.rag.aingest( + ingest_options=ingest_options, + file_data=file_data, + ) + + print(f"Ingest Response: {ingest_response}") + assert ingest_response["status"] == "completed" + vector_store_id = ingest_response["vector_store_id"] + assert vector_store_id + + # Step 2: Query with retry (indexing may take time) + search_results = None + max_retries = 10 + for attempt in range(max_retries): + await asyncio.sleep(3) + + search_results = await self.query_vector_store( + vector_store_id=vector_store_id, + query=f"Test document {unique_id}", + ) + + if search_results: + break + + print( + f"Attempt {attempt + 1}/{max_retries}: " + "Waiting for document to be indexed..." + ) + + print(f"Search Results: {search_results}") + + # Validate search results + assert search_results is not None, "Document not found after retries" + + print("Query successful!") + + except litellm.InternalServerError: + pytest.skip("Skipping test due to litellm.InternalServerError") + diff --git a/tests/vector_store_tests/rag/test_document.txt b/tests/vector_store_tests/rag/test_document.txt new file mode 100644 index 00000000000..3570e715e83 --- /dev/null +++ b/tests/vector_store_tests/rag/test_document.txt @@ -0,0 +1,4 @@ +Test document abc123 for RAG ingestion. +This is a sample document to test the RAG ingest API. +LiteLLM provides a unified interface for vector stores. + diff --git a/tests/vector_store_tests/rag/test_rag_bedrock.py b/tests/vector_store_tests/rag/test_rag_bedrock.py new file mode 100644 index 00000000000..c991052d325 --- /dev/null +++ b/tests/vector_store_tests/rag/test_rag_bedrock.py @@ -0,0 +1,91 @@ +""" +Bedrock Knowledge Base RAG ingestion tests. + +Requires environment variables: +- AWS_ACCESS_KEY_ID +- AWS_SECRET_ACCESS_KEY +- AWS_REGION_NAME (optional, defaults to us-west-2) + +Optional (for using existing KB instead of auto-creating): +- BEDROCK_KNOWLEDGE_BASE_ID +""" + +import os +import sys +from typing import Any, Dict, Optional + +import pytest + +sys.path.insert(0, os.path.abspath("../../..")) + +import litellm +from litellm.types.rag import RAGIngestOptions, BedrockVectorStoreOptions +from tests.vector_store_tests.rag.base_rag_tests import BaseRAGTest + + +class TestRAGBedrock(BaseRAGTest): + """Test RAG Ingest with Bedrock Knowledge Base.""" + + @pytest.fixture(autouse=True) + def check_env_vars(self): + """Check required environment variables before each test.""" + aws_key = os.environ.get("AWS_ACCESS_KEY_ID") + aws_secret = os.environ.get("AWS_SECRET_ACCESS_KEY") + + if not aws_key or not aws_secret: + pytest.skip("Skipping Bedrock test: AWS credentials required") + + def get_base_ingest_options(self) -> RAGIngestOptions: + """ + Return Bedrock-specific ingest options. + + Uses unified interface - no vector_store_id means auto-create KB. + If BEDROCK_KNOWLEDGE_BASE_ID is set, uses existing KB. + """ + # Use existing KB if provided, otherwise auto-create + existing_kb_id = os.environ.get("BEDROCK_KNOWLEDGE_BASE_ID") + + return { + "vector_store": BedrockVectorStoreOptions( + custom_llm_provider="bedrock", + vector_store_id=existing_kb_id, # None = auto-create + # wait_for_ingestion defaults to False - returns immediately + ), + } + + async def query_vector_store( + self, + vector_store_id: str, + query: str, + ) -> Optional[Dict[str, Any]]: + """Query Bedrock Knowledge Base.""" + try: + import boto3 + except ImportError: + pytest.skip("boto3 required for Bedrock tests") + + session = boto3.Session( + aws_access_key_id=os.environ.get("AWS_ACCESS_KEY_ID"), + aws_secret_access_key=os.environ.get("AWS_SECRET_ACCESS_KEY"), + region_name=os.environ.get("AWS_REGION_NAME", "us-west-2"), + ) + bedrock_agent_runtime = session.client("bedrock-agent-runtime") + + response = bedrock_agent_runtime.retrieve( + knowledgeBaseId=vector_store_id, + retrievalQuery={"text": query}, + retrievalConfiguration={ + "vectorSearchConfiguration": {"numberOfResults": 5} + }, + ) + + if response.get("retrievalResults") and len(response["retrievalResults"]) > 0: + # Check if query terms appear in results + for result in response["retrievalResults"]: + # Extract unique_id from query if present + if query in result["content"]["text"]: + return response + # Return results even if exact match not found + return response + return None + diff --git a/tests/vector_store_tests/rag/test_rag_openai.py b/tests/vector_store_tests/rag/test_rag_openai.py new file mode 100644 index 00000000000..d077ebe0cb6 --- /dev/null +++ b/tests/vector_store_tests/rag/test_rag_openai.py @@ -0,0 +1,45 @@ +""" +OpenAI RAG ingestion tests. +""" + +import os +import sys +from typing import Any, Dict, Optional + +import pytest + +sys.path.insert(0, os.path.abspath("../../..")) + +import litellm +from litellm.types.rag import RAGIngestOptions, OpenAIVectorStoreOptions +from tests.vector_store_tests.rag.base_rag_tests import BaseRAGTest + + +class TestRAGOpenAI(BaseRAGTest): + """Test RAG Ingest with OpenAI provider.""" + + def get_base_ingest_options(self) -> RAGIngestOptions: + """Return OpenAI-specific ingest options.""" + return { + "vector_store": OpenAIVectorStoreOptions( + custom_llm_provider="openai", + ), + } + + async def query_vector_store( + self, + vector_store_id: str, + query: str, + ) -> Optional[Dict[str, Any]]: + """Query OpenAI vector store.""" + search_response = await litellm.vector_stores.asearch( + vector_store_id=vector_store_id, + query=query, + custom_llm_provider="openai", + ) + + if search_response.get("data") and len(search_response["data"]) > 0: + return search_response + return None + + \ No newline at end of file diff --git a/tests/vector_store_tests/test_gemini_vector_store.py b/tests/vector_store_tests/test_gemini_vector_store.py new file mode 100644 index 00000000000..92512a3a3cb --- /dev/null +++ b/tests/vector_store_tests/test_gemini_vector_store.py @@ -0,0 +1,29 @@ +""" +Minimal Gemini File Search vector store tests. +""" + +import os +import sys + +sys.path.insert(0, os.path.abspath("../..")) + +from base_vector_store_test import BaseVectorStoreTest + + +class TestGeminiVectorStore(BaseVectorStoreTest): + """Reuses the shared vector store smoke suite with Gemini.""" + + def get_base_request_args(self) -> dict: + """Provide arguments for the shared search test.""" + return { + "vector_store_id": os.getenv("GEMINI_TEST_STORE_ID", "fileSearchStores/example-test-store"), + "custom_llm_provider": "gemini", + "query": "LiteLLM", + } + + def get_base_create_vector_store_args(self) -> dict: + """Ensure we always call Gemini when creating a vector store.""" + return { + "custom_llm_provider": "gemini", + } + diff --git a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/AllModelsTab.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/AllModelsTab.test.tsx index 114bcf0d671..a4bb20128e0 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/AllModelsTab.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/AllModelsTab.test.tsx @@ -1,26 +1,9 @@ import * as useAuthorizedModule from "@/app/(dashboard)/hooks/useAuthorized"; import * as useTeamsModule from "@/app/(dashboard)/hooks/useTeams"; import { render, screen, waitFor } from "@testing-library/react"; -import { beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; +import { beforeEach, describe, expect, it, vi } from "vitest"; import AllModelsTab from "./AllModelsTab"; -// Mock window.matchMedia for Ant Design components -beforeAll(() => { - Object.defineProperty(window, "matchMedia", { - writable: true, - value: (query: string) => ({ - matches: false, - media: query, - onchange: null, - addListener: () => {}, - removeListener: () => {}, - addEventListener: () => {}, - removeEventListener: () => {}, - dispatchEvent: () => false, - }), - }); -}); - describe("AllModelsTab", () => { const mockSetSelectedModelGroup = vi.fn(); const mockSetSelectedModelId = vi.fn(); @@ -51,24 +34,18 @@ describe("AllModelsTab", () => { showSSOBanner: false, }; - beforeAll(() => { - // Mock useAuthorized hook + beforeEach(() => { + vi.clearAllMocks(); vi.spyOn(useAuthorizedModule, "default").mockReturnValue(mockUseAuthorized); }); - beforeEach(() => { - vi.clearAllMocks(); - }); - it("should render with empty data", () => { - // Mock useTeams hook vi.spyOn(useTeamsModule, "default").mockReturnValue({ teams: [], setTeams: vi.fn(), }); - const { container } = render(); - expect(container).toBeTruthy(); + render(); expect(screen.getByText("Current Team:")).toBeInTheDocument(); }); @@ -89,7 +66,6 @@ describe("AllModelsTab", () => { }, ]; - // Mock useTeams hook with team data vi.spyOn(useTeamsModule, "default").mockReturnValue({ teams: mockTeams, setTeams: vi.fn(), @@ -101,7 +77,7 @@ describe("AllModelsTab", () => { model_name: "gpt-4-accessible", model_info: { id: "model-1", - access_via_team_ids: ["team-456"], // Direct team access + access_via_team_ids: ["team-456"], access_groups: [], }, }, @@ -109,7 +85,7 @@ describe("AllModelsTab", () => { model_name: "gpt-3.5-turbo-blocked", model_info: { id: "model-2", - access_via_team_ids: ["team-789"], // Different team + access_via_team_ids: ["team-789"], access_groups: [], }, }, @@ -118,7 +94,6 @@ describe("AllModelsTab", () => { render(); - // Initially on "personal" team, should show 0 results (no models have direct_access) await waitFor(() => { expect(screen.getByText("Showing 0 results")).toBeInTheDocument(); }); @@ -129,7 +104,7 @@ describe("AllModelsTab", () => { { team_id: "team-sales", team_alias: "Sales Team", - models: ["sales-model-group"], // Team has this model group + models: ["sales-model-group"], max_budget: null, budget_duration: null, tpm_limit: null, @@ -141,7 +116,6 @@ describe("AllModelsTab", () => { }, ]; - // Mock useTeams hook vi.spyOn(useTeamsModule, "default").mockReturnValue({ teams: mockTeams, setTeams: vi.fn(), @@ -153,8 +127,8 @@ describe("AllModelsTab", () => { model_name: "gpt-4-sales", model_info: { id: "model-sales-1", - access_via_team_ids: [], // No direct team access - access_groups: ["sales-model-group"], // But has access group that matches team's models + access_via_team_ids: [], + access_groups: ["sales-model-group"], }, }, { @@ -162,7 +136,7 @@ describe("AllModelsTab", () => { model_info: { id: "model-eng-1", access_via_team_ids: [], - access_groups: ["engineering-model-group"], // Different access group + access_groups: ["engineering-model-group"], }, }, ], @@ -170,14 +144,12 @@ describe("AllModelsTab", () => { render(); - // Initially on "personal" team, should show 0 results await waitFor(() => { expect(screen.getByText("Showing 0 results")).toBeInTheDocument(); }); }); it("should filter models by direct_access for personal team", async () => { - // Mock useTeams hook vi.spyOn(useTeamsModule, "default").mockReturnValue({ teams: [], setTeams: vi.fn(), @@ -189,7 +161,7 @@ describe("AllModelsTab", () => { model_name: "gpt-4-personal", model_info: { id: "model-personal-1", - direct_access: true, // Available for personal use + direct_access: true, access_via_team_ids: [], access_groups: [], }, @@ -198,7 +170,7 @@ describe("AllModelsTab", () => { model_name: "gpt-4-team-only", model_info: { id: "model-team-1", - direct_access: false, // Not available for personal use + direct_access: false, access_via_team_ids: ["team-123"], access_groups: [], }, @@ -208,16 +180,12 @@ describe("AllModelsTab", () => { render(); - // When currentTeam is "personal" (default), it should filter by direct_access === true - // This tests the personal access logic in lines 72-73 - // Should show 1 result (only gpt-4-personal with direct_access=true) await waitFor(() => { expect(screen.getByText("Showing 1 - 1 of 1 results")).toBeInTheDocument(); }); }); - it("should show disabled delete icon for config models", async () => { - // Mock useTeams hook + it("should show config model status for models defined in configs", async () => { vi.spyOn(useTeamsModule, "default").mockReturnValue({ teams: [], setTeams: vi.fn(), @@ -231,7 +199,7 @@ describe("AllModelsTab", () => { provider: "openai", model_info: { id: "model-config-1", - db_model: false, // Config model (no db_model) + db_model: false, direct_access: true, access_via_team_ids: [], access_groups: [], @@ -246,7 +214,7 @@ describe("AllModelsTab", () => { provider: "openai", model_info: { id: "model-db-1", - db_model: true, // DB model + db_model: true, direct_access: true, access_via_team_ids: [], access_groups: [], @@ -258,19 +226,42 @@ describe("AllModelsTab", () => { ], }; - const { container } = render(); + render(); await waitFor(() => { - expect(screen.getByText(/Showing \d+ - \d+ of 2 results/)).toBeInTheDocument(); + expect(screen.getByText("Config Model")).toBeInTheDocument(); + expect(screen.getByText("DB Model")).toBeInTheDocument(); + }); + }); + + it("should show 'Defined in config' for models defined in configs", async () => { + vi.spyOn(useTeamsModule, "default").mockReturnValue({ + teams: [], + setTeams: vi.fn(), }); - const disabledIcons = container.querySelectorAll(".opacity-50.cursor-not-allowed"); - expect(disabledIcons.length).toBeGreaterThan(0); + const modelData = { + data: [ + { + model_name: "gpt-4-config-model", + litellm_model_name: "gpt-4-config-model", + provider: "openai", + model_info: { + id: "model-config-defined", + db_model: false, + direct_access: true, + access_via_team_ids: [], + access_groups: [], + created_by: "user-123", + created_at: "2024-01-01", + updated_at: "2024-01-01", + }, + }, + ], + }; - const configModelIcon = Array.from(disabledIcons).find((icon) => { - const parent = icon.closest('[class*="actions"], [class*="flex items-center justify-end"]'); - return parent !== null; - }); - expect(configModelIcon).toBeTruthy(); + render(); + + expect(screen.getByText("Defined in config")).toBeInTheDocument(); }); }); diff --git a/ui/litellm-dashboard/src/components/OldTeams.test.tsx b/ui/litellm-dashboard/src/components/OldTeams.test.tsx index 261178191f8..7f4ec3b09ca 100644 --- a/ui/litellm-dashboard/src/components/OldTeams.test.tsx +++ b/ui/litellm-dashboard/src/components/OldTeams.test.tsx @@ -1,5 +1,6 @@ -import { act, fireEvent, render, screen } from "@testing-library/react"; +import { act, fireEvent, render, screen, waitFor } from "@testing-library/react"; import { beforeEach, describe, expect, it, vi } from "vitest"; +import { fetchAvailableModelsForTeamOrKey } from "./key_team_helpers/fetch_available_models_team_key"; import { teamCreateCall } from "./networking"; import OldTeams from "./OldTeams"; @@ -23,6 +24,28 @@ vi.mock("./molecules/notifications_manager", () => ({ }, })); +vi.mock("./key_team_helpers/fetch_available_models_team_key", () => ({ + fetchAvailableModelsForTeamOrKey: vi.fn(), + getModelDisplayName: vi.fn((model: string) => model), + unfurlWildcardModelsInList: vi.fn((teamModels: string[], allModels: string[]) => { + const wildcardDisplayNames: string[] = []; + const expandedModels: string[] = []; + + teamModels.forEach((teamModel) => { + if (teamModel.endsWith("/*")) { + const provider = teamModel.replace("/*", ""); + const matchingModels = allModels.filter((model) => model.startsWith(provider + "/")); + expandedModels.push(...matchingModels); + wildcardDisplayNames.push(teamModel); + } else { + expandedModels.push(teamModel); + } + }); + + return [...wildcardDisplayNames, ...expandedModels].filter((item, index, array) => array.indexOf(item) === index); + }), +})); + describe("OldTeams - handleCreate organization handling", () => { beforeEach(() => { vi.clearAllMocks(); @@ -236,7 +259,7 @@ describe("OldTeams - handleCreate organization handling", () => { }); it("should clear the delete modal when the cancel button is clicked", async () => { - const { getByRole, getByTestId } = render( + render( { organizations={[]} />, ); - const deleteTeamButton = getByTestId("delete-team-button"); + const deleteTeamButton = screen.getByTestId("delete-team-button"); act(() => { fireEvent.click(deleteTeamButton); }); @@ -275,7 +298,7 @@ describe("OldTeams - empty state", () => { }); it("should display empty state message when teams array is empty", () => { - const { getByText } = render( + render( { />, ); - expect(getByText("No teams found")).toBeInTheDocument(); - expect(getByText("Adjust your filters or create a new team")).toBeInTheDocument(); + expect(screen.getByText("No teams found")).toBeInTheDocument(); + expect(screen.getByText("Adjust your filters or create a new team")).toBeInTheDocument(); }); it("should display empty state message when teams is null", () => { - const { getByText } = render( + render( { />, ); - expect(getByText("No teams found")).toBeInTheDocument(); - expect(getByText("Adjust your filters or create a new team")).toBeInTheDocument(); + expect(screen.getByText("No teams found")).toBeInTheDocument(); + expect(screen.getByText("Adjust your filters or create a new team")).toBeInTheDocument(); }); it("should not display empty state when teams array has items", () => { - const { queryByText, getByText } = render( + render( { />, ); - expect(queryByText("No teams found")).not.toBeInTheDocument(); - expect(queryByText("Adjust your filters or create a new team")).not.toBeInTheDocument(); - expect(getByText("Test Team")).toBeInTheDocument(); + expect(screen.queryByText("No teams found")).not.toBeInTheDocument(); + expect(screen.queryByText("Adjust your filters or create a new team")).not.toBeInTheDocument(); + expect(screen.getByText("Test Team")).toBeInTheDocument(); }); }); @@ -473,7 +496,7 @@ describe("OldTeams - Default Team Settings tab visibility", () => { }); it("should show Default Team Settings tab for Admin role", () => { - const { getByRole } = render( + render( { />, ); - expect(getByRole("tab", { name: "Default Team Settings" })).toBeInTheDocument(); + expect(screen.getByRole("tab", { name: "Default Team Settings" })).toBeInTheDocument(); }); it("should show Default Team Settings tab for proxy_admin role", () => { - const { getByRole } = render( + render( { />, ); - expect(getByRole("tab", { name: "Default Team Settings" })).toBeInTheDocument(); + expect(screen.getByRole("tab", { name: "Default Team Settings" })).toBeInTheDocument(); }); it("should not show Default Team Settings tab for proxy_admin_viewer role", () => { - const { queryByRole } = render( + render( { />, ); - expect(queryByRole("tab", { name: "Default Team Settings" })).not.toBeInTheDocument(); + expect(screen.queryByRole("tab", { name: "Default Team Settings" })).not.toBeInTheDocument(); }); it("should not show Default Team Settings tab for Admin Viewer role", () => { - const { queryByRole } = render( + render( { />, ); - expect(queryByRole("tab", { name: "Default Team Settings" })).not.toBeInTheDocument(); + expect(screen.queryByRole("tab", { name: "Default Team Settings" })).not.toBeInTheDocument(); + }); +}); + +describe("OldTeams - all-proxy-models dropdown visibility", () => { + beforeEach(() => { + vi.clearAllMocks(); + vi.mocked(fetchAvailableModelsForTeamOrKey).mockResolvedValue(["gpt-4", "gpt-3.5-turbo"]); + }); + + it("should not show all-proxy-models option when user has no access to it", async () => { + vi.mocked(fetchAvailableModelsForTeamOrKey).mockResolvedValue(["gpt-4", "gpt-3.5-turbo"]); + + render( + , + ); + + await waitFor(() => { + expect(fetchAvailableModelsForTeamOrKey).toHaveBeenCalled(); + }); + + const createButton = screen.getByRole("button", { name: /create new team/i }); + act(() => { + fireEvent.click(createButton); + }); + + await waitFor(() => { + expect(screen.getByLabelText(/models/i)).toBeInTheDocument(); + }); + const allProxyModelsOption = screen.queryByText("All Proxy Models"); + expect(allProxyModelsOption).not.toBeInTheDocument(); }); }); diff --git a/ui/litellm-dashboard/src/components/OldTeams.tsx b/ui/litellm-dashboard/src/components/OldTeams.tsx index eefb0302a89..83ec28a5177 100644 --- a/ui/litellm-dashboard/src/components/OldTeams.tsx +++ b/ui/litellm-dashboard/src/components/OldTeams.tsx @@ -30,8 +30,7 @@ import { Text, TextInput, } from "@tremor/react"; -import { Button as Button2, Form, Input, Modal, Select as Select2, Tooltip, Typography } from "antd"; -import { AlertTriangleIcon, XIcon } from "lucide-react"; +import { Button as Button2, Form, Input, Modal, Select as Select2, Switch, Tooltip, Typography } from "antd"; import React, { useEffect, useState } from "react"; import { formatNumberWithCommas } from "../utils/dataUtils"; import { fetchTeams } from "./common_components/fetch_teams"; @@ -77,6 +76,7 @@ interface EditTeamModalProps { } import { updateExistingKeys } from "@/utils/dataUtils"; +import DeleteResourceModal from "./common_components/DeleteResourceModal"; import { Member, teamCreateCall, v2TeamListCall } from "./networking"; interface TeamInfo { @@ -1139,11 +1139,22 @@ const Teams: React.FC = ({ } + rules={[ + { + required: true, + message: "Please select at least one model", + }, + ]} name="models" > - - All Proxy Models + {(isProxyAdminRole(userRole || "") || userModels.includes("all-proxy-models")) && ( + + All Proxy Models + + )} + + No Default Models {modelsToPick.map((model) => ( diff --git a/ui/litellm-dashboard/src/components/SSOSettings.tsx b/ui/litellm-dashboard/src/components/SSOSettings.tsx index 917aa1864e7..6402220f374 100644 --- a/ui/litellm-dashboard/src/components/SSOSettings.tsx +++ b/ui/litellm-dashboard/src/components/SSOSettings.tsx @@ -274,7 +274,9 @@ const SSOSettings: React.FC = ({ accessToken, possibleUIRoles, onChange={(value) => handleTextInputChange(key, value)} className="mt-2" > - + {availableModels.map((model: string) => (