diff --git a/.circleci/config.yml b/.circleci/config.yml index 4306fa5cf05..c523b1d191e 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -95,7 +95,7 @@ jobs: pip install opentelemetry-api==1.25.0 pip install opentelemetry-sdk==1.25.0 pip install opentelemetry-exporter-otlp==1.25.0 - pip install openai==1.68.2 + pip install openai==1.81.0 pip install prisma==0.11.0 pip install "detect_secrets==1.5.0" pip install "httpx==0.24.1" @@ -218,7 +218,7 @@ jobs: pip install opentelemetry-api==1.25.0 pip install opentelemetry-sdk==1.25.0 pip install opentelemetry-exporter-otlp==1.25.0 - pip install openai==1.68.2 + pip install openai==1.81.0 pip install prisma==0.11.0 pip install "detect_secrets==1.5.0" pip install "httpx==0.24.1" @@ -325,7 +325,7 @@ jobs: pip install opentelemetry-api==1.25.0 pip install opentelemetry-sdk==1.25.0 pip install opentelemetry-exporter-otlp==1.25.0 - pip install openai==1.68.2 + pip install openai==1.81.0 pip install prisma==0.11.0 pip install "detect_secrets==1.5.0" pip install "httpx==0.24.1" @@ -581,7 +581,7 @@ jobs: pip install opentelemetry-api==1.25.0 pip install opentelemetry-sdk==1.25.0 pip install opentelemetry-exporter-otlp==1.25.0 - pip install openai==1.68.2 + pip install openai==1.81.0 pip install prisma==0.11.0 pip install "detect_secrets==1.5.0" pip install "httpx==0.24.1" @@ -1472,7 +1472,7 @@ jobs: pip install "aiodynamo==23.10.1" pip install "asyncio==3.4.3" pip install "PyGithub==1.59.1" - pip install "openai==1.68.2" + pip install "openai==1.81.0" - run: name: Install Grype command: | @@ -1610,7 +1610,7 @@ jobs: pip install "aiodynamo==23.10.1" pip install "asyncio==3.4.3" pip install "PyGithub==1.59.1" - pip install "openai==1.68.2" + pip install "openai==1.81.0" # Run pytest and generate JUnit XML report - run: name: Build Docker image @@ -1733,7 +1733,7 @@ jobs: pip install "aiodynamo==23.10.1" pip install "asyncio==3.4.3" pip install "PyGithub==1.59.1" - pip install "openai==1.68.2" + pip install "openai==1.81.0" - run: name: Build Docker image command: docker build -t my-app:latest -f ./docker/Dockerfile.database . @@ -2256,7 +2256,7 @@ jobs: pip install "pytest-asyncio==0.21.1" pip install "google-cloud-aiplatform==1.43.0" pip install aiohttp - pip install "openai==1.68.2" + pip install "openai==1.81.0" pip install "assemblyai==0.37.0" python -m pip install --upgrade pip pip install "pydantic==2.10.2" @@ -2644,7 +2644,7 @@ jobs: pip install "pytest-retry==1.6.3" pip install "pytest-asyncio==0.21.1" pip install aiohttp - pip install "openai==1.68.2" + pip install "openai==1.81.0" python -m pip install --upgrade pip pip install "pydantic==2.10.2" pip install "pytest==7.3.1" diff --git a/.circleci/requirements.txt b/.circleci/requirements.txt index 0e2362c4e3d..b720d15a7fd 100644 --- a/.circleci/requirements.txt +++ b/.circleci/requirements.txt @@ -1,5 +1,5 @@ # used by CI/CD testing -openai==1.68.2 +openai==1.81.0 python-dotenv tiktoken importlib_metadata diff --git a/.github/ISSUE_TEMPLATE/feature_request.yml b/.github/ISSUE_TEMPLATE/feature_request.yml index 72943d0e6a2..13a2132ec95 100644 --- a/.github/ISSUE_TEMPLATE/feature_request.yml +++ b/.github/ISSUE_TEMPLATE/feature_request.yml @@ -23,10 +23,10 @@ body: validations: required: true - type: dropdown - id: ml-ops-team + id: hiring-interest attributes: - label: Are you a ML Ops Team? - description: This helps us prioritize your requests correctly + label: LiteLLM is hiring a founding backend engineer, are you interested in joining us and shipping to all our users? + description: If yes, apply here - https://www.ycombinator.com/companies/litellm/jobs/6uvoBp3-founding-backend-engineer options: - "No" - "Yes" diff --git a/.github/workflows/test-linting.yml b/.github/workflows/test-linting.yml index 0e1c895c3a4..ceeedbe7e13 100644 --- a/.github/workflows/test-linting.yml +++ b/.github/workflows/test-linting.yml @@ -22,9 +22,9 @@ jobs: - name: Install dependencies run: | - pip install openai==1.68.2 + pip install openai==1.81.0 poetry install --with dev - pip install openai==1.68.2 + pip install openai==1.81.0 diff --git a/Dockerfile b/Dockerfile index 3a74c46e688..b972aab0961 100644 --- a/Dockerfile +++ b/Dockerfile @@ -51,7 +51,7 @@ FROM $LITELLM_RUNTIME_IMAGE AS runtime USER root # Install runtime dependencies -RUN apk add --no-cache openssl +RUN apk add --no-cache openssl tzdata WORKDIR /app # Copy the current directory contents into the container at /app @@ -74,5 +74,5 @@ EXPOSE 4000/tcp ENTRYPOINT ["docker/prod_entrypoint.sh"] -# Append "--detailed_debug" to the end of CMD to view detailed debug logs +# Append "--detailed_debug" to the end of CMD to view detailed debug logs CMD ["--port", "4000"] diff --git a/deploy/charts/litellm-helm/Chart.yaml b/deploy/charts/litellm-helm/Chart.yaml index 5de591fd730..bd63ca6bfca 100644 --- a/deploy/charts/litellm-helm/Chart.yaml +++ b/deploy/charts/litellm-helm/Chart.yaml @@ -18,7 +18,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.4.3 +version: 0.4.4 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to diff --git a/deploy/charts/litellm-helm/README.md b/deploy/charts/litellm-helm/README.md index a0ba5781dfd..31bda3f7d79 100644 --- a/deploy/charts/litellm-helm/README.md +++ b/deploy/charts/litellm-helm/README.md @@ -34,6 +34,7 @@ If `db.useStackgresOperator` is used (not yet implemented): | `serviceAccount.create` | Whether or not to create a Kubernetes Service Account for this deployment. The default is `false` because LiteLLM has no need to access the Kubernetes API. | `false` | | `service.type` | Kubernetes Service type (e.g. `LoadBalancer`, `ClusterIP`, etc.) | `ClusterIP` | | `service.port` | TCP port that the Kubernetes Service will listen on. Also the TCP port within the Pod that the proxy will listen on. | `4000` | +| `service.loadBalancerClass` | Optional LoadBalancer implementation class (only used when `service.type` is `LoadBalancer`) | `""` | | `ingress.*` | See [values.yaml](./values.yaml) for example settings | N/A | | `proxy_config.*` | See [values.yaml](./values.yaml) for default settings. See [example_config_yaml](../../../litellm/proxy/example_config_yaml/) for configuration examples. | N/A | | `extraContainers[]` | An array of additional containers to be deployed as sidecars alongside the LiteLLM Proxy. | `[]` | diff --git a/deploy/charts/litellm-helm/templates/service.yaml b/deploy/charts/litellm-helm/templates/service.yaml index d8d81e78c89..11812208929 100644 --- a/deploy/charts/litellm-helm/templates/service.yaml +++ b/deploy/charts/litellm-helm/templates/service.yaml @@ -10,6 +10,9 @@ metadata: {{- include "litellm.labels" . | nindent 4 }} spec: type: {{ .Values.service.type }} + {{- if and (eq .Values.service.type "LoadBalancer") .Values.service.loadBalancerClass }} + loadBalancerClass: {{ .Values.service.loadBalancerClass }} + {{- end }} ports: - port: {{ .Values.service.port }} targetPort: http diff --git a/deploy/charts/litellm-helm/tests/service_tests.yaml b/deploy/charts/litellm-helm/tests/service_tests.yaml new file mode 100644 index 00000000000..43ed0180bc8 --- /dev/null +++ b/deploy/charts/litellm-helm/tests/service_tests.yaml @@ -0,0 +1,116 @@ +suite: Service Configuration Tests +templates: + - service.yaml +tests: + - it: should create a default ClusterIP service + template: service.yaml + asserts: + - isKind: + of: Service + - equal: + path: spec.type + value: ClusterIP + - equal: + path: spec.ports[0].port + value: 4000 + - equal: + path: spec.ports[0].targetPort + value: http + - equal: + path: spec.ports[0].protocol + value: TCP + - equal: + path: spec.ports[0].name + value: http + - isNull: + path: spec.loadBalancerClass + + - it: should create a NodePort service when specified + template: service.yaml + set: + service.type: NodePort + asserts: + - isKind: + of: Service + - equal: + path: spec.type + value: NodePort + - isNull: + path: spec.loadBalancerClass + + - it: should create a LoadBalancer service when specified + template: service.yaml + set: + service.type: LoadBalancer + asserts: + - isKind: + of: Service + - equal: + path: spec.type + value: LoadBalancer + - isNull: + path: spec.loadBalancerClass + + - it: should add loadBalancerClass when specified with LoadBalancer type + template: service.yaml + set: + service.type: LoadBalancer + service.loadBalancerClass: tailscale + asserts: + - isKind: + of: Service + - equal: + path: spec.type + value: LoadBalancer + - equal: + path: spec.loadBalancerClass + value: tailscale + + - it: should not add loadBalancerClass when specified with ClusterIP type + template: service.yaml + set: + service.type: ClusterIP + service.loadBalancerClass: tailscale + asserts: + - isKind: + of: Service + - equal: + path: spec.type + value: ClusterIP + - isNull: + path: spec.loadBalancerClass + + - it: should use custom port when specified + template: service.yaml + set: + service.port: 8080 + asserts: + - equal: + path: spec.ports[0].port + value: 8080 + + - it: should add service annotations when specified + template: service.yaml + set: + service.annotations: + cloud.google.com/load-balancer-type: "Internal" + service.beta.kubernetes.io/aws-load-balancer-internal: "true" + asserts: + - isKind: + of: Service + - equal: + path: metadata.annotations + value: + cloud.google.com/load-balancer-type: "Internal" + service.beta.kubernetes.io/aws-load-balancer-internal: "true" + + - it: should use the correct selector labels + template: service.yaml + asserts: + - isNotNull: + path: spec.selector + - equal: + path: spec.selector + value: + app.kubernetes.io/name: litellm + app.kubernetes.io/instance: RELEASE-NAME diff --git a/deploy/charts/litellm-helm/values.yaml b/deploy/charts/litellm-helm/values.yaml index 0440e28eed0..213db35a20f 100644 --- a/deploy/charts/litellm-helm/values.yaml +++ b/deploy/charts/litellm-helm/values.yaml @@ -56,6 +56,9 @@ environmentConfigMaps: [] service: type: ClusterIP port: 4000 + # If service type is `LoadBalancer` you can + # optionally specify loadBalancerClass + # loadBalancerClass: tailscale ingress: enabled: false diff --git a/docs/my-website/docs/pass_through/vertex_ai.md b/docs/my-website/docs/pass_through/vertex_ai.md index b99f0fcf982..d3f4e75e31d 100644 --- a/docs/my-website/docs/pass_through/vertex_ai.md +++ b/docs/my-website/docs/pass_through/vertex_ai.md @@ -116,7 +116,7 @@ curl \ ```bash -curl http://localhost:4000/vertex_ai/vertex_ai/v1/projects/${PROJECT_ID}/locations/us-central1/publishers/google/models/${MODEL_ID}:generateContent \ +curl http://localhost:4000/vertex_ai/v1/projects/${PROJECT_ID}/locations/us-central1/publishers/google/models/${MODEL_ID}:generateContent \ -H "Content-Type: application/json" \ -H "x-litellm-api-key: Bearer sk-1234" \ -d '{ diff --git a/docs/my-website/docs/providers/anthropic.md b/docs/my-website/docs/providers/anthropic.md index 990a9b5122e..4ab4eb06086 100644 --- a/docs/my-website/docs/providers/anthropic.md +++ b/docs/my-website/docs/providers/anthropic.md @@ -4,6 +4,8 @@ import TabItem from '@theme/TabItem'; # Anthropic LiteLLM supports all anthropic models. +- `claude-4` (`claude-opus-4-20250514`, `claude-sonnet-4-20250514`) +- `claude-3.7` (`claude-3-7-sonnet-20250219`) - `claude-3.5` (`claude-3-5-sonnet-20240620`) - `claude-3` (`claude-3-haiku-20240307`, `claude-3-opus-20240229`, `claude-3-sonnet-20240229`) - `claude-2` @@ -64,7 +66,7 @@ from litellm import completion os.environ["ANTHROPIC_API_KEY"] = "your-api-key" messages = [{"role": "user", "content": "Hey! how's it going?"}] -response = completion(model="claude-3-opus-20240229", messages=messages) +response = completion(model="claude-opus-4-20250514", messages=messages) print(response) ``` @@ -80,7 +82,7 @@ from litellm import completion os.environ["ANTHROPIC_API_KEY"] = "your-api-key" messages = [{"role": "user", "content": "Hey! how's it going?"}] -response = completion(model="claude-3-opus-20240229", messages=messages, stream=True) +response = completion(model="claude-opus-4-20250514", messages=messages, stream=True) for chunk in response: print(chunk["choices"][0]["delta"]["content"]) # same as openai format ``` @@ -102,9 +104,9 @@ export ANTHROPIC_API_KEY="your-api-key" ```yaml model_list: - - model_name: claude-3 ### RECEIVED MODEL NAME ### + - model_name: claude-4 ### RECEIVED MODEL NAME ### litellm_params: # all params accepted by litellm.completion() - https://docs.litellm.ai/docs/completion/input - model: claude-3-opus-20240229 ### MODEL NAME sent to `litellm.completion()` ### + model: claude-opus-4-20250514 ### MODEL NAME sent to `litellm.completion()` ### api_key: "os.environ/ANTHROPIC_API_KEY" # does os.getenv("AZURE_API_KEY_EU") ``` @@ -156,7 +158,7 @@ curl --location 'http://0.0.0.0:4000/chat/completions' \ ```bash -$ litellm --model claude-3-opus-20240229 +$ litellm --model claude-opus-4-20250514 # Server running on http://0.0.0.0:4000 ``` @@ -244,6 +246,9 @@ print(response) | Model Name | Function Call | |------------------|--------------------------------------------| +| claude-opus-4 | `completion('claude-opus-4-20250514', messages)` | `os.environ['ANTHROPIC_API_KEY']` | +| claude-sonnet-4 | `completion('claude-sonnet-4-20250514', messages)` | `os.environ['ANTHROPIC_API_KEY']` | +| claude-3.7 | `completion('claude-3-7-sonnet-20250219', messages)` | `os.environ['ANTHROPIC_API_KEY']` | | claude-3-5-sonnet | `completion('claude-3-5-sonnet-20240620', messages)` | `os.environ['ANTHROPIC_API_KEY']` | | claude-3-haiku | `completion('claude-3-haiku-20240307', messages)` | `os.environ['ANTHROPIC_API_KEY']` | | claude-3-opus | `completion('claude-3-opus-20240229', messages)` | `os.environ['ANTHROPIC_API_KEY']` | diff --git a/docs/my-website/docs/providers/openai/responses_api.md b/docs/my-website/docs/providers/openai/responses_api.md index 578ce038f37..e88512ecfd4 100644 --- a/docs/my-website/docs/providers/openai/responses_api.md +++ b/docs/my-website/docs/providers/openai/responses_api.md @@ -318,3 +318,133 @@ print(response) + + +## MCP Tools + + + + +```python showLineNumbers title="MCP Tools with LiteLLM SDK" +import litellm +from typing import Optional + +# Configure MCP Tools +MCP_TOOLS = [ + { + "type": "mcp", + "server_label": "deepwiki", + "server_url": "https://mcp.deepwiki.com/mcp", + "allowed_tools": ["ask_question"] + } +] + +# Step 1: Make initial request - OpenAI will use MCP LIST and return MCP calls for approval +response = litellm.responses( + model="openai/gpt-4.1", + tools=MCP_TOOLS, + input="What transport protocols does the 2025-03-26 version of the MCP spec support?" +) + +# Get the MCP approval ID +mcp_approval_id = None +for output in response.output: + if output.type == "mcp_approval_request": + mcp_approval_id = output.id + break + +# Step 2: Send followup with approval for the MCP call +response_with_mcp_call = litellm.responses( + model="openai/gpt-4.1", + tools=MCP_TOOLS, + input=[ + { + "type": "mcp_approval_response", + "approve": True, + "approval_request_id": mcp_approval_id + } + ], + previous_response_id=response.id, +) + +print(response_with_mcp_call) +``` + + + + +1. Set up config.yaml + +```yaml showLineNumbers title="OpenAI Proxy Configuration" +model_list: + - model_name: openai/gpt-4.1 + litellm_params: + model: openai/gpt-4.1 + api_key: os.environ/OPENAI_API_KEY +``` + +2. Start LiteLLM Proxy Server + +```bash title="Start LiteLLM Proxy Server" +litellm --config /path/to/config.yaml + +# RUNNING on http://0.0.0.0:4000 +``` + +3. Test it! + +```python showLineNumbers title="MCP Tools with OpenAI SDK via LiteLLM Proxy" +from openai import OpenAI +from typing import Optional + +# Initialize client with your proxy URL +client = OpenAI( + base_url="http://localhost:4000", # Your proxy URL + api_key="your-api-key" # Your proxy API key +) + +# Configure MCP Tools +MCP_TOOLS = [ + { + "type": "mcp", + "server_label": "deepwiki", + "server_url": "https://mcp.deepwiki.com/mcp", + "allowed_tools": ["ask_question"] + } +] + +# Step 1: Make initial request - OpenAI will use MCP LIST and return MCP calls for approval +response = client.responses.create( + model="openai/gpt-4.1", + tools=MCP_TOOLS, + input="What transport protocols does the 2025-03-26 version of the MCP spec support?" +) + +# Get the MCP approval ID +mcp_approval_id = None +for output in response.output: + if output.type == "mcp_approval_request": + mcp_approval_id = output.id + break + +# Step 2: Send followup with approval for the MCP call +response_with_mcp_call = client.responses.create( + model="openai/gpt-4.1", + tools=MCP_TOOLS, + input=[ + { + "type": "mcp_approval_response", + "approve": True, + "approval_request_id": mcp_approval_id + } + ], + previous_response_id=response.id, +) + +print(response_with_mcp_call) +``` + + + + + diff --git a/docs/my-website/docs/proxy/config_settings.md b/docs/my-website/docs/proxy/config_settings.md index fdd68c953f6..f8d219c6a02 100644 --- a/docs/my-website/docs/proxy/config_settings.md +++ b/docs/my-website/docs/proxy/config_settings.md @@ -528,6 +528,7 @@ router_settings: | MAX_TOKEN_TRIMMING_ATTEMPTS | Maximum number of attempts to trim a token message. Default is 10 | MAXIMUM_TRACEBACK_LINES_TO_LOG | Maximum number of lines to log in traceback in LiteLLM Logs UI. Default is 100 | MAX_RETRY_DELAY | Maximum delay in seconds for retrying requests. Default is 8.0 +| MAX_LANGFUSE_INITIALIZED_CLIENTS | Maximum number of Langfuse clients to initialize on proxy. Default is 20. This is set since langfuse initializes 1 thread everytime a client is initialized. We've had an incident in the past where we reached 100% cpu utilization because Langfuse was initialized several times. | MIN_NON_ZERO_TEMPERATURE | Minimum non-zero temperature value. Default is 0.0001 | MINIMUM_PROMPT_CACHE_TOKEN_COUNT | Minimum token count for caching a prompt. Default is 1024 | MISTRAL_API_BASE | Base URL for Mistral API diff --git a/docs/my-website/docs/proxy/guardrails/pii_masking_v2.md b/docs/my-website/docs/proxy/guardrails/pii_masking_v2.md index 427308cf221..c93eb52a2a7 100644 --- a/docs/my-website/docs/proxy/guardrails/pii_masking_v2.md +++ b/docs/my-website/docs/proxy/guardrails/pii_masking_v2.md @@ -418,7 +418,7 @@ guardrails: Send ad-hoc recognizers to presidio `/analyze` by passing a json file to the proxy -[**Example** ad-hoc recognizer](../../../../litellm/proxy/hooks/example_presidio_ad_hoc_recognize) +[**Example** ad-hoc recognizer](https://github.com/BerriAI/litellm/blob/b69b7503db5aa039a49b7ca96ae5b34db0d25a3d/litellm/proxy/hooks/example_presidio_ad_hoc_recognizer.json) #### Define ad-hoc recognizer on your LiteLLM config.yaml diff --git a/enterprise/enterprise_hooks/__init__.py b/enterprise/enterprise_hooks/__init__.py index 830d97886a6..9cfe9218f00 100644 --- a/enterprise/enterprise_hooks/__init__.py +++ b/enterprise/enterprise_hooks/__init__.py @@ -1,4 +1,3 @@ -import os from typing import Dict, Literal, Type, Union from litellm.integrations.custom_logger import CustomLogger diff --git a/enterprise/enterprise_hooks/managed_files.py b/enterprise/enterprise_hooks/managed_files.py index 0dc86294d36..7410ad793b2 100644 --- a/enterprise/enterprise_hooks/managed_files.py +++ b/enterprise/enterprise_hooks/managed_files.py @@ -1,29 +1,44 @@ # What is this? ## This hook is used to check for LiteLLM managed files in the request body, and replace them with model-specific file id +import asyncio import base64 import json import uuid -from typing import TYPE_CHECKING, Any, Dict, List, Literal, Optional, Union, cast +from typing import TYPE_CHECKING, Any, Dict, List, Literal, Optional, Tuple, Union, cast + +from fastapi import HTTPException from litellm import Router, verbose_logger from litellm.caching.caching import DualCache from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.prompt_templates.common_utils import extract_file_data from litellm.llms.base_llm.files.transformation import BaseFileEndpoints -from litellm.proxy._types import CallTypes, LiteLLM_ManagedFileTable, UserAPIKeyAuth +from litellm.proxy._types import ( + CallTypes, + LiteLLM_ManagedFileTable, + LiteLLM_ManagedObjectTable, + UserAPIKeyAuth, +) from litellm.proxy.openai_files_endpoints.common_utils import ( _is_base64_encoded_unified_file_id, convert_b64_uid_to_unified_uid, ) from litellm.types.llms.openai import ( AllMessageValues, + AsyncCursorPage, ChatCompletionFileObject, CreateFileRequest, + FileObject, OpenAIFileObject, OpenAIFilesPurpose, ) -from litellm.types.utils import LiteLLMBatch, LLMResponseTypes, SpecialEnums +from litellm.types.utils import ( + LiteLLMBatch, + LiteLLMFineTuningJob, + LLMResponseTypes, + SpecialEnums, +) if TYPE_CHECKING: from opentelemetry.trace import Span as _Span @@ -54,6 +69,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): file_object: OpenAIFileObject, litellm_parent_otel_span: Optional[Span], model_mappings: Dict[str, str], + user_api_key_dict: UserAPIKeyAuth, ) -> None: verbose_logger.info( f"Storing LiteLLM Managed File object with id={file_id} in cache" @@ -62,6 +78,9 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): unified_file_id=file_id, file_object=file_object, model_mappings=model_mappings, + flat_model_file_ids=list(model_mappings.values()), + created_by=user_api_key_dict.user_id, + updated_by=user_api_key_dict.user_id, ) await self.internal_usage_cache.async_set_cache( key=file_id, @@ -74,6 +93,44 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): "unified_file_id": file_id, "file_object": file_object.model_dump_json(), "model_mappings": json.dumps(model_mappings), + "flat_model_file_ids": list(model_mappings.values()), + "created_by": user_api_key_dict.user_id, + "updated_by": user_api_key_dict.user_id, + } + ) + + async def store_unified_object_id( + self, + unified_object_id: str, + file_object: Union[LiteLLMBatch, LiteLLMFineTuningJob], + litellm_parent_otel_span: Optional[Span], + model_object_id: str, + file_purpose: Literal["batch", "fine-tune"], + user_api_key_dict: UserAPIKeyAuth, + ) -> None: + verbose_logger.info( + f"Storing LiteLLM Managed {file_purpose} object with id={unified_object_id} in cache" + ) + litellm_managed_object = LiteLLM_ManagedObjectTable( + unified_object_id=unified_object_id, + model_object_id=model_object_id, + file_purpose=file_purpose, + file_object=file_object, + ) + await self.internal_usage_cache.async_set_cache( + key=unified_object_id, + value=litellm_managed_object.model_dump(), + litellm_parent_otel_span=litellm_parent_otel_span, + ) + + await self.prisma_client.db.litellm_managedobjecttable.create( + data={ + "unified_object_id": unified_object_id, + "file_object": file_object.model_dump_json(), + "model_object_id": model_object_id, + "file_purpose": file_purpose, + "created_by": user_api_key_dict.user_id, + "updated_by": user_api_key_dict.user_id, } ) @@ -121,6 +178,71 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): ) return initial_value.file_object + async def can_user_call_unified_file_id( + self, unified_file_id: str, user_api_key_dict: UserAPIKeyAuth + ) -> bool: + ## check if the user has access to the unified file id + user_id = user_api_key_dict.user_id + managed_file = await self.prisma_client.db.litellm_managedfiletable.find_first( + where={"unified_file_id": unified_file_id} + ) + if managed_file: + return managed_file.created_by == user_id + return False + + async def can_user_call_unified_object_id( + self, unified_object_id: str, user_api_key_dict: UserAPIKeyAuth + ) -> bool: + ## check if the user has access to the unified object id + ## check if the user has access to the unified object id + user_id = user_api_key_dict.user_id + managed_object = ( + await self.prisma_client.db.litellm_managedobjecttable.find_first( + where={"unified_object_id": unified_object_id} + ) + ) + if managed_object: + return managed_object.created_by == user_id + return False + + async def get_user_created_file_ids( + self, user_api_key_dict: UserAPIKeyAuth, model_object_ids: List[str] + ) -> List[OpenAIFileObject]: + """ + Get all file ids created by the user for a list of model object ids + + Returns: + - List of OpenAIFileObject's + """ + file_ids = await self.prisma_client.db.litellm_managedfiletable.find_many( + where={ + "created_by": user_api_key_dict.user_id, + "flat_model_file_ids": {"hasSome": model_object_ids}, + } + ) + return [OpenAIFileObject(**file_object.file_object) for file_object in file_ids] + + async def check_managed_file_id_access( + self, data: Dict, user_api_key_dict: UserAPIKeyAuth + ) -> bool: + retrieve_file_id = cast(Optional[str], data.get("file_id")) + potential_file_id = ( + _is_base64_encoded_unified_file_id(retrieve_file_id) + if retrieve_file_id + else False + ) + if potential_file_id and retrieve_file_id: + if await self.can_user_call_unified_file_id( + retrieve_file_id, user_api_key_dict + ): + return True + else: + raise HTTPException( + status_code=403, + detail=f"User {user_api_key_dict.user_id} does not have access to the file {retrieve_file_id}", + ) + return False + async def async_pre_call_hook( self, user_api_key_dict: UserAPIKeyAuth, @@ -137,13 +259,28 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): "rerank", "acreate_batch", "aretrieve_batch", + "acreate_file", + "afile_list", + "afile_delete", "afile_content", + "acreate_fine_tuning_job", + "aretrieve_fine_tuning_job", + "alist_fine_tuning_jobs", + "acancel_fine_tuning_job", ], ) -> Union[Exception, str, Dict, None]: """ - Detect litellm_proxy/ file_id - add dictionary of mappings of litellm_proxy/ file_id -> provider_file_id => {litellm_proxy/file_id: {"model_id": id, "file_id": provider_file_id}} """ + ### HANDLE FILE ACCESS ### - ensure user has access to the file + if ( + call_type == CallTypes.afile_content.value + or call_type == CallTypes.afile_delete.value + ): + await self.check_managed_file_id_access(data, user_api_key_dict) + + ### HANDLE TRANSFORMATIONS ### if call_type == CallTypes.completion.value: messages = data.get("messages") if messages: @@ -176,25 +313,56 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): ) data["model_file_id_mapping"] = model_file_id_mapping - elif call_type == CallTypes.aretrieve_batch.value: - retrieve_batch_id = cast(Optional[str], data.get("batch_id")) - potential_batch_id = ( - _is_base64_encoded_unified_file_id(retrieve_batch_id) - if retrieve_batch_id + elif ( + call_type == CallTypes.aretrieve_batch.value + or call_type == CallTypes.acancel_fine_tuning_job.value + or call_type == CallTypes.aretrieve_fine_tuning_job.value + ): + accessor_key: Optional[str] = None + retrieve_object_id: Optional[str] = None + if call_type == CallTypes.aretrieve_batch.value: + accessor_key = "batch_id" + elif ( + call_type == CallTypes.acancel_fine_tuning_job.value + or call_type == CallTypes.aretrieve_fine_tuning_job.value + ): + accessor_key = "fine_tuning_job_id" + + if accessor_key: + retrieve_object_id = cast(Optional[str], data.get(accessor_key)) + + potential_llm_object_id = ( + _is_base64_encoded_unified_file_id(retrieve_object_id) + if retrieve_object_id else False ) - if potential_batch_id: + if potential_llm_object_id and retrieve_object_id: + ## VALIDATE USER HAS ACCESS TO THE OBJECT ## + if not await self.can_user_call_unified_object_id( + retrieve_object_id, user_api_key_dict + ): + raise HTTPException( + status_code=403, + detail=f"User {user_api_key_dict.user_id} does not have access to the object {retrieve_object_id}", + ) + ## for managed batch id - get the model id potential_model_id = self.get_model_id_from_unified_batch_id( - potential_batch_id + potential_llm_object_id ) if potential_model_id is None: raise Exception( - f"LiteLLM Managed Batch ID with id={retrieve_batch_id} is invalid - does not contain encoded model_id." + f"LiteLLM Managed {accessor_key} with id={retrieve_object_id} is invalid - does not contain encoded model_id." ) data["model"] = potential_model_id - data["batch_id"] = self.get_batch_id_from_unified_batch_id( - potential_batch_id + data[accessor_key] = self.get_batch_id_from_unified_batch_id( + potential_llm_object_id + ) + elif call_type == CallTypes.acreate_fine_tuning_job.value: + input_file_id = cast(Optional[str], data.get("training_file")) + if input_file_id: + model_file_id_mapping = await self.get_model_file_id_mapping( + [input_file_id], user_api_key_dict.parent_otel_span ) return data @@ -205,8 +373,16 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): """ Allow modifying the request just before it's sent to the deployment. """ + accessor_key: Optional[str] = None if call_type and call_type == CallTypes.acreate_batch: - input_file_id = cast(Optional[str], kwargs.get("input_file_id")) + accessor_key = "input_file_id" + elif call_type and call_type == CallTypes.acreate_fine_tuning_job: + accessor_key = "training_file" + else: + return kwargs + + if accessor_key: + input_file_id = cast(Optional[str], kwargs.get(accessor_key)) model_file_id_mapping = cast( Optional[Dict[str, Dict[str, str]]], kwargs.get("model_file_id_mapping") ) @@ -217,7 +393,8 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): model_id, None ) if mapped_file_id: - kwargs["input_file_id"] = mapped_file_id + kwargs[accessor_key] = mapped_file_id + return kwargs def get_file_ids_from_messages(self, messages: List[AllMessageValues]) -> List[str]: @@ -305,6 +482,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): llm_router: Router, target_model_names_list: List[str], litellm_parent_otel_span: Span, + user_api_key_dict: UserAPIKeyAuth, ) -> OpenAIFileObject: responses = await self.create_file_for_each_model( llm_router=llm_router, @@ -337,6 +515,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): file_object=response, litellm_parent_otel_span=litellm_parent_otel_span, model_mappings=model_mappings, + user_api_key_dict=user_api_key_dict, ) return response @@ -383,6 +562,20 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): return response + def get_unified_generic_response_id( + self, model_id: str, generic_response_id: str + ) -> str: + unified_generic_response_id = ( + SpecialEnums.LITELLM_MANAGED_GENERIC_RESPONSE_COMPLETE_STR.value.format( + model_id, generic_response_id + ) + ) + return ( + base64.urlsafe_b64encode(unified_generic_response_id.encode()) + .decode() + .rstrip("=") + ) + def get_unified_batch_id(self, batch_id: str, model_id: str) -> str: unified_batch_id = SpecialEnums.LITELLM_MANAGED_BATCH_COMPLETE_STR.value.format( model_id, batch_id @@ -427,11 +620,15 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): def get_batch_id_from_unified_batch_id(self, file_id: str) -> str: ## use regex to get the batch_id from the file_id - return file_id.split("llm_batch_id:")[1].split(",")[0] + if "llm_batch_id" in file_id: + return file_id.split("llm_batch_id:")[1].split(",")[0] + else: + return file_id.split("generic_response_id:")[1].split(",")[0] async def async_post_call_success_hook( self, data: Dict, user_api_key_dict: UserAPIKeyAuth, response: LLMResponseTypes ) -> Any: + print(f"response: {response}, type: {type(response)}") if isinstance(response, LiteLLMBatch): ## Check if unified_file_id is in the response unified_file_id = response._hidden_params.get( @@ -442,6 +639,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): ) # managed batch id model_id = cast(Optional[str], response._hidden_params.get("model_id")) model_name = cast(Optional[str], response._hidden_params.get("model_name")) + original_response_id = response.id if (unified_batch_id or unified_file_id) and model_id: response.id = self.get_unified_batch_id( batch_id=response.id, model_id=model_id @@ -455,10 +653,67 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): model_id=model_id, model_name=model_name, ) - - return await super().async_post_call_success_hook( - data, user_api_key_dict, response - ) + asyncio.create_task( + self.store_unified_object_id( + unified_object_id=response.id, + file_object=response, + litellm_parent_otel_span=user_api_key_dict.parent_otel_span, + model_object_id=original_response_id, + file_purpose="batch", + user_api_key_dict=user_api_key_dict, + ) + ) + elif isinstance(response, LiteLLMFineTuningJob): + ## Check if unified_file_id is in the response + unified_file_id = response._hidden_params.get( + "unified_file_id" + ) # managed file id + unified_finetuning_job_id = response._hidden_params.get( + "unified_finetuning_job_id" + ) # managed finetuning job id + model_id = cast(Optional[str], response._hidden_params.get("model_id")) + model_name = cast(Optional[str], response._hidden_params.get("model_name")) + original_response_id = response.id + if (unified_file_id or unified_finetuning_job_id) and model_id: + response.id = self.get_unified_generic_response_id( + model_id=model_id, generic_response_id=response.id + ) + asyncio.create_task( + self.store_unified_object_id( + unified_object_id=response.id, + file_object=response, + litellm_parent_otel_span=user_api_key_dict.parent_otel_span, + model_object_id=original_response_id, + file_purpose="fine-tune", + user_api_key_dict=user_api_key_dict, + ) + ) + elif isinstance(response, AsyncCursorPage): + """ + For listing files, filter for the ones created by the user + """ + print("INSIDE ASYNC CURSOR PAGE BLOCK") + ## check if file object + if hasattr(response, "data") and isinstance(response.data, list): + if all( + isinstance(file_object, FileObject) for file_object in response.data + ): + ## Get all file id's + ## Check which file id's were created by the user + ## Filter the response to only include the files created by the user + ## Return the filtered response + file_ids = [ + file_object.id + for file_object in cast(List[FileObject], response.data) # type: ignore + ] + user_created_file_ids = await self.get_user_created_file_ids( + user_api_key_dict, file_ids + ) + ## Filter the response to only include the files created by the user + response.data = user_created_file_ids # type: ignore + return response + return response + return response async def afile_retrieve( self, file_id: str, litellm_parent_otel_span: Optional[Span] @@ -477,6 +732,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): litellm_parent_otel_span: Optional[Span], **data: Dict, ) -> List[OpenAIFileObject]: + """Handled in files_endpoints.py""" return [] async def afile_delete( diff --git a/enterprise/litellm_enterprise/enterprise_callbacks/send_emails/base_email.py b/enterprise/litellm_enterprise/enterprise_callbacks/send_emails/base_email.py index 9ea22074c01..779d4f2eb37 100644 --- a/enterprise/litellm_enterprise/enterprise_callbacks/send_emails/base_email.py +++ b/enterprise/litellm_enterprise/enterprise_callbacks/send_emails/base_email.py @@ -189,7 +189,7 @@ class BaseEmailLogger(CustomLogger): # get the latest invitation link for the user invitation_rows = await prisma_client.db.litellm_invitationlink.find_many( where={"user_id": user_id}, - orderBy={"created_at": "desc"}, + order={"created_at": "desc"}, ) if len(invitation_rows) > 0: invitation_row = invitation_rows[0] diff --git a/enterprise/litellm_enterprise/proxy/auth/user_api_key_auth.py b/enterprise/litellm_enterprise/proxy/auth/user_api_key_auth.py new file mode 100644 index 00000000000..37bab50971e --- /dev/null +++ b/enterprise/litellm_enterprise/proxy/auth/user_api_key_auth.py @@ -0,0 +1,30 @@ +from typing import Any, Optional + +from fastapi import Request + +from litellm._logging import verbose_proxy_logger +from litellm.proxy._types import UserAPIKeyAuth + + +async def enterprise_custom_auth( + request: Request, api_key: str, user_custom_auth: Any +) -> Optional[UserAPIKeyAuth]: + from litellm_enterprise.proxy.proxy_server import custom_auth_settings + + if custom_auth_settings is None: + return None + + if custom_auth_settings["mode"] == "on": + return await user_custom_auth(request, api_key) + elif custom_auth_settings["mode"] == "off": + return None + elif custom_auth_settings["mode"] == "auto": + try: + return await user_custom_auth(request, api_key) + except Exception as e: + verbose_proxy_logger.debug( + f"Error in custom auth, checking litellm auth: {e}" + ) + return None + else: + raise ValueError(f"Invalid mode: {custom_auth_settings['mode']}") diff --git a/enterprise/litellm_enterprise/proxy/proxy_server.py b/enterprise/litellm_enterprise/proxy/proxy_server.py new file mode 100644 index 00000000000..481d65a9433 --- /dev/null +++ b/enterprise/litellm_enterprise/proxy/proxy_server.py @@ -0,0 +1,25 @@ +from typing import Optional + +from litellm_enterprise.types.proxy.proxy_server import CustomAuthSettings + +custom_auth_settings: Optional[CustomAuthSettings] = None + + +class EnterpriseProxyConfig: + async def load_custom_auth_settings( + self, general_settings: dict + ) -> CustomAuthSettings: + print(f"General settings: {general_settings}") + custom_auth_settings = general_settings.get("custom_auth_settings", None) + print(f"Custom auth settings: {custom_auth_settings}") + if custom_auth_settings is not None: + custom_auth_settings = CustomAuthSettings( + mode=custom_auth_settings.get("mode"), + ) + print(f"Custom auth settings: {custom_auth_settings}") + return custom_auth_settings + + async def load_enterprise_config(self, general_settings: dict) -> None: + global custom_auth_settings + custom_auth_settings = await self.load_custom_auth_settings(general_settings) + return None diff --git a/enterprise/litellm_enterprise/types/proxy/proxy_server.py b/enterprise/litellm_enterprise/types/proxy/proxy_server.py new file mode 100644 index 00000000000..497be59c4b9 --- /dev/null +++ b/enterprise/litellm_enterprise/types/proxy/proxy_server.py @@ -0,0 +1,5 @@ +from typing import Literal, TypedDict + + +class CustomAuthSettings(TypedDict): + mode: Literal["on", "off", "auto"] diff --git a/litellm-proxy-extras/litellm_proxy_extras/migrations/20250522223020_managed_object_table/migration.sql b/litellm-proxy-extras/litellm_proxy_extras/migrations/20250522223020_managed_object_table/migration.sql new file mode 100644 index 00000000000..95fb8372458 --- /dev/null +++ b/litellm-proxy-extras/litellm_proxy_extras/migrations/20250522223020_managed_object_table/migration.sql @@ -0,0 +1,32 @@ +-- AlterTable +ALTER TABLE "LiteLLM_ManagedFileTable" ADD COLUMN "created_by" TEXT, +ADD COLUMN "flat_model_file_ids" TEXT[] DEFAULT ARRAY[]::TEXT[], +ADD COLUMN "updated_by" TEXT; + +-- CreateTable +CREATE TABLE "LiteLLM_ManagedObjectTable" ( + "id" TEXT NOT NULL, + "unified_object_id" TEXT NOT NULL, + "model_object_id" TEXT NOT NULL, + "file_object" JSONB NOT NULL, + "file_purpose" TEXT NOT NULL, + "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "created_by" TEXT, + "updated_at" TIMESTAMP(3) NOT NULL, + "updated_by" TEXT, + + CONSTRAINT "LiteLLM_ManagedObjectTable_pkey" PRIMARY KEY ("id") +); + +-- CreateIndex +CREATE UNIQUE INDEX "LiteLLM_ManagedObjectTable_unified_object_id_key" ON "LiteLLM_ManagedObjectTable"("unified_object_id"); + +-- CreateIndex +CREATE UNIQUE INDEX "LiteLLM_ManagedObjectTable_model_object_id_key" ON "LiteLLM_ManagedObjectTable"("model_object_id"); + +-- CreateIndex +CREATE INDEX "LiteLLM_ManagedObjectTable_unified_object_id_idx" ON "LiteLLM_ManagedObjectTable"("unified_object_id"); + +-- CreateIndex +CREATE INDEX "LiteLLM_ManagedObjectTable_model_object_id_idx" ON "LiteLLM_ManagedObjectTable"("model_object_id"); + diff --git a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma index 1d6f3b52118..58064abd1dc 100644 --- a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma +++ b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma @@ -453,13 +453,30 @@ model LiteLLM_ManagedFileTable { id String @id @default(uuid()) unified_file_id String @unique // The base64 encoded unified file ID file_object Json // Stores the OpenAIFileObject - model_mappings Json // Stores the mapping of model_id -> provider_file_id + model_mappings Json + flat_model_file_ids String[] @default([]) // Flat list of model file id's - for faster querying of model id -> unified file id created_at DateTime @default(now()) + created_by String? updated_at DateTime @updatedAt + updated_by String? @@index([unified_file_id]) } +model LiteLLM_ManagedObjectTable { // for batches or finetuning jobs which use the + id String @id @default(uuid()) + unified_object_id String @unique // The base64 encoded unified file ID + model_object_id String @unique // the id returned by the backend API provider + file_object Json // Stores the OpenAIFileObject + file_purpose String // either 'batch' or 'fine-tune' + created_at DateTime @default(now()) + created_by String? + updated_at DateTime @updatedAt + updated_by String? + + @@index([unified_object_id]) + @@index([model_object_id]) +} model LiteLLM_ManagedVectorStoresTable { vector_store_id String @id diff --git a/litellm/__init__.py b/litellm/__init__.py index dd09810d46f..5e40d3e3d8d 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -126,6 +126,7 @@ _known_custom_logger_compatible_callbacks: List = list( callbacks: List[ Union[Callable, _custom_logger_compatible_callbacks_literal, CustomLogger] ] = [] +initialized_langfuse_clients: int = 0 langfuse_default_tags: Optional[List[str]] = None langsmith_batch_size: Optional[int] = None prometheus_initialize_budget_metrics: Optional[bool] = False @@ -354,6 +355,8 @@ project = None config_path = None vertex_ai_safety_settings: Optional[dict] = None BEDROCK_CONVERSE_MODELS = [ + "anthropic.claude-opus-4-20250514-v1:0", + "anthropic.claude-sonnet-4-20250514-v1:0", "anthropic.claude-3-7-sonnet-20250219-v1:0", "anthropic.claude-3-5-haiku-20241022-v1:0", "anthropic.claude-3-5-sonnet-20241022-v2:0", diff --git a/litellm/constants.py b/litellm/constants.py index 37d44f64c54..bb5d56978be 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -141,6 +141,7 @@ DEFAULT_MAX_TOKENS_FOR_TRITON = int(os.getenv("DEFAULT_MAX_TOKENS_FOR_TRITON", 2 #### Networking settings #### request_timeout: float = float(os.getenv("REQUEST_TIMEOUT", 6000)) # time in seconds STREAM_SSE_DONE_STRING: str = "[DONE]" +STREAM_SSE_DATA_PREFIX: str = "data: " ### SPEND TRACKING ### DEFAULT_REPLICATE_GPU_PRICE_PER_SECOND = float( os.getenv("DEFAULT_REPLICATE_GPU_PRICE_PER_SECOND", 0.001400) @@ -152,6 +153,9 @@ FIREWORKS_AI_16_B = int(os.getenv("FIREWORKS_AI_16_B", 16)) FIREWORKS_AI_80_B = int(os.getenv("FIREWORKS_AI_80_B", 80)) #### Logging callback constants #### REDACTED_BY_LITELM_STRING = "REDACTED_BY_LITELM" +MAX_LANGFUSE_INITIALIZED_CLIENTS = int( + os.getenv("MAX_LANGFUSE_INITIALIZED_CLIENTS", 20) +) ############### LLM Provider Constants ############### ### ANTHROPIC CONSTANTS ### diff --git a/litellm/fine_tuning/main.py b/litellm/fine_tuning/main.py index b7efcb40d42..f5b8b097026 100644 --- a/litellm/fine_tuning/main.py +++ b/litellm/fine_tuning/main.py @@ -22,12 +22,9 @@ from litellm.llms.azure.fine_tuning.handler import AzureOpenAIFineTuningAPI from litellm.llms.openai.fine_tuning.handler import OpenAIFineTuningAPI from litellm.llms.vertex_ai.fine_tuning.handler import VertexFineTuningAPI from litellm.secret_managers.main import get_secret_str -from litellm.types.llms.openai import ( - FineTuningJob, - FineTuningJobCreate, - Hyperparameters, -) +from litellm.types.llms.openai import FineTuningJobCreate, Hyperparameters from litellm.types.router import * +from litellm.types.utils import LiteLLMFineTuningJob from litellm.utils import client, supports_httpx_timeout ####### ENVIRONMENT VARIABLES ################### @@ -50,7 +47,7 @@ async def acreate_fine_tuning_job( extra_headers: Optional[Dict[str, str]] = None, extra_body: Optional[Dict[str, str]] = None, **kwargs, -) -> FineTuningJob: +) -> LiteLLMFineTuningJob: """ Async: Creates and executes a batch from an uploaded file of request @@ -104,7 +101,7 @@ def create_fine_tuning_job( extra_headers: Optional[Dict[str, str]] = None, extra_body: Optional[Dict[str, str]] = None, **kwargs, -) -> Union[FineTuningJob, Coroutine[Any, Any, FineTuningJob]]: +) -> Union[LiteLLMFineTuningJob, Coroutine[Any, Any, LiteLLMFineTuningJob]]: """ Creates a fine-tuning job which begins the process of creating a new model from a given dataset. @@ -288,13 +285,14 @@ def create_fine_tuning_job( raise e +@client async def acancel_fine_tuning_job( fine_tuning_job_id: str, custom_llm_provider: Literal["openai", "azure", "vertex_ai"] = "openai", extra_headers: Optional[Dict[str, str]] = None, extra_body: Optional[Dict[str, str]] = None, **kwargs, -) -> FineTuningJob: +) -> LiteLLMFineTuningJob: """ Async: Immediately cancel a fine-tune job. """ @@ -325,13 +323,14 @@ async def acancel_fine_tuning_job( raise e +@client def cancel_fine_tuning_job( fine_tuning_job_id: str, custom_llm_provider: Literal["openai", "azure", "vertex_ai"] = "openai", extra_headers: Optional[Dict[str, str]] = None, extra_body: Optional[Dict[str, str]] = None, **kwargs, -) -> Union[FineTuningJob, Coroutine[Any, Any, FineTuningJob]]: +) -> Union[LiteLLMFineTuningJob, Coroutine[Any, Any, LiteLLMFineTuningJob]]: """ Immediately cancel a fine-tune job. @@ -609,13 +608,14 @@ def list_fine_tuning_jobs( raise e +@client async def aretrieve_fine_tuning_job( fine_tuning_job_id: str, custom_llm_provider: Literal["openai", "azure", "vertex_ai"] = "openai", extra_headers: Optional[Dict[str, str]] = None, extra_body: Optional[Dict[str, str]] = None, **kwargs, -) -> FineTuningJob: +) -> LiteLLMFineTuningJob: """ Async: Get info about a fine-tuning job. """ @@ -646,13 +646,14 @@ async def aretrieve_fine_tuning_job( raise e +@client def retrieve_fine_tuning_job( fine_tuning_job_id: str, custom_llm_provider: Literal["openai", "azure", "vertex_ai"] = "openai", extra_headers: Optional[Dict[str, str]] = None, extra_body: Optional[Dict[str, str]] = None, **kwargs, -) -> Union[FineTuningJob, Coroutine[Any, Any, FineTuningJob]]: +) -> Union[LiteLLMFineTuningJob, Coroutine[Any, Any, LiteLLMFineTuningJob]]: """ Get info about a fine-tuning job. """ diff --git a/litellm/integrations/SlackAlerting/utils.py b/litellm/integrations/SlackAlerting/utils.py index 0dc8bae5a6a..e695266c88b 100644 --- a/litellm/integrations/SlackAlerting/utils.py +++ b/litellm/integrations/SlackAlerting/utils.py @@ -5,6 +5,7 @@ Utils used for slack alerting import asyncio from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union +import litellm from litellm.proxy._types import AlertType from litellm.secret_managers.main import get_secret @@ -69,7 +70,12 @@ async def _add_langfuse_trace_id_to_alert( -> trace_id -> litellm_call_id """ - # do nothing for now + if "langfuse" not in litellm.logging_callback_manager._get_all_callbacks(): + return None + ######################################################### + # Only run if langfuse is added as a callback + ######################################################### + if ( request_data is not None and request_data.get("litellm_logging_obj", None) is not None @@ -82,11 +88,12 @@ async def _add_langfuse_trace_id_to_alert( if trace_id is not None: break await asyncio.sleep(3) # wait 3s before retrying for trace id - - _langfuse_object = litellm_logging_obj._get_callback_object( + ######################################################### + langfuse_object = litellm_logging_obj._get_callback_object( service_name="langfuse" ) - if _langfuse_object is not None: - base_url = _langfuse_object.Langfuse.base_url + if langfuse_object is not None: + base_url = langfuse_object.Langfuse.base_url return f"{base_url}/trace/{trace_id}" + return None diff --git a/litellm/integrations/anthropic_cache_control_hook.py b/litellm/integrations/anthropic_cache_control_hook.py index c138b3cc254..5c75e452ab7 100644 --- a/litellm/integrations/anthropic_cache_control_hook.py +++ b/litellm/integrations/anthropic_cache_control_hook.py @@ -28,6 +28,7 @@ class AnthropicCacheControlHook(CustomPromptManagement): prompt_id: Optional[str], prompt_variables: Optional[dict], dynamic_callback_params: StandardCallbackDynamicParams, + prompt_label: Optional[str] = None, ) -> Tuple[str, List[AllMessageValues], dict]: """ Apply cache control directives based on specified injection points. @@ -79,10 +80,10 @@ class AnthropicCacheControlHook(CustomPromptManagement): # Case 1: Target by specific index if targetted_index is not None: if 0 <= targetted_index < len(messages): - messages[targetted_index] = ( - AnthropicCacheControlHook._safe_insert_cache_control_in_message( - messages[targetted_index], control - ) + messages[ + targetted_index + ] = AnthropicCacheControlHook._safe_insert_cache_control_in_message( + messages[targetted_index], control ) # Case 2: Target by role elif targetted_role is not None: diff --git a/litellm/integrations/custom_logger.py b/litellm/integrations/custom_logger.py index 960dc715e7e..ce97b9a292d 100644 --- a/litellm/integrations/custom_logger.py +++ b/litellm/integrations/custom_logger.py @@ -87,6 +87,7 @@ class CustomLogger: # https://docs.litellm.ai/docs/observability/custom_callbac dynamic_callback_params: StandardCallbackDynamicParams, litellm_logging_obj: LiteLLMLoggingObj, tools: Optional[List[Dict]] = None, + prompt_label: Optional[str] = None, ) -> Tuple[str, List[AllMessageValues], dict]: """ Returns: @@ -104,6 +105,7 @@ class CustomLogger: # https://docs.litellm.ai/docs/observability/custom_callbac prompt_id: Optional[str], prompt_variables: Optional[dict], dynamic_callback_params: StandardCallbackDynamicParams, + prompt_label: Optional[str] = None, ) -> Tuple[str, List[AllMessageValues], dict]: """ Returns: diff --git a/litellm/integrations/custom_prompt_management.py b/litellm/integrations/custom_prompt_management.py index 9d05e7b2426..061aadc3c05 100644 --- a/litellm/integrations/custom_prompt_management.py +++ b/litellm/integrations/custom_prompt_management.py @@ -18,6 +18,7 @@ class CustomPromptManagement(CustomLogger, PromptManagementBase): prompt_id: Optional[str], prompt_variables: Optional[dict], dynamic_callback_params: StandardCallbackDynamicParams, + prompt_label: Optional[str] = None, ) -> Tuple[str, List[AllMessageValues], dict]: """ Returns: @@ -43,6 +44,7 @@ class CustomPromptManagement(CustomLogger, PromptManagementBase): prompt_id: str, prompt_variables: Optional[dict], dynamic_callback_params: StandardCallbackDynamicParams, + prompt_label: Optional[str] = None, ) -> PromptManagementClient: raise NotImplementedError( "Custom prompt management does not support compile prompt helper" diff --git a/litellm/integrations/humanloop.py b/litellm/integrations/humanloop.py index 853fbe148cc..c62ab1110ff 100644 --- a/litellm/integrations/humanloop.py +++ b/litellm/integrations/humanloop.py @@ -155,11 +155,8 @@ class HumanloopLogger(CustomLogger): prompt_id: Optional[str], prompt_variables: Optional[dict], dynamic_callback_params: StandardCallbackDynamicParams, - ) -> Tuple[ - str, - List[AllMessageValues], - dict, - ]: + prompt_label: Optional[str] = None, + ) -> Tuple[str, List[AllMessageValues], dict,]: humanloop_api_key = dynamic_callback_params.get( "humanloop_api_key" ) or get_secret_str("HUMANLOOP_API_KEY") diff --git a/litellm/integrations/langfuse/langfuse.py b/litellm/integrations/langfuse/langfuse.py index f116eb8041b..4d478fdcca0 100644 --- a/litellm/integrations/langfuse/langfuse.py +++ b/litellm/integrations/langfuse/langfuse.py @@ -10,6 +10,7 @@ from packaging.version import Version import litellm from litellm._logging import verbose_logger +from litellm.constants import MAX_LANGFUSE_INITIALIZED_CLIENTS from litellm.litellm_core_utils.redact_messages import redact_user_api_key_info from litellm.llms.custom_httpx.http_handler import _get_httpx_client from litellm.secret_managers.main import str_to_bool @@ -27,12 +28,13 @@ from litellm.types.utils import ( ) if TYPE_CHECKING: - from langfuse.client import StatefulTraceClient + from langfuse.client import Langfuse, StatefulTraceClient from litellm.litellm_core_utils.litellm_logging import DynamicLoggingCache else: DynamicLoggingCache = Any StatefulTraceClient = Any + Langfuse = Any class LangFuseLogger: @@ -84,8 +86,7 @@ class LangFuseLogger: if Version(self.langfuse_sdk_version) >= Version("2.6.0"): parameters["sdk_integration"] = "litellm" - - self.Langfuse = Langfuse(**parameters) + self.Langfuse: Langfuse = self.safe_init_langfuse_client(parameters) # set the current langfuse project id in the environ # this is used by Alerting to link to the correct project @@ -124,6 +125,24 @@ class LangFuseLogger: else: self.upstream_langfuse = None + def safe_init_langfuse_client(self, parameters: dict) -> Langfuse: + """ + Safely init a langfuse client if the number of initialized clients is less than the max + + Note: + - Langfuse initializes 1 thread everytime a client is initialized. + - We've had an incident in the past where we reached 100% cpu utilization because Langfuse was initialized several times. + """ + from langfuse import Langfuse + + if litellm.initialized_langfuse_clients >= MAX_LANGFUSE_INITIALIZED_CLIENTS: + raise Exception( + f"Max langfuse clients reached: {litellm.initialized_langfuse_clients} is greater than {MAX_LANGFUSE_INITIALIZED_CLIENTS}" + ) + langfuse_client = Langfuse(**parameters) + litellm.initialized_langfuse_clients += 1 + return langfuse_client + @staticmethod def add_metadata_from_header(litellm_params: dict, metadata: dict) -> dict: """ diff --git a/litellm/integrations/langfuse/langfuse_prompt_management.py b/litellm/integrations/langfuse/langfuse_prompt_management.py index b4149d7ad97..8fe9cb63dea 100644 --- a/litellm/integrations/langfuse/langfuse_prompt_management.py +++ b/litellm/integrations/langfuse/langfuse_prompt_management.py @@ -130,9 +130,12 @@ class LangfusePromptManagement(LangFuseLogger, PromptManagementBase, CustomLogge return "langfuse" def _get_prompt_from_id( - self, langfuse_prompt_id: str, langfuse_client: LangfuseClass + self, + langfuse_prompt_id: str, + langfuse_client: LangfuseClass, + prompt_label: Optional[str] = None, ) -> PROMPT_CLIENT: - return langfuse_client.get_prompt(langfuse_prompt_id) + return langfuse_client.get_prompt(langfuse_prompt_id, label=prompt_label) def _compile_prompt( self, @@ -176,11 +179,8 @@ class LangfusePromptManagement(LangFuseLogger, PromptManagementBase, CustomLogge dynamic_callback_params: StandardCallbackDynamicParams, litellm_logging_obj: LiteLLMLoggingObj, tools: Optional[List[Dict]] = None, - ) -> Tuple[ - str, - List[AllMessageValues], - dict, - ]: + prompt_label: Optional[str] = None, + ) -> Tuple[str, List[AllMessageValues], dict,]: return self.get_chat_completion_prompt( model, messages, @@ -188,6 +188,7 @@ class LangfusePromptManagement(LangFuseLogger, PromptManagementBase, CustomLogge prompt_id, prompt_variables, dynamic_callback_params, + prompt_label=prompt_label, ) def should_run_prompt_management( @@ -211,6 +212,7 @@ class LangfusePromptManagement(LangFuseLogger, PromptManagementBase, CustomLogge prompt_id: str, prompt_variables: Optional[dict], dynamic_callback_params: StandardCallbackDynamicParams, + prompt_label: Optional[str] = None, ) -> PromptManagementClient: langfuse_client = langfuse_client_init( langfuse_public_key=dynamic_callback_params.get("langfuse_public_key"), @@ -219,7 +221,9 @@ class LangfusePromptManagement(LangFuseLogger, PromptManagementBase, CustomLogge langfuse_host=dynamic_callback_params.get("langfuse_host"), ) langfuse_prompt_client = self._get_prompt_from_id( - langfuse_prompt_id=prompt_id, langfuse_client=langfuse_client + langfuse_prompt_id=prompt_id, + langfuse_client=langfuse_client, + prompt_label=prompt_label, ) ## SET PROMPT diff --git a/litellm/integrations/prompt_management_base.py b/litellm/integrations/prompt_management_base.py index 270c34be8a6..c9e7adbccbd 100644 --- a/litellm/integrations/prompt_management_base.py +++ b/litellm/integrations/prompt_management_base.py @@ -33,6 +33,7 @@ class PromptManagementBase(ABC): prompt_id: str, prompt_variables: Optional[dict], dynamic_callback_params: StandardCallbackDynamicParams, + prompt_label: Optional[str] = None, ) -> PromptManagementClient: pass @@ -49,11 +50,13 @@ class PromptManagementBase(ABC): prompt_variables: Optional[dict], client_messages: List[AllMessageValues], dynamic_callback_params: StandardCallbackDynamicParams, + prompt_label: Optional[str] = None, ) -> PromptManagementClient: compiled_prompt_client = self._compile_prompt_helper( prompt_id=prompt_id, prompt_variables=prompt_variables, dynamic_callback_params=dynamic_callback_params, + prompt_label=prompt_label, ) try: @@ -82,6 +85,7 @@ class PromptManagementBase(ABC): prompt_id: Optional[str], prompt_variables: Optional[dict], dynamic_callback_params: StandardCallbackDynamicParams, + prompt_label: Optional[str] = None, ) -> Tuple[str, List[AllMessageValues], dict]: if prompt_id is None: raise ValueError("prompt_id is required for Prompt Management Base class") @@ -95,6 +99,7 @@ class PromptManagementBase(ABC): prompt_variables=prompt_variables, client_messages=messages, dynamic_callback_params=dynamic_callback_params, + prompt_label=prompt_label, ) completed_messages = prompt_template["completed_messages"] or messages diff --git a/litellm/integrations/vector_stores/bedrock_vector_store.py b/litellm/integrations/vector_stores/bedrock_vector_store.py index e0af1a66364..9015757000b 100644 --- a/litellm/integrations/vector_stores/bedrock_vector_store.py +++ b/litellm/integrations/vector_stores/bedrock_vector_store.py @@ -75,6 +75,7 @@ class BedrockVectorStore(BaseVectorStore, BaseAWSLLM): dynamic_callback_params: StandardCallbackDynamicParams, litellm_logging_obj: LiteLLMLoggingObj, tools: Optional[List[Dict]] = None, + prompt_label: Optional[str] = None, ) -> Tuple[str, List[AllMessageValues], dict]: """ Retrieves the context from the Bedrock Knowledge Base and appends it to the messages. @@ -99,10 +100,11 @@ class BedrockVectorStore(BaseVectorStore, BaseAWSLLM): f"Bedrock Knowledge Base Response: {bedrock_kb_response}" ) - context_message, context_string = ( - self.get_chat_completion_message_from_bedrock_kb_response( - bedrock_kb_response - ) + ( + context_message, + context_string, + ) = self.get_chat_completion_message_from_bedrock_kb_response( + bedrock_kb_response ) if context_message is not None: messages.append(context_message) @@ -126,9 +128,9 @@ class BedrockVectorStore(BaseVectorStore, BaseAWSLLM): ) ) - litellm_logging_obj.model_call_details["vector_store_request_metadata"] = ( - vector_store_request_metadata - ) + litellm_logging_obj.model_call_details[ + "vector_store_request_metadata" + ] = vector_store_request_metadata return model, messages, non_default_params @@ -140,9 +142,9 @@ class BedrockVectorStore(BaseVectorStore, BaseAWSLLM): """ Transform a BedrockKBResponse to a VectorStoreSearchResponse """ - retrieval_results: Optional[List[BedrockKBRetrievalResult]] = ( - bedrock_kb_response.get("retrievalResults", None) - ) + retrieval_results: Optional[ + List[BedrockKBRetrievalResult] + ] = bedrock_kb_response.get("retrievalResults", None) vector_store_search_response: VectorStoreSearchResponse = ( VectorStoreSearchResponse(search_query=query, data=[]) ) diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 88ce34245a6..dc5cffa2290 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -539,6 +539,7 @@ class Logging(LiteLLMLoggingBaseClass): prompt_id: Optional[str], prompt_variables: Optional[dict], prompt_management_logger: Optional[CustomLogger] = None, + prompt_label: Optional[str] = None, ) -> Tuple[str, List[AllMessageValues], dict]: custom_logger = ( prompt_management_logger @@ -559,6 +560,7 @@ class Logging(LiteLLMLoggingBaseClass): prompt_id=prompt_id, prompt_variables=prompt_variables, dynamic_callback_params=self.standard_callback_dynamic_params, + prompt_label=prompt_label, ) self.messages = messages return model, messages, non_default_params @@ -572,6 +574,7 @@ class Logging(LiteLLMLoggingBaseClass): prompt_variables: Optional[dict], prompt_management_logger: Optional[CustomLogger] = None, tools: Optional[List[Dict]] = None, + prompt_label: Optional[str] = None, ) -> Tuple[str, List[AllMessageValues], dict]: custom_logger = ( prompt_management_logger @@ -594,6 +597,7 @@ class Logging(LiteLLMLoggingBaseClass): dynamic_callback_params=self.standard_callback_dynamic_params, litellm_logging_obj=self, tools=tools, + prompt_label=prompt_label, ) self.messages = messages return model, messages, non_default_params diff --git a/litellm/litellm_core_utils/logging_callback_manager.py b/litellm/litellm_core_utils/logging_callback_manager.py index dec3add4e1b..e1bddc65497 100644 --- a/litellm/litellm_core_utils/logging_callback_manager.py +++ b/litellm/litellm_core_utils/logging_callback_manager.py @@ -266,3 +266,12 @@ class LoggingCallbackManager: if isinstance(callback, callback_type) and callback not in all_callbacks: all_callbacks.append(callback) return all_callbacks + + def callback_is_active(self, callback_type: Type[CustomLogger]) -> bool: + """ + Returns True if any of the active callbacks are of the given type + """ + return any( + isinstance(callback, callback_type) + for callback in self._get_all_callbacks() + ) diff --git a/litellm/litellm_core_utils/prompt_templates/common_utils.py b/litellm/litellm_core_utils/prompt_templates/common_utils.py index 387c072ffd7..a99a2677e8f 100644 --- a/litellm/litellm_core_utils/prompt_templates/common_utils.py +++ b/litellm/litellm_core_utils/prompt_templates/common_utils.py @@ -573,3 +573,12 @@ def get_tool_call_names(tools: List[ChatCompletionToolParam]) -> List[str]: if tool_call_name: tool_call_names.append(tool_call_name) return tool_call_names + + +def is_function_call(optional_params: dict) -> bool: + """ + Checks if the optional params contain the function call + """ + if "functions" in optional_params and optional_params.get("functions"): + return True + return False diff --git a/litellm/litellm_core_utils/streaming_handler.py b/litellm/litellm_core_utils/streaming_handler.py index dcc6ea36a30..5ae1dcf9889 100644 --- a/litellm/litellm_core_utils/streaming_handler.py +++ b/litellm/litellm_core_utils/streaming_handler.py @@ -149,14 +149,14 @@ class CustomStreamWrapper: ) def check_is_function_call(self, logging_obj) -> bool: + from litellm.litellm_core_utils.prompt_templates.common_utils import ( + is_function_call, + ) + if hasattr(logging_obj, "optional_params") and isinstance( logging_obj.optional_params, dict ): - if ( - "litellm_param_is_function_call" in logging_obj.optional_params - and logging_obj.optional_params["litellm_param_is_function_call"] - is True - ): + if is_function_call(logging_obj.optional_params): return True return False diff --git a/litellm/llms/anthropic/chat/transformation.py b/litellm/llms/anthropic/chat/transformation.py index 9052cec97cf..d7756bead07 100644 --- a/litellm/llms/anthropic/chat/transformation.py +++ b/litellm/llms/anthropic/chat/transformation.py @@ -49,6 +49,7 @@ from litellm.utils import ( Usage, add_dummy_tool, has_tool_call_blocks, + supports_reasoning, token_counter, ) @@ -121,7 +122,10 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): "web_search_options", ] - if "claude-3-7-sonnet" in model: + if "claude-3-7-sonnet" in model or supports_reasoning( + model=model, + custom_llm_provider=self.custom_llm_provider, + ): params.append("thinking") return params diff --git a/litellm/llms/base_llm/files/transformation.py b/litellm/llms/base_llm/files/transformation.py index 4d749af21e1..38a6dc48092 100644 --- a/litellm/llms/base_llm/files/transformation.py +++ b/litellm/llms/base_llm/files/transformation.py @@ -3,6 +3,7 @@ from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union import httpx +from litellm.proxy._types import UserAPIKeyAuth from litellm.types.llms.openai import ( AllMessageValues, CreateFileRequest, @@ -115,6 +116,7 @@ class BaseFileEndpoints(ABC): llm_router: Router, target_model_names_list: List[str], litellm_parent_otel_span: Span, + user_api_key_dict: UserAPIKeyAuth, ) -> OpenAIFileObject: pass diff --git a/litellm/llms/bedrock/chat/converse_transformation.py b/litellm/llms/bedrock/chat/converse_transformation.py index 7dad73d871d..f857f47af32 100644 --- a/litellm/llms/bedrock/chat/converse_transformation.py +++ b/litellm/llms/bedrock/chat/converse_transformation.py @@ -45,7 +45,7 @@ from litellm.types.utils import ( PromptTokensDetailsWrapper, Usage, ) -from litellm.utils import add_dummy_tool, has_tool_call_blocks +from litellm.utils import add_dummy_tool, has_tool_call_blocks, supports_reasoning from ..common_utils import BedrockError, BedrockModelInfo, get_bedrock_tool_name @@ -146,9 +146,10 @@ class AmazonConverseConfig(BaseConfig): # only anthropic and mistral support tool choice config. otherwise (E.g. cohere) will fail the call - https://docs.aws.amazon.com/bedrock/latest/APIReference/API_runtime_ToolChoice.html supported_params.append("tool_choice") - if ( - "claude-3-7" in model - ): # [TODO]: move to a 'supports_reasoning_content' param from model cost map + if "claude-3-7" in model or supports_reasoning( + model=model, + custom_llm_provider=self.custom_llm_provider, + ): supported_params.append("thinking") supported_params.append("reasoning_effort") return supported_params diff --git a/litellm/llms/bedrock/image/cost_calculator.py b/litellm/llms/bedrock/image/cost_calculator.py index 0a20b44cb38..a0dc91d7119 100644 --- a/litellm/llms/bedrock/image/cost_calculator.py +++ b/litellm/llms/bedrock/image/cost_calculator.py @@ -37,5 +37,7 @@ def cost_calculator( ) output_cost_per_image: float = _model_info.get("output_cost_per_image") or 0.0 - num_images: int = len(image_response.data) + num_images: int = 0 + if image_response.data: + num_images = len(image_response.data) return output_cost_per_image * num_images diff --git a/litellm/llms/custom_llm.py b/litellm/llms/custom_llm.py index a2d04b1838d..390258e4e82 100644 --- a/litellm/llms/custom_llm.py +++ b/litellm/llms/custom_llm.py @@ -14,7 +14,7 @@ import httpx from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler from litellm.types.utils import GenericStreamingChunk -from litellm.utils import ImageResponse, ModelResponse +from litellm.utils import ImageResponse, ModelResponse, EmbeddingResponse from .base import BaseLLM @@ -152,6 +152,30 @@ class CustomLLM(BaseLLM): ) -> ImageResponse: raise CustomLLMError(status_code=500, message="Not implemented yet!") + def embedding( + self, + model: str, + input: list, + model_response: EmbeddingResponse, + print_verbose: Callable, + logging_obj: Any, + optional_params: dict, + litellm_params=None, + ) -> EmbeddingResponse: + raise CustomLLMError(status_code=500, message="Not implemented yet!") + + async def aembedding( + self, + model: str, + input: list, + model_response: EmbeddingResponse, + print_verbose: Callable, + logging_obj: Any, + optional_params: dict, + litellm_params=None, + ) -> EmbeddingResponse: + raise CustomLLMError(status_code=500, message="Not implemented yet!") + def custom_chat_llm_router( async_fn: bool, stream: Optional[bool], custom_llm: CustomLLM diff --git a/litellm/llms/ollama/common_utils.py b/litellm/llms/ollama/common_utils.py index 4bee5e358c1..daff7a12065 100644 --- a/litellm/llms/ollama/common_utils.py +++ b/litellm/llms/ollama/common_utils.py @@ -1,4 +1,4 @@ -from typing import Optional, Union +from typing import List, Optional, Union import httpx @@ -67,7 +67,7 @@ class OllamaModelInfo(BaseLLMModelInfo): # env var OLLAMA_API_BASE or default return api_base or get_secret_str("OLLAMA_API_BASE") or "http://localhost:11434" - def get_models(self, api_key=None, api_base: Optional[str] = None) -> list[str]: + def get_models(self, api_key=None, api_base: Optional[str] = None) -> List[str]: """ List all models available on the Ollama server via /api/tags endpoint. """ diff --git a/litellm/llms/openai/fine_tuning/handler.py b/litellm/llms/openai/fine_tuning/handler.py index 2b697f85d2d..9804ff3539e 100644 --- a/litellm/llms/openai/fine_tuning/handler.py +++ b/litellm/llms/openai/fine_tuning/handler.py @@ -1,10 +1,10 @@ -from typing import Any, Coroutine, Optional, Union +from typing import Any, Coroutine, Optional, Union, cast import httpx from openai import AsyncAzureOpenAI, AsyncOpenAI, AzureOpenAI, OpenAI -from openai.types.fine_tuning import FineTuningJob from litellm._logging import verbose_logger +from litellm.types.utils import LiteLLMFineTuningJob class OpenAIFineTuningAPI: @@ -55,11 +55,12 @@ class OpenAIFineTuningAPI: self, create_fine_tuning_job_data: dict, openai_client: Union[AsyncOpenAI, AsyncAzureOpenAI], - ) -> FineTuningJob: + ) -> LiteLLMFineTuningJob: response = await openai_client.fine_tuning.jobs.create( **create_fine_tuning_job_data ) - return response + + return LiteLLMFineTuningJob(**response.model_dump()) def create_fine_tuning_job( self, @@ -74,7 +75,7 @@ class OpenAIFineTuningAPI: client: Optional[ Union[OpenAI, AsyncOpenAI, AzureOpenAI, AsyncAzureOpenAI] ] = None, - ) -> Union[FineTuningJob, Coroutine[Any, Any, FineTuningJob]]: + ) -> Union[LiteLLMFineTuningJob, Coroutine[Any, Any, LiteLLMFineTuningJob]]: openai_client: Optional[ Union[OpenAI, AsyncOpenAI, AzureOpenAI, AsyncAzureOpenAI] ] = self.get_openai_client( @@ -104,18 +105,20 @@ class OpenAIFineTuningAPI: verbose_logger.debug( "creating fine tuning job, args= %s", create_fine_tuning_job_data ) - response = openai_client.fine_tuning.jobs.create(**create_fine_tuning_job_data) - return response + response = cast(OpenAI, openai_client).fine_tuning.jobs.create( + **create_fine_tuning_job_data + ) + return LiteLLMFineTuningJob(**response.model_dump()) async def acancel_fine_tuning_job( self, fine_tuning_job_id: str, openai_client: Union[AsyncOpenAI, AsyncAzureOpenAI], - ) -> FineTuningJob: + ) -> LiteLLMFineTuningJob: response = await openai_client.fine_tuning.jobs.cancel( fine_tuning_job_id=fine_tuning_job_id ) - return response + return LiteLLMFineTuningJob(**response.model_dump()) def cancel_fine_tuning_job( self, @@ -130,7 +133,7 @@ class OpenAIFineTuningAPI: client: Optional[ Union[OpenAI, AsyncOpenAI, AzureOpenAI, AsyncAzureOpenAI] ] = None, - ): + ) -> Union[LiteLLMFineTuningJob, Coroutine[Any, Any, LiteLLMFineTuningJob]]: openai_client: Optional[ Union[OpenAI, AsyncOpenAI, AzureOpenAI, AsyncAzureOpenAI] ] = self.get_openai_client( @@ -158,10 +161,10 @@ class OpenAIFineTuningAPI: openai_client=openai_client, ) verbose_logger.debug("canceling fine tuning job, args= %s", fine_tuning_job_id) - response = openai_client.fine_tuning.jobs.cancel( + response = cast(OpenAI, openai_client).fine_tuning.jobs.cancel( fine_tuning_job_id=fine_tuning_job_id ) - return response + return LiteLLMFineTuningJob(**response.model_dump()) async def alist_fine_tuning_jobs( self, @@ -222,11 +225,11 @@ class OpenAIFineTuningAPI: self, fine_tuning_job_id: str, openai_client: Union[AsyncOpenAI, AsyncAzureOpenAI], - ) -> FineTuningJob: + ) -> LiteLLMFineTuningJob: response = await openai_client.fine_tuning.jobs.retrieve( fine_tuning_job_id=fine_tuning_job_id ) - return response + return LiteLLMFineTuningJob(**response.model_dump()) def retrieve_fine_tuning_job( self, @@ -241,7 +244,7 @@ class OpenAIFineTuningAPI: client: Optional[ Union[OpenAI, AsyncOpenAI, AzureOpenAI, AsyncAzureOpenAI] ] = None, - ): + ) -> Union[LiteLLMFineTuningJob, Coroutine[Any, Any, LiteLLMFineTuningJob]]: openai_client: Optional[ Union[OpenAI, AsyncOpenAI, AzureOpenAI, AsyncAzureOpenAI] ] = self.get_openai_client( @@ -269,7 +272,7 @@ class OpenAIFineTuningAPI: openai_client=openai_client, ) verbose_logger.debug("retrieving fine tuning job, id= %s", fine_tuning_job_id) - response = openai_client.fine_tuning.jobs.retrieve( + response = cast(OpenAI, openai_client).fine_tuning.jobs.retrieve( fine_tuning_job_id=fine_tuning_job_id ) - return response + return LiteLLMFineTuningJob(**response.model_dump()) diff --git a/litellm/llms/openrouter/chat/transformation.py b/litellm/llms/openrouter/chat/transformation.py index 77f402a1317..e3f9d5c3dd0 100644 --- a/litellm/llms/openrouter/chat/transformation.py +++ b/litellm/llms/openrouter/chat/transformation.py @@ -120,6 +120,7 @@ class OpenRouterChatCompletionStreamingHandler(BaseModelResponseIterator): id=chunk["id"], object="chat.completion.chunk", created=chunk["created"], + usage=chunk.get("usage"), model=chunk["model"], choices=new_choices, ) diff --git a/litellm/llms/vertex_ai/fine_tuning/handler.py b/litellm/llms/vertex_ai/fine_tuning/handler.py index 7ea8527fd41..4d7f8cec02d 100644 --- a/litellm/llms/vertex_ai/fine_tuning/handler.py +++ b/litellm/llms/vertex_ai/fine_tuning/handler.py @@ -1,10 +1,9 @@ import json import traceback from datetime import datetime -from typing import Literal, Optional, Union +from typing import Any, Coroutine, Literal, Optional, Union import httpx -from openai.types.fine_tuning.fine_tuning_job import FineTuningJob import litellm from litellm._logging import verbose_logger @@ -20,6 +19,7 @@ from litellm.types.llms.vertex_ai import ( ResponseSupervisedTuningSpec, ResponseTuningJob, ) +from litellm.types.utils import LiteLLMFineTuningJob class VertexFineTuningAPI(VertexLLM): @@ -113,7 +113,7 @@ class VertexFineTuningAPI(VertexLLM): def convert_vertex_response_to_open_ai_response( self, response: ResponseTuningJob - ) -> FineTuningJob: + ) -> LiteLLMFineTuningJob: status: Literal[ "validating_files", "queued", "running", "succeeded", "failed", "cancelled" ] = "queued" @@ -134,7 +134,7 @@ class VertexFineTuningAPI(VertexLLM): response.get("supervisedTuningSpec", None) or {} ) training_uri: str = _supervisedTuningSpec.get("trainingDatasetUri", "") or "" - return FineTuningJob( + return LiteLLMFineTuningJob( id=response.get("name", "") or "", created_at=created_at, fine_tuned_model=response.get("tunedModelDisplayName", ""), @@ -226,7 +226,7 @@ class VertexFineTuningAPI(VertexLLM): timeout: Union[float, httpx.Timeout], kwargs: Optional[dict] = None, original_hyperparameters: Optional[dict] = {}, - ): + ) -> Union[LiteLLMFineTuningJob, Coroutine[Any, Any, LiteLLMFineTuningJob]]: verbose_logger.debug( "creating fine tuning job, args= %s", create_fine_tuning_job_data ) diff --git a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py index cd67be3545a..902f8257248 100644 --- a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py +++ b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py @@ -455,9 +455,6 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): and value ): optional_params["tools"] = self._map_function(value=value) - optional_params["litellm_param_is_function_call"] = ( - True if param == "functions" else False - ) elif param == "tool_choice" and ( isinstance(value, str) or isinstance(value, dict) ): @@ -880,8 +877,14 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): else: return "stop" - def _process_candidates(self, _candidates, model_response, litellm_params): + def _process_candidates( + self, _candidates, model_response, standard_optional_params: dict + ): """Helper method to process candidates and extract metadata""" + from litellm.litellm_core_utils.prompt_templates.common_utils import ( + is_function_call, + ) + grounding_metadata: List[dict] = [] safety_ratings: List = [] citation_metadata: List = [] @@ -918,9 +921,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): functions, tools = self._transform_parts( parts=candidate["content"]["parts"], index=candidate.get("index", idx), - is_function_call=litellm_params.get( - "litellm_param_is_function_call" - ), + is_function_call=is_function_call(standard_optional_params), ) if "logprobsResult" in candidate: @@ -1019,7 +1020,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): safety_ratings, citation_metadata, ) = self._process_candidates( - _candidates, model_response, litellm_params + _candidates, model_response, logging_obj.optional_params ) usage = self._calculate_usage(completion_response=completion_response) diff --git a/litellm/llms/vertex_ai/image_generation/cost_calculator.py b/litellm/llms/vertex_ai/image_generation/cost_calculator.py index 2ba18c095bd..646c6080a2e 100644 --- a/litellm/llms/vertex_ai/image_generation/cost_calculator.py +++ b/litellm/llms/vertex_ai/image_generation/cost_calculator.py @@ -19,5 +19,7 @@ def cost_calculator( ) output_cost_per_image: float = _model_info.get("output_cost_per_image") or 0.0 - num_images: int = len(image_response.data) + num_images: int = 0 + if image_response.data: + num_images = len(image_response.data) return output_cost_per_image * num_images diff --git a/litellm/main.py b/litellm/main.py index 7cae5acd97b..44611e203f0 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -97,6 +97,7 @@ from litellm.utils import ( get_optional_params_image_gen, get_optional_params_transcription, get_secret, + get_standard_openai_params, mock_completion_streaming_obj, read_config_args, supports_httpx_timeout, @@ -428,6 +429,7 @@ async def acompletion( prompt_id=kwargs.get("prompt_id", None), prompt_variables=kwargs.get("prompt_variables", None), tools=tools, + prompt_label=kwargs.get("prompt_label", None), ) ######################################################### @@ -983,6 +985,7 @@ def completion( # type: ignore # noqa: PLR0915 assistant_continue_message=assistant_continue_message, ) ######## end of unpacking kwargs ########### + standard_openai_params = get_standard_openai_params(params=args) non_default_params = get_non_default_completion_params(kwargs=kwargs) litellm_params = {} # used to prevent unbound var errors ## PROMPT MANAGEMENT HOOKS ## @@ -1001,6 +1004,7 @@ def completion( # type: ignore # noqa: PLR0915 non_default_params=non_default_params, prompt_id=prompt_id, prompt_variables=prompt_variables, + prompt_label=kwargs.get("prompt_label", None), ) try: @@ -1234,10 +1238,13 @@ def completion( # type: ignore # noqa: PLR0915 max_retries=max_retries, timeout=timeout, ) - logging.update_environment_variables( + cast(LiteLLMLoggingObj, logging).update_environment_variables( model=model, user=user, - optional_params=optional_params, + optional_params={ + **standard_openai_params, + **non_default_params, + }, # [IMPORTANT] - using standard_openai_params ensures consistent params logged to langfuse for finetuning / eval datasets. litellm_params=litellm_params, custom_llm_provider=custom_llm_provider, ) @@ -4020,6 +4027,30 @@ def embedding( # noqa: PLR0915 client=client, aembedding=aembedding, ) + elif ( + custom_llm_provider in litellm._custom_providers + ): + custom_handler: Optional[CustomLLM] = None + for item in litellm.custom_provider_map: + if item["provider"] == custom_llm_provider: + custom_handler = item["custom_handler"] + + if custom_handler is None: + raise LiteLLMUnknownProvider( + model=model, custom_llm_provider=custom_llm_provider + ) + + handler_fn = custom_handler.embedding if not aembedding else custom_handler.aembedding + + response = handler_fn( + model=model, + input=input, + logging_obj=logging, + optional_params=optional_params, + model_response=EmbeddingResponse(), + print_verbose=print_verbose, + litellm_params=litellm_params + ) else: raise LiteLLMUnknownProvider( model=model, custom_llm_provider=custom_llm_provider diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 8d7d062b818..ad53729a148 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -4684,6 +4684,58 @@ "deprecation_date": "2025-06-01", "supports_tool_choice": true }, + "claude-opus-4-20250514": { + "max_tokens": 32000, + "max_input_tokens": 200000, + "max_output_tokens": 32000, + "input_cost_per_token": 15e-6, + "output_cost_per_token": 75e-6, + "search_context_cost_per_query": { + "search_context_size_low": 1e-2, + "search_context_size_medium": 1e-2, + "search_context_size_high": 1e-2 + }, + "cache_creation_input_token_cost": 18.75e-6, + "cache_read_input_token_cost": 1.5e-6, + "litellm_provider": "anthropic", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "tool_use_system_prompt_tokens": 159, + "supports_assistant_prefill": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_computer_use": true + }, + "claude-sonnet-4-20250514": { + "max_tokens": 64000, + "max_input_tokens": 200000, + "max_output_tokens": 64000, + "input_cost_per_token": 3e-6, + "output_cost_per_token": 15e-6, + "search_context_cost_per_query": { + "search_context_size_low": 1e-2, + "search_context_size_medium": 1e-2, + "search_context_size_high": 1e-2 + }, + "cache_creation_input_token_cost": 3.75e-6, + "cache_read_input_token_cost": 0.3e-6, + "litellm_provider": "anthropic", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "tool_use_system_prompt_tokens": 159, + "supports_assistant_prefill": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_computer_use": true + }, "claude-3-7-sonnet-latest": { "supports_computer_use": true, "max_tokens": 128000, @@ -6753,6 +6805,58 @@ "supports_reasoning": true, "supports_tool_choice": true }, + "vertex_ai/claude-opus-4@20250514": { + "max_tokens": 32000, + "max_input_tokens": 200000, + "max_output_tokens": 32000, + "input_cost_per_token": 15e-6, + "output_cost_per_token": 75e-6, + "search_context_cost_per_query": { + "search_context_size_low": 1e-2, + "search_context_size_medium": 1e-2, + "search_context_size_high": 1e-2 + }, + "cache_creation_input_token_cost": 18.75e-6, + "cache_read_input_token_cost": 1.5e-6, + "litellm_provider": "vertex_ai-anthropic_models", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "tool_use_system_prompt_tokens": 159, + "supports_assistant_prefill": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_computer_use": true + }, + "vertex_ai/claude-sonnet-4@20250514": { + "max_tokens": 64000, + "max_input_tokens": 200000, + "max_output_tokens": 64000, + "input_cost_per_token": 3e-6, + "output_cost_per_token": 15e-6, + "search_context_cost_per_query": { + "search_context_size_low": 1e-2, + "search_context_size_medium": 1e-2, + "search_context_size_high": 1e-2 + }, + "cache_creation_input_token_cost": 3.75e-6, + "cache_read_input_token_cost": 0.3e-6, + "litellm_provider": "vertex_ai-anthropic_models", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "tool_use_system_prompt_tokens": 159, + "supports_assistant_prefill": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_computer_use": true + }, "vertex_ai/claude-3-haiku": { "max_tokens": 4096, "max_input_tokens": 200000, @@ -9332,6 +9436,58 @@ "supports_pdf_input": true, "supports_tool_choice": true }, + "anthropic.claude-opus-4-20250514-v1:0": { + "max_tokens": 32000, + "max_input_tokens": 200000, + "max_output_tokens": 32000, + "input_cost_per_token": 15e-6, + "output_cost_per_token": 75e-6, + "search_context_cost_per_query": { + "search_context_size_low": 1e-2, + "search_context_size_medium": 1e-2, + "search_context_size_high": 1e-2 + }, + "cache_creation_input_token_cost": 18.75e-6, + "cache_read_input_token_cost": 1.5e-6, + "litellm_provider": "bedrock_converse", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "tool_use_system_prompt_tokens": 159, + "supports_assistant_prefill": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_computer_use": true + }, + "anthropic.claude-sonnet-4-20250514-v1:0": { + "max_tokens": 64000, + "max_input_tokens": 200000, + "max_output_tokens": 64000, + "input_cost_per_token": 3e-6, + "output_cost_per_token": 15e-6, + "search_context_cost_per_query": { + "search_context_size_low": 1e-2, + "search_context_size_medium": 1e-2, + "search_context_size_high": 1e-2 + }, + "cache_creation_input_token_cost": 3.75e-6, + "cache_read_input_token_cost": 0.3e-6, + "litellm_provider": "bedrock_converse", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "tool_use_system_prompt_tokens": 159, + "supports_assistant_prefill": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_computer_use": true + }, "anthropic.claude-3-7-sonnet-20250219-v1:0": { "supports_computer_use": true, "max_tokens": 8192, @@ -9482,6 +9638,58 @@ "supports_tool_choice": true, "supports_reasoning": true }, + "us.anthropic.claude-opus-4-20250514-v1:0": { + "max_tokens": 32000, + "max_input_tokens": 200000, + "max_output_tokens": 32000, + "input_cost_per_token": 15e-6, + "output_cost_per_token": 75e-6, + "search_context_cost_per_query": { + "search_context_size_low": 1e-2, + "search_context_size_medium": 1e-2, + "search_context_size_high": 1e-2 + }, + "cache_creation_input_token_cost": 18.75e-6, + "cache_read_input_token_cost": 1.5e-6, + "litellm_provider": "bedrock_converse", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "tool_use_system_prompt_tokens": 159, + "supports_assistant_prefill": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_computer_use": true + }, + "us.anthropic.claude-sonnet-4-20250514-v1:0": { + "max_tokens": 64000, + "max_input_tokens": 200000, + "max_output_tokens": 64000, + "input_cost_per_token": 3e-6, + "output_cost_per_token": 15e-6, + "search_context_cost_per_query": { + "search_context_size_low": 1e-2, + "search_context_size_medium": 1e-2, + "search_context_size_high": 1e-2 + }, + "cache_creation_input_token_cost": 3.75e-6, + "cache_read_input_token_cost": 0.3e-6, + "litellm_provider": "bedrock_converse", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "tool_use_system_prompt_tokens": 159, + "supports_assistant_prefill": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_computer_use": true + }, "us.anthropic.claude-3-haiku-20240307-v1:0": { "max_tokens": 4096, "max_input_tokens": 200000, @@ -9603,6 +9811,58 @@ "supports_pdf_input": true, "supports_tool_choice": true }, + "eu.anthropic.claude-opus-4-20250514-v1:0": { + "max_tokens": 32000, + "max_input_tokens": 200000, + "max_output_tokens": 32000, + "input_cost_per_token": 15e-6, + "output_cost_per_token": 75e-6, + "search_context_cost_per_query": { + "search_context_size_low": 1e-2, + "search_context_size_medium": 1e-2, + "search_context_size_high": 1e-2 + }, + "cache_creation_input_token_cost": 18.75e-6, + "cache_read_input_token_cost": 1.5e-6, + "litellm_provider": "bedrock_converse", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "tool_use_system_prompt_tokens": 159, + "supports_assistant_prefill": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_computer_use": true + }, + "eu.anthropic.claude-sonnet-4-20250514-v1:0": { + "max_tokens": 64000, + "max_input_tokens": 200000, + "max_output_tokens": 64000, + "input_cost_per_token": 3e-6, + "output_cost_per_token": 15e-6, + "search_context_cost_per_query": { + "search_context_size_low": 1e-2, + "search_context_size_medium": 1e-2, + "search_context_size_high": 1e-2 + }, + "cache_creation_input_token_cost": 3.75e-6, + "cache_read_input_token_cost": 0.3e-6, + "litellm_provider": "bedrock_converse", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "tool_use_system_prompt_tokens": 159, + "supports_assistant_prefill": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_computer_use": true + }, "eu.anthropic.claude-3-5-haiku-20241022-v1:0": { "max_tokens": 8192, "max_input_tokens": 200000, @@ -12297,7 +12557,7 @@ "max_output_tokens": 128000, "input_cost_per_token": 0.0000025, "input_dbu_cost_per_token": 0.00003571, - "output_cost_per_token": 0.00017857, + "output_cost_per_token": 0.000017857, "output_db_cost_per_token": 0.000214286, "litellm_provider": "databricks", "mode": "chat", @@ -12350,6 +12610,20 @@ "metadata": {"notes": "Input/output cost per token is dbu cost * $0.070, based on databricks Llama 3.1 70B conversion. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation."}, "supports_tool_choice": true }, + "databricks/databricks-llama-4-maverick": { + "max_tokens": 128000, + "max_input_tokens": 128000, + "max_output_tokens": 128000, + "input_cost_per_token": 0.000005, + "input_dbu_cost_per_token": 0.00007143, + "output_cost_per_token": 0.000015, + "output_dbu_cost_per_token": 0.00021429, + "litellm_provider": "databricks", + "mode": "chat", + "source": "https://www.databricks.com/product/pricing/foundation-model-serving", + "metadata": {"notes": "Databricks documentation now provides both DBU costs (_dbu_cost_per_token) and dollar costs(_cost_per_token)."}, + "supports_tool_choice": true + }, "databricks/databricks-dbrx-instruct": { "max_tokens": 32768, "max_input_tokens": 32768, @@ -12976,4 +13250,4 @@ "litellm_provider": "featherless_ai", "mode": "chat" } -} \ No newline at end of file +} diff --git a/litellm/proxy/_new_secret_config.yaml b/litellm/proxy/_new_secret_config.yaml index 74f81459b88..a255f32f5e3 100644 --- a/litellm/proxy/_new_secret_config.yaml +++ b/litellm/proxy/_new_secret_config.yaml @@ -1,17 +1,17 @@ model_list: - - model_name: "gemini-2.0-flash" + - model_name: "gemini-2.0-flash-gemini" litellm_params: - model: gemini/gemini-2.0-flash-live-001 - - model_name: "gpt-4o-mini-openai" + model: gemini/gemini-2.0-flash + - model_name: "gpt-4.1-openai" litellm_params: - model: gpt-4o-mini + model: gpt-4.1 api_key: os.environ/OPENAI_API_KEY model_info: access_groups: ["default-openai-models"] - model_name: "gpt-4o-realtime-preview" litellm_params: model: gpt-4o-realtime-preview-2024-10-01 - api_key: os.environ/OPENAI_API_KEY + api_key: os.environ/OPENAI_API_KEY_2 - model_name: "bedrock-nova" litellm_params: model: us.amazon.nova-pro-v1:0 @@ -71,6 +71,16 @@ model_list: model: mistral/* api_key: os.environ/MISTRAL_API_KEY access_groups: ["beta-models"] + - model_name: my-langfuse-model + litellm_params: + model: langfuse/gpt-3.5-turbo + prompt_id: "jokes" + prompt_label: "latest" + api_key: os.environ/OPENAI_API_KEY -litellm_settings: - cache: true \ No newline at end of file +general_settings: + store_model_in_db: true + store_prompts_in_spend_logs: true + custom_auth: custom_auth_auto.user_api_key_auth + custom_auth_settings: + mode: "auto" \ No newline at end of file diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 088d1dd7d4c..c1feb3dc330 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -23,6 +23,8 @@ from litellm.types.utils import ( EmbeddingResponse, GenericBudgetConfigType, ImageResponse, + LiteLLMBatch, + LiteLLMFineTuningJob, LiteLLMPydanticObjectBase, ModelResponse, ProviderField, @@ -938,6 +940,7 @@ class NewCustomerRequest(BudgetNewRequest): alias: Optional[str] = None # human-friendly alias blocked: bool = False # allow/disallow requests for this end-user budget_id: Optional[str] = None # give either a budget_id or max_budget + spend: Optional[float] = None allowed_model_region: Optional[ AllowedModelRegion ] = None # require all user requests to use models in this specific region @@ -1222,6 +1225,7 @@ class TeamRequest(LiteLLMPydanticObjectBase): class LiteLLM_BudgetTable(LiteLLMPydanticObjectBase): """Represents user-controllable params for a LiteLLM_BudgetTable record""" + budget_id: Optional[str] = None soft_budget: Optional[float] = None max_budget: Optional[float] = None max_parallel_requests: Optional[int] = None @@ -2879,3 +2883,13 @@ class LiteLLM_ManagedFileTable(LiteLLMPydanticObjectBase): unified_file_id: str file_object: OpenAIFileObject model_mappings: Dict[str, str] + flat_model_file_ids: List[str] + created_by: Optional[str] + updated_by: Optional[str] + + +class LiteLLM_ManagedObjectTable(LiteLLMPydanticObjectBase): + unified_object_id: str + model_object_id: str + file_purpose: Literal["batch", "fine-tune"] + file_object: Union[LiteLLMBatch, LiteLLMFineTuningJob] diff --git a/litellm/proxy/anthropic_endpoints/endpoints.py b/litellm/proxy/anthropic_endpoints/endpoints.py index 78078b93f81..a84c1e84ab0 100644 --- a/litellm/proxy/anthropic_endpoints/endpoints.py +++ b/litellm/proxy/anthropic_endpoints/endpoints.py @@ -12,6 +12,8 @@ from fastapi.responses import StreamingResponse import litellm from litellm._logging import verbose_proxy_logger +from litellm.constants import STREAM_SSE_DATA_PREFIX +from litellm.litellm_core_utils.safe_json_dumps import safe_dumps from litellm.proxy._types import * from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing @@ -22,6 +24,24 @@ from litellm.proxy.utils import ProxyLogging router = APIRouter() +def return_anthropic_chunk(chunk: Any) -> str: + """ + Helper function to format streaming chunks for Anthropic API format + + Args: + chunk: A string or dictionary to be returned in SSE format + + Returns: + str: A properly formatted SSE chunk string + """ + if isinstance(chunk, dict): + # Use safe_dumps for proper JSON serialization with circular reference detection + chunk_str = safe_dumps(chunk) + return f"{STREAM_SSE_DATA_PREFIX}{chunk_str}\n\n" + else: + return chunk + + async def async_data_generator_anthropic( response, user_api_key_dict: UserAPIKeyAuth, @@ -40,7 +60,8 @@ async def async_data_generator_anthropic( user_api_key_dict=user_api_key_dict, response=chunk ) - yield chunk + # Format chunk using helper function + yield return_anthropic_chunk(chunk) except Exception as e: verbose_proxy_logger.exception( "litellm.proxy.proxy_server.async_data_generator(): Exception occured - {}".format( @@ -69,7 +90,7 @@ async def async_data_generator_anthropic( code=getattr(e, "status_code", 500), ) error_returned = json.dumps({"error": proxy_exception.to_dict()}) - yield f"data: {error_returned}\n\n" + yield f"{STREAM_SSE_DATA_PREFIX}{error_returned}\n\n" @router.post( diff --git a/litellm/proxy/auth/auth_checks.py b/litellm/proxy/auth/auth_checks.py index 3c759e839ec..1ac694f9475 100644 --- a/litellm/proxy/auth/auth_checks.py +++ b/litellm/proxy/auth/auth_checks.py @@ -670,15 +670,20 @@ class UserObjectCache: - update user object in cache """ if isinstance(user_object, LiteLLM_UserTable): - user_object = user_object.model_dump() - for k, v in user_object.items(): - if isinstance(v, datetime): - user_object[k] = v.isoformat() - await self.user_api_key_cache.async_set_cache(key=user_id, value=user_object) + user_object_dict = user_object.model_dump() + else: + user_object_dict = user_object + + for k, v in user_object_dict.items(): + if isinstance(v, datetime): + user_object_dict[k] = v.isoformat() + await self.user_api_key_cache.async_set_cache( + key=user_id, value=user_object_dict + ) if self.internal_usage_cache is not None: await self.internal_usage_cache.async_set_cache( key=user_id, - value=user_object, + value=user_object_dict, litellm_parent_otel_span=litellm_parent_otel_span, ) diff --git a/litellm/proxy/auth/user_api_key_auth.py b/litellm/proxy/auth/user_api_key_auth.py index 91536e08cd7..6007700ac2d 100644 --- a/litellm/proxy/auth/user_api_key_auth.py +++ b/litellm/proxy/auth/user_api_key_auth.py @@ -55,6 +55,16 @@ from litellm.proxy.utils import PrismaClient, ProxyLogging from litellm.secret_managers.main import get_secret_bool from litellm.types.services import ServiceTypes +try: + from litellm_enterprise.proxy.auth.user_api_key_auth import ( + enterprise_custom_auth as _enterprise_custom_auth, + ) + + enterprise_custom_auth: Optional[Callable] = _enterprise_custom_auth +except ImportError as e: + verbose_proxy_logger.debug(f"Error in enterprise custom auth: {e}") + enterprise_custom_auth = None + user_api_key_service_logger_obj = ServiceLogging() # used for tracking latency on OTEL custom_litellm_key_header = APIKeyHeader( @@ -346,7 +356,13 @@ async def _user_api_key_auth_builder( # noqa: PLR0915 ) ### USER-DEFINED AUTH FUNCTION ### - if user_custom_auth is not None: + if enterprise_custom_auth is not None: + response = await enterprise_custom_auth( + request=request, api_key=api_key, user_custom_auth=user_custom_auth + ) + if response is not None: + return UserAPIKeyAuth.model_validate(response) + elif user_custom_auth is not None: response = await user_custom_auth(request=request, api_key=api_key) # type: ignore return UserAPIKeyAuth.model_validate(response) diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index 2fd56af0b91..678dd2693a8 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -117,6 +117,10 @@ class ProxyBaseLLMRequestProcessing: "acreate_batch", "aretrieve_batch", "afile_content", + "acreate_fine_tuning_job", + "acancel_fine_tuning_job", + "alist_fine_tuning_jobs", + "aretrieve_fine_tuning_job", ], version: Optional[str] = None, user_model: Optional[str] = None, diff --git a/litellm/proxy/custom_auth_auto.py b/litellm/proxy/custom_auth_auto.py new file mode 100644 index 00000000000..df6391021c1 --- /dev/null +++ b/litellm/proxy/custom_auth_auto.py @@ -0,0 +1,18 @@ +""" +Example custom auth function. + +This will allow all keys starting with "my-custom-key" to pass through. +""" +from fastapi import Request + +from litellm.proxy._types import UserAPIKeyAuth + + +async def user_api_key_auth(request: Request, api_key: str) -> UserAPIKeyAuth: + try: + if api_key.startswith("my-custom-key"): + return UserAPIKeyAuth(api_key=api_key) + else: + raise Exception("Invalid API key") + except Exception: + raise Exception("Invalid API key") diff --git a/litellm/proxy/custom_prompt_management.py b/litellm/proxy/custom_prompt_management.py index fc16f4a4903..8cf20da5e92 100644 --- a/litellm/proxy/custom_prompt_management.py +++ b/litellm/proxy/custom_prompt_management.py @@ -15,6 +15,7 @@ class X42PromptManagement(CustomPromptManagement): prompt_id: Optional[str], prompt_variables: Optional[dict], dynamic_callback_params: StandardCallbackDynamicParams, + prompt_label: Optional[str] = None, ) -> Tuple[str, List[AllMessageValues], dict]: """ Returns: diff --git a/litellm/proxy/fine_tuning_endpoints/endpoints.py b/litellm/proxy/fine_tuning_endpoints/endpoints.py index d4c4250b37f..be7f83c65ef 100644 --- a/litellm/proxy/fine_tuning_endpoints/endpoints.py +++ b/litellm/proxy/fine_tuning_endpoints/endpoints.py @@ -6,17 +6,20 @@ ########################################################################## import asyncio -import traceback -from typing import Optional +from typing import Optional, cast -from fastapi import APIRouter, Depends, Request, Response +from fastapi import APIRouter, Depends, HTTPException, Query, Request, Response import litellm from litellm._logging import verbose_proxy_logger from litellm.proxy._types import * from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing +from litellm.proxy.openai_files_endpoints.common_utils import ( + _is_base64_encoded_unified_file_id, +) from litellm.proxy.utils import handle_exception_on_proxy +from litellm.types.utils import LiteLLMFineTuningJob router = APIRouter() @@ -96,8 +99,8 @@ async def create_fine_tuning_job( ``` """ from litellm.proxy.proxy_server import ( - add_litellm_data_to_request, general_settings, + llm_router, premium_user, proxy_config, proxy_logging_obj, @@ -117,25 +120,67 @@ async def create_fine_tuning_job( ) # Include original request and headers in the data - data = await add_litellm_data_to_request( - data=data, + base_llm_response_processor = ProxyBaseLLMRequestProcessing(data=data) + ( + data, + litellm_logging_obj, + ) = await base_llm_response_processor.common_processing_pre_call_logic( request=request, general_settings=general_settings, user_api_key_dict=user_api_key_dict, version=version, + proxy_logging_obj=proxy_logging_obj, proxy_config=proxy_config, + route_type="acreate_fine_tuning_job", ) - # get configs for custom_llm_provider - llm_provider_config = get_fine_tuning_provider_config( - custom_llm_provider=fine_tuning_request.custom_llm_provider, + ## CHECK IF MANAGED FILE ID + unified_file_id: Union[str, Literal[False]] = False + training_file = fine_tuning_request.training_file + response: Optional[LiteLLMFineTuningJob] = None + if training_file: + unified_file_id = _is_base64_encoded_unified_file_id(training_file) + ## IF SO, Route based on that + if unified_file_id: + """ """ + if llm_router is None: + raise HTTPException( + status_code=500, + detail={ + "error": "LLM Router not initialized. Ensure models added to proxy." + }, + ) + + response = cast( + LiteLLMFineTuningJob, await llm_router.acreate_fine_tuning_job(**data) + ) + response.training_file = unified_file_id + response._hidden_params["unified_file_id"] = unified_file_id + ## ELSE, Route based on custom_llm_provider + elif fine_tuning_request.custom_llm_provider: + # get configs for custom_llm_provider + llm_provider_config = get_fine_tuning_provider_config( + custom_llm_provider=fine_tuning_request.custom_llm_provider, + ) + # add llm_provider_config to data + if llm_provider_config is not None: + data.update(llm_provider_config) + + response = await litellm.acreate_fine_tuning_job(**data) + + if response is None: + raise ValueError( + "Invalid request, No litellm managed file id or custom_llm_provider provided." + ) + + ### CALL HOOKS ### - modify outgoing data + _response = await proxy_logging_obj.post_call_success_hook( + data=data, + user_api_key_dict=user_api_key_dict, + response=response, ) - - # add llm_provider_config to data - if llm_provider_config is not None: - data.update(llm_provider_config) - - response = await litellm.acreate_fine_tuning_job(**data) + if _response is not None and isinstance(_response, LiteLLMFineTuningJob): + response = _response ### ALERTING ### asyncio.create_task( @@ -166,12 +211,11 @@ async def create_fine_tuning_job( await proxy_logging_obj.post_call_failure_hook( user_api_key_dict=user_api_key_dict, original_exception=e, request_data=data ) - verbose_proxy_logger.error( + verbose_proxy_logger.exception( "litellm.proxy.proxy_server.create_fine_tuning_job(): Exception occurred - {}".format( str(e) ) ) - verbose_proxy_logger.debug(traceback.format_exc()) raise handle_exception_on_proxy(e) @@ -191,7 +235,7 @@ async def retrieve_fine_tuning_job( request: Request, fastapi_response: Response, fine_tuning_job_id: str, - custom_llm_provider: Literal["openai", "azure"], + custom_llm_provider: Optional[Literal["openai", "azure"]] = None, user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), ): """ @@ -203,41 +247,99 @@ async def retrieve_fine_tuning_job( - `fine_tuning_job_id`: The ID of the fine-tuning job to retrieve. """ from litellm.proxy.proxy_server import ( - add_litellm_data_to_request, general_settings, + llm_router, premium_user, proxy_config, proxy_logging_obj, version, ) - data: dict = {} + data: dict = {"fine_tuning_job_id": fine_tuning_job_id} try: if premium_user is not True: raise ValueError( f"Only premium users can use this endpoint + {CommonProxyErrors.not_premium_user.value}" ) # Include original request and headers in the data - data = await add_litellm_data_to_request( - data=data, + base_llm_response_processor = ProxyBaseLLMRequestProcessing(data=data) + ( + data, + litellm_logging_obj, + ) = await base_llm_response_processor.common_processing_pre_call_logic( request=request, general_settings=general_settings, user_api_key_dict=user_api_key_dict, version=version, + proxy_logging_obj=proxy_logging_obj, proxy_config=proxy_config, + route_type=CallTypes.aretrieve_fine_tuning_job.value, ) - # get configs for custom_llm_provider - llm_provider_config = get_fine_tuning_provider_config( - custom_llm_provider=custom_llm_provider + try: + request_body = await request.json() + except Exception: + request_body = {} + + custom_llm_provider = request_body.get("custom_llm_provider", None) + + ## CHECK IF MANAGED FILE ID + unified_finetuning_job_id: Union[str, Literal[False]] = False + response: Optional[LiteLLMFineTuningJob] = None + if fine_tuning_job_id: + unified_finetuning_job_id = _is_base64_encoded_unified_file_id( + fine_tuning_job_id + ) + if unified_finetuning_job_id: + if llm_router is None: + raise HTTPException( + status_code=500, + detail={ + "error": "LLM Router not initialized. Ensure models added to proxy." + }, + ) + response = cast( + LiteLLMFineTuningJob, + await llm_router.aretrieve_fine_tuning_job( + **data, + ), + ) + response._hidden_params[ + "unified_finetuning_job_id" + ] = unified_finetuning_job_id + elif custom_llm_provider: + # get configs for custom_llm_provider + llm_provider_config = get_fine_tuning_provider_config( + custom_llm_provider=custom_llm_provider + ) + + if llm_provider_config is not None: + data.update(llm_provider_config) + + response = await litellm.aretrieve_fine_tuning_job( + **data, + ) + + if response is None: + raise HTTPException( + status_code=400, + detail="Invalid request, No litellm managed file id or custom_llm_provider provided.", + ) + + ### CALL HOOKS ### - modify outgoing data + _response = await proxy_logging_obj.post_call_success_hook( + data=data, + user_api_key_dict=user_api_key_dict, + response=response, ) + if _response is not None and isinstance(_response, LiteLLMFineTuningJob): + response = _response - if llm_provider_config is not None: - data.update(llm_provider_config) - - response = await litellm.aretrieve_fine_tuning_job( - **data, - fine_tuning_job_id=fine_tuning_job_id, + ### ALERTING ### + asyncio.create_task( + proxy_logging_obj.update_request_status( + litellm_call_id=data.get("litellm_call_id", ""), status="success" + ) ) ### RESPONSE HEADERS ### @@ -263,12 +365,11 @@ async def retrieve_fine_tuning_job( await proxy_logging_obj.post_call_failure_hook( user_api_key_dict=user_api_key_dict, original_exception=e, request_data=data ) - verbose_proxy_logger.error( - "litellm.proxy.proxy_server.list_fine_tuning_jobs(): Exception occurred - {}".format( + verbose_proxy_logger.exception( + "litellm.proxy.proxy_server.retrieve_fine_tuning_job(): Exception occurred - {}".format( str(e) ) ) - verbose_proxy_logger.debug(traceback.format_exc()) raise handle_exception_on_proxy(e) @@ -287,7 +388,11 @@ async def retrieve_fine_tuning_job( async def list_fine_tuning_jobs( request: Request, fastapi_response: Response, - custom_llm_provider: Literal["openai", "azure"], + custom_llm_provider: Optional[Literal["openai", "azure"]] = None, + target_model_names: Optional[str] = Query( + default=None, + description="Comma separated list of model names to filter by. Example: 'gpt-4o,gpt-4o-mini'", + ), after: Optional[str] = None, limit: Optional[int] = None, user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), @@ -302,8 +407,8 @@ async def list_fine_tuning_jobs( - `limit`: Number of fine-tuning jobs to retrieve (default is 20). """ from litellm.proxy.proxy_server import ( - add_litellm_data_to_request, general_settings, + llm_router, premium_user, proxy_config, proxy_logging_obj, @@ -317,28 +422,60 @@ async def list_fine_tuning_jobs( f"Only premium users can use this endpoint + {CommonProxyErrors.not_premium_user.value}" ) # Include original request and headers in the data - data = await add_litellm_data_to_request( - data=data, + base_llm_response_processor = ProxyBaseLLMRequestProcessing(data=data) + ( + data, + litellm_logging_obj, + ) = await base_llm_response_processor.common_processing_pre_call_logic( request=request, general_settings=general_settings, user_api_key_dict=user_api_key_dict, version=version, + proxy_logging_obj=proxy_logging_obj, proxy_config=proxy_config, + route_type=CallTypes.alist_fine_tuning_jobs.value, ) - # get configs for custom_llm_provider - llm_provider_config = get_fine_tuning_provider_config( - custom_llm_provider=custom_llm_provider - ) + response: Optional[Any] = None + if target_model_names and isinstance(target_model_names, str): + target_model_names_list = target_model_names.split(",") + if len(target_model_names_list) != 1: + raise HTTPException( + status_code=400, + detail="target_model_names on list fine-tuning jobs must be a list of one model name. Example: ['gpt-4o']", + ) + ## Use router to list fine-tuning jobs for that model + if llm_router is None: + raise HTTPException( + status_code=500, + detail="LLM Router not initialized. Ensure models added to proxy.", + ) + data["model"] = target_model_names_list[0] + response = await llm_router.alist_fine_tuning_jobs( + **data, + after=after, + limit=limit, + ) + return response + elif custom_llm_provider: + # get configs for custom_llm_provider + llm_provider_config = get_fine_tuning_provider_config( + custom_llm_provider=custom_llm_provider + ) - if llm_provider_config is not None: - data.update(llm_provider_config) + if llm_provider_config is not None: + data.update(llm_provider_config) - response = await litellm.alist_fine_tuning_jobs( - **data, - after=after, - limit=limit, - ) + response = await litellm.alist_fine_tuning_jobs( + **data, + after=after, + limit=limit, + ) + if response is None: + raise HTTPException( + status_code=400, + detail="Invalid request, No litellm managed file id or custom_llm_provider provided.", + ) ### RESPONSE HEADERS ### hidden_params = getattr(response, "_hidden_params", {}) or {} @@ -363,12 +500,11 @@ async def list_fine_tuning_jobs( await proxy_logging_obj.post_call_failure_hook( user_api_key_dict=user_api_key_dict, original_exception=e, request_data=data ) - verbose_proxy_logger.error( + verbose_proxy_logger.exception( "litellm.proxy.proxy_server.list_fine_tuning_jobs(): Exception occurred - {}".format( str(e) ) ) - verbose_proxy_logger.debug(traceback.format_exc()) raise handle_exception_on_proxy(e) @@ -400,45 +536,99 @@ async def cancel_fine_tuning_job( - `fine_tuning_job_id`: The ID of the fine-tuning job to cancel. """ from litellm.proxy.proxy_server import ( - add_litellm_data_to_request, general_settings, + llm_router, premium_user, proxy_config, proxy_logging_obj, version, ) - data: dict = {} + data: dict = {"fine_tuning_job_id": fine_tuning_job_id} try: if premium_user is not True: raise ValueError( f"Only premium users can use this endpoint + {CommonProxyErrors.not_premium_user.value}" ) # Include original request and headers in the data - data = await add_litellm_data_to_request( - data=data, + base_llm_response_processor = ProxyBaseLLMRequestProcessing(data=data) + ( + data, + litellm_logging_obj, + ) = await base_llm_response_processor.common_processing_pre_call_logic( request=request, general_settings=general_settings, user_api_key_dict=user_api_key_dict, version=version, + proxy_logging_obj=proxy_logging_obj, proxy_config=proxy_config, + route_type=CallTypes.acancel_fine_tuning_job.value, ) - request_body = await request.json() + try: + request_body = await request.json() + except Exception: + request_body = {} custom_llm_provider = request_body.get("custom_llm_provider", None) - # get configs for custom_llm_provider - llm_provider_config = get_fine_tuning_provider_config( - custom_llm_provider=custom_llm_provider + ## CHECK IF MANAGED FILE ID + unified_finetuning_job_id: Union[str, Literal[False]] = False + response: Optional[LiteLLMFineTuningJob] = None + if fine_tuning_job_id: + unified_finetuning_job_id = _is_base64_encoded_unified_file_id( + fine_tuning_job_id + ) + if unified_finetuning_job_id: + if llm_router is None: + raise HTTPException( + status_code=500, + detail={ + "error": "LLM Router not initialized. Ensure models added to proxy." + }, + ) + response = cast( + LiteLLMFineTuningJob, + await llm_router.acancel_fine_tuning_job( + **data, + ), + ) + response._hidden_params[ + "unified_finetuning_job_id" + ] = unified_finetuning_job_id + else: + # get configs for custom_llm_provider + llm_provider_config = get_fine_tuning_provider_config( + custom_llm_provider=custom_llm_provider + ) + + if llm_provider_config is not None: + data.update(llm_provider_config) + + response = await litellm.acancel_fine_tuning_job( + **data, + ) + + if response is None: + raise HTTPException( + status_code=400, + detail="Invalid request, No litellm managed file id or custom_llm_provider provided.", + ) + + ### CALL HOOKS ### - modify outgoing data + _response = await proxy_logging_obj.post_call_success_hook( + data=data, + user_api_key_dict=user_api_key_dict, + response=response, ) + if _response is not None and isinstance(_response, LiteLLMFineTuningJob): + response = _response - if llm_provider_config is not None: - data.update(llm_provider_config) - - response = await litellm.acancel_fine_tuning_job( - **data, - fine_tuning_job_id=fine_tuning_job_id, + ### ALERTING ### + asyncio.create_task( + proxy_logging_obj.update_request_status( + litellm_call_id=data.get("litellm_call_id", ""), status="success" + ) ) ### RESPONSE HEADERS ### @@ -464,10 +654,9 @@ async def cancel_fine_tuning_job( await proxy_logging_obj.post_call_failure_hook( user_api_key_dict=user_api_key_dict, original_exception=e, request_data=data ) - verbose_proxy_logger.error( - "litellm.proxy.proxy_server.list_fine_tuning_jobs(): Exception occurred - {}".format( + verbose_proxy_logger.exception( + "litellm.proxy.proxy_server.cancel_fine_tuning_job(): Exception occurred - {}".format( str(e) ) ) - verbose_proxy_logger.debug(traceback.format_exc()) raise handle_exception_on_proxy(e) diff --git a/litellm/proxy/management_endpoints/customer_endpoints.py b/litellm/proxy/management_endpoints/customer_endpoints.py index 1f6f846bc77..b75e3644e1c 100644 --- a/litellm/proxy/management_endpoints/customer_endpoints.py +++ b/litellm/proxy/management_endpoints/customer_endpoints.py @@ -185,6 +185,7 @@ async def new_end_user( - model_max_budget: Optional[dict] - [Not Implemented Yet] Specify max budget for a given model. Example: {"openai/gpt-4o-mini": {"max_budget": 100.0, "budget_duration": "1d"}} - max_parallel_requests: Optional[int] - [Not Implemented Yet] Specify max parallel requests for a given customer. - soft_budget: Optional[float] - [Not Implemented Yet] Get alerts when customer crosses given budget, doesn't block requests. + - spend: Optional[float] - Specify initial spend for a given customer. - Allow specifying allowed regions @@ -424,13 +425,65 @@ async def update_end_user( ): # models default to [], spend defaults to 0, we should not reset these values non_default_values[k] = v - ## ADD USER, IF NEW ## + ## Get end user table data ## + end_user_table_data = await prisma_client.db.litellm_endusertable.find_first( + where={"user_id": data.user_id}, include={"litellm_budget_table": True} + ) + + if end_user_table_data is None: + raise HTTPException( + status_code=400, + detail={ + "error": "End User Id={} does not exist in db".format(data.user_id) + }, + ) + + end_user_table_data_typed = LiteLLM_EndUserTable( + **end_user_table_data.model_dump() + ) + + ## Get budget table data ## + end_user_budget_table = end_user_table_data_typed.litellm_budget_table + + ## Get all params for budget table ## + budget_table_data = {} + update_end_user_table_data = {} + for k, v in non_default_values.items(): + if k in LiteLLM_BudgetTable.model_fields.keys(): + budget_table_data[k] = v + + if k in LiteLLM_EndUserTable.model_fields.keys(): + update_end_user_table_data[k] = v + + ## Check if budget id is set ## + if budget_table_data: + if end_user_budget_table is None: + ## Create new budget ## + budget_table_data_record = ( + await prisma_client.db.litellm_budgettable.create( + data=budget_table_data, include={"litellm_endusertable": True} + ) + ) + + update_end_user_table_data[ + "budget_id" + ] = budget_table_data_record.budget_id + else: + ## Update existing budget ## + budget_table_data_record = ( + await prisma_client.db.litellm_budgettable.update( + where={"budget_id": end_user_budget_table.budget_id}, + data=budget_table_data, + ) + ) + + ## Update user table, with update params + new budget id (if set) ## verbose_proxy_logger.debug("/customer/update: Received data = %s", data) if data.user_id is not None and len(data.user_id) > 0: - non_default_values["user_id"] = data.user_id # type: ignore + update_end_user_table_data["user_id"] = data.user_id # type: ignore verbose_proxy_logger.debug("In update customer, user_id condition block.") response = await prisma_client.db.litellm_endusertable.update( - where={"user_id": data.user_id}, data=non_default_values # type: ignore + where={"user_id": data.user_id}, data=update_end_user_table_data, include={"litellm_budget_table": True} # type: ignore ) if response is None: raise ValueError( @@ -444,13 +497,13 @@ async def update_end_user( raise ValueError(f"user_id is required, passed user_id = {data.user_id}") # update based on remaining passed in values + except Exception as e: - verbose_proxy_logger.error( + verbose_proxy_logger.exception( "litellm.proxy.proxy_server.update_end_user(): Exception occured - {}".format( str(e) ) ) - verbose_proxy_logger.debug(traceback.format_exc()) if isinstance(e, HTTPException): raise ProxyException( message=getattr(e, "detail", f"Internal Server Error({str(e)})"), diff --git a/litellm/proxy/management_endpoints/team_endpoints.py b/litellm/proxy/management_endpoints/team_endpoints.py index f2ea89dd318..60f05902522 100644 --- a/litellm/proxy/management_endpoints/team_endpoints.py +++ b/litellm/proxy/management_endpoints/team_endpoints.py @@ -1110,7 +1110,7 @@ async def team_member_update( Update team member budgets and team member role """ - from litellm.proxy.proxy_server import prisma_client + from litellm.proxy.proxy_server import prisma_client, premium_user if prisma_client is None: raise HTTPException(status_code=500, detail={"error": "No db connected"}) @@ -1118,6 +1118,13 @@ async def team_member_update( if data.team_id is None: raise HTTPException(status_code=400, detail={"error": "No team id passed in"}) + + if data.role == "admin" and not premium_user: + # exactly the same text your proxy throws for add: + raise HTTPException( + status_code=400, + detail="Assigning team admins is a premium feature. You must be a LiteLLM Enterprise user to use this feature. If you have a license please set `LITELLM_LICENSE` in your env. Get a 7 day trial key here: https://www.litellm.ai/#trial. Pricing: https://www.litellm.ai/#pricing" + ) if data.user_id is None and data.user_email is None: raise HTTPException( status_code=400, diff --git a/litellm/proxy/openai_files_endpoints/files_endpoints.py b/litellm/proxy/openai_files_endpoints/files_endpoints.py index 3c2c3d80dcb..79345021490 100644 --- a/litellm/proxy/openai_files_endpoints/files_endpoints.py +++ b/litellm/proxy/openai_files_endpoints/files_endpoints.py @@ -179,6 +179,7 @@ async def route_create_file( create_file_request=_create_file_request, target_model_names_list=target_model_names_list, litellm_parent_otel_span=user_api_key_dict.parent_otel_span, + user_api_key_dict=user_api_key_dict, ) else: # get configs for custom_llm_provider @@ -869,6 +870,7 @@ async def list_files( fastapi_response: Response, user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), provider: Optional[str] = None, + target_model_names: Optional[str] = None, purpose: Optional[str] = None, ): """ @@ -885,8 +887,8 @@ async def list_files( ``` """ from litellm.proxy.proxy_server import ( - add_litellm_data_to_request, general_settings, + llm_router, proxy_config, proxy_logging_obj, version, @@ -894,24 +896,62 @@ async def list_files( data: Dict = {} try: - custom_llm_provider = ( - provider - or await get_custom_llm_provider_from_request_body(request=request) - or "openai" - ) # Include original request and headers in the data - data = await add_litellm_data_to_request( - data=data, + base_llm_response_processor = ProxyBaseLLMRequestProcessing(data=data) + ( + data, + litellm_logging_obj, + ) = await base_llm_response_processor.common_processing_pre_call_logic( request=request, general_settings=general_settings, user_api_key_dict=user_api_key_dict, version=version, + proxy_logging_obj=proxy_logging_obj, proxy_config=proxy_config, + route_type=CallTypes.alist_fine_tuning_jobs.value, ) - response = await litellm.afile_list( - custom_llm_provider=custom_llm_provider, purpose=purpose, **data # type: ignore + response: Optional[Any] = None + if target_model_names and isinstance(target_model_names, str): + target_model_names_list = target_model_names.split(",") + if len(target_model_names_list) != 1: + raise HTTPException( + status_code=400, + detail="target_model_names on list files must be a list of one model name. Example: ['gpt-4o']", + ) + ## Use router to list fine-tuning jobs for that model + if llm_router is None: + raise HTTPException( + status_code=500, + detail="LLM Router not initialized. Ensure models added to proxy.", + ) + data["model"] = target_model_names_list[0] + response = await llm_router.afile_list( + **data, + ) + else: + custom_llm_provider = ( + provider + or await get_custom_llm_provider_from_request_body(request=request) + or "openai" + ) + + response = await litellm.afile_list( + custom_llm_provider=custom_llm_provider, purpose=purpose, **data # type: ignore + ) + + if response is None: + raise HTTPException( + status_code=500, + detail="Either 'provider' or 'target_model_names' must be provided e.g. `?target_model_names=gpt-4o`", + ) + + ## POST CALL HOOKS ### + _response = await proxy_logging_obj.post_call_success_hook( + data=data, user_api_key_dict=user_api_key_dict, response=response ) + if _response is not None and isinstance(_response, OpenAIFileObject): + response = _response ### ALERTING ### asyncio.create_task( diff --git a/litellm/proxy/proxy_cli.py b/litellm/proxy/proxy_cli.py index 4c022991f11..0a9ff2f064b 100644 --- a/litellm/proxy/proxy_cli.py +++ b/litellm/proxy/proxy_cli.py @@ -659,7 +659,7 @@ def run_server( # noqa: PLR0915 **key_management_settings ) database_url = general_settings.get("database_url", None) - if database_url is None: + if database_url is None and os.getenv("DATABASE_URL") is None: # Check if all required variables are provided database_host = os.getenv("DATABASE_HOST") database_username = os.getenv("DATABASE_USERNAME") diff --git a/litellm/proxy/proxy_config.yaml b/litellm/proxy/proxy_config.yaml index ae0c4b2de48..2e37fbe9776 100644 --- a/litellm/proxy/proxy_config.yaml +++ b/litellm/proxy/proxy_config.yaml @@ -2,25 +2,10 @@ model_list: - model_name: openai/gpt-4o litellm_params: model: openai/gpt-4o - api_key: any_key - api_base: https://exampleopenaiendpoint-production.up.railway.app/ + api_key: a + api_base: hi general_settings: store_prompts_in_spend_logs: true - - - -guardrails: - - guardrail_name: "custom-pre-guard" - litellm_params: - guardrail: custom_guardrail.myCustomGuardrail # 👈 Key change - mode: "pre_call" # runs async_pre_call_hook - - guardrail_name: "custom-during-guard" - litellm_params: - guardrail: custom_guardrail.myCustomGuardrail - mode: "during_call" # runs async_moderation_hook - - guardrail_name: "custom-post-guard" - litellm_params: - guardrail: custom_guardrail.myCustomGuardrail - mode: "post_call" # runs async_post_call_success_hook \ No newline at end of file + alerting: ["slack"] \ No newline at end of file diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 1c457baf2f9..c132b12ac6c 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -395,10 +395,12 @@ except Exception: # Import enterprise routes try: from litellm_enterprise.proxy.enterprise_routes import router as _enterprise_router + from litellm_enterprise.proxy.proxy_server import EnterpriseProxyConfig enterprise_router = _enterprise_router + enterprise_proxy_config: Optional[EnterpriseProxyConfig] = EnterpriseProxyConfig() except ImportError: - pass + enterprise_proxy_config = None ################### server_root_path = os.getenv("SERVER_ROOT_PATH", "") @@ -1863,6 +1865,9 @@ class ProxyConfig: value=custom_sso, config_file_path=config_file_path ) + if enterprise_proxy_config is not None: + await enterprise_proxy_config.load_enterprise_config(general_settings) + ## pass through endpoints if general_settings.get("pass_through_endpoints", None) is not None: await initialize_pass_through_endpoints( diff --git a/litellm/proxy/schema.prisma b/litellm/proxy/schema.prisma index 1d6f3b52118..58064abd1dc 100644 --- a/litellm/proxy/schema.prisma +++ b/litellm/proxy/schema.prisma @@ -453,13 +453,30 @@ model LiteLLM_ManagedFileTable { id String @id @default(uuid()) unified_file_id String @unique // The base64 encoded unified file ID file_object Json // Stores the OpenAIFileObject - model_mappings Json // Stores the mapping of model_id -> provider_file_id + model_mappings Json + flat_model_file_ids String[] @default([]) // Flat list of model file id's - for faster querying of model id -> unified file id created_at DateTime @default(now()) + created_by String? updated_at DateTime @updatedAt + updated_by String? @@index([unified_file_id]) } +model LiteLLM_ManagedObjectTable { // for batches or finetuning jobs which use the + id String @id @default(uuid()) + unified_object_id String @unique // The base64 encoded unified file ID + model_object_id String @unique // the id returned by the backend API provider + file_object Json // Stores the OpenAIFileObject + file_purpose String // either 'batch' or 'fine-tune' + created_at DateTime @default(now()) + created_by String? + updated_at DateTime @updatedAt + updated_by String? + + @@index([unified_object_id]) + @@index([model_object_id]) +} model LiteLLM_ManagedVectorStoresTable { vector_store_id String @id diff --git a/litellm/responses/litellm_completion_transformation/transformation.py b/litellm/responses/litellm_completion_transformation/transformation.py index 5812daad462..baffca3ac68 100644 --- a/litellm/responses/litellm_completion_transformation/transformation.py +++ b/litellm/responses/litellm_completion_transformation/transformation.py @@ -457,8 +457,8 @@ class LiteLLMCompletionResponsesConfig: function=ChatCompletionToolParamFunctionChunk( name=tool["name"], description=tool.get("description") or "", - parameters=tool.get("parameters", {}), - strict=tool.get("strict", False), + parameters=dict(tool.get("parameters", {}) or {}), + strict=tool.get("strict", False) or False, ), ) ) diff --git a/litellm/responses/streaming_iterator.py b/litellm/responses/streaming_iterator.py index a111fbec094..e9e41789f09 100644 --- a/litellm/responses/streaming_iterator.py +++ b/litellm/responses/streaming_iterator.py @@ -44,7 +44,7 @@ class BaseResponsesAPIStreamingIterator: self.responses_api_provider_config = responses_api_provider_config self.completed_response: Optional[ResponsesAPIStreamingResponse] = None self.start_time = datetime.now() - + # set request kwargs self.litellm_metadata = litellm_metadata self.custom_llm_provider = custom_llm_provider @@ -330,7 +330,8 @@ class MockResponsesAPIStreamingIterator(BaseResponsesAPIStreamingIterator): def _collect_text(self, resp: ResponsesAPIResponse) -> str: out = "" for out_item in resp.output: - if out_item.type == "message": + item_type = getattr(out_item, "type", None) + if item_type == "message": for c in getattr(out_item, "content", []): out += c.text return out diff --git a/litellm/router.py b/litellm/router.py index 38f187d9a8b..0dd52234000 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -752,6 +752,21 @@ class Router: self._arealtime = self.factory_function( litellm._arealtime, call_type="_arealtime" ) + self.acreate_fine_tuning_job = self.factory_function( + litellm.acreate_fine_tuning_job, call_type="acreate_fine_tuning_job" + ) + self.acancel_fine_tuning_job = self.factory_function( + litellm.acancel_fine_tuning_job, call_type="acancel_fine_tuning_job" + ) + self.alist_fine_tuning_jobs = self.factory_function( + litellm.alist_fine_tuning_jobs, call_type="alist_fine_tuning_jobs" + ) + self.aretrieve_fine_tuning_job = self.factory_function( + litellm.aretrieve_fine_tuning_job, call_type="aretrieve_fine_tuning_job" + ) + self.afile_list = self.factory_function( + litellm.afile_list, call_type="alist_files" + ) def validate_fallbacks(self, fallback_param: Optional[List]): """ @@ -1697,9 +1712,13 @@ class Router: specific_deployment=kwargs.pop("specific_deployment", None), ) - litellm_model = prompt_management_deployment["litellm_params"].get( - "model", None + self._update_kwargs_with_deployment( + deployment=prompt_management_deployment, kwargs=kwargs ) + data = prompt_management_deployment["litellm_params"].copy() + + litellm_model = data.get("model", None) + prompt_id = kwargs.get("prompt_id") or prompt_management_deployment[ "litellm_params" ].get("prompt_id", None) @@ -1708,6 +1727,9 @@ class Router: ) or prompt_management_deployment["litellm_params"].get( "prompt_variables", None ) + prompt_label = kwargs.get("prompt_label", None) or prompt_management_deployment[ + "litellm_params" + ].get("prompt_label", None) if prompt_id is None or not isinstance(prompt_id, str): raise ValueError( @@ -1728,14 +1750,16 @@ class Router: non_default_params=get_non_default_completion_params(kwargs=kwargs), prompt_id=prompt_id, prompt_variables=prompt_variables, + prompt_label=prompt_label, ) - kwargs = {**kwargs, **optional_params} + kwargs = {**data, **kwargs, **optional_params} kwargs["model"] = model kwargs["messages"] = messages kwargs["litellm_logging_obj"] = litellm_logging_object kwargs["prompt_id"] = prompt_id kwargs["prompt_variables"] = prompt_variables + kwargs["prompt_label"] = prompt_label _model_list = self.get_model_list(model_name=model) if _model_list is None or len(_model_list) == 0: # if direct call to model @@ -2427,6 +2451,7 @@ class Router: messages=kwargs.get("messages", None), specific_deployment=kwargs.pop("specific_deployment", None), ) + self._update_kwargs_with_deployment( deployment=deployment, kwargs=kwargs, function_name="generic_api_call" ) @@ -3159,6 +3184,11 @@ class Router: "afile_delete", "afile_content", "_arealtime", + "acreate_fine_tuning_job", + "acancel_fine_tuning_job", + "alist_fine_tuning_jobs", + "aretrieve_fine_tuning_job", + "alist_files", ] = "assistants", ): """ @@ -3207,6 +3237,11 @@ class Router: "anthropic_messages", "aresponses", "_arealtime", + "acreate_fine_tuning_job", + "acancel_fine_tuning_job", + "alist_fine_tuning_jobs", + "aretrieve_fine_tuning_job", + "alist_files", ): return await self._ageneric_api_call_with_fallbacks( original_function=original_function, diff --git a/litellm/router_strategy/lowest_tpm_rpm.py b/litellm/router_strategy/lowest_tpm_rpm.py index 121df00d305..735ddb3f802 100644 --- a/litellm/router_strategy/lowest_tpm_rpm.py +++ b/litellm/router_strategy/lowest_tpm_rpm.py @@ -96,6 +96,8 @@ class LowestTPMLoggingHandler(CustomLogger): if kwargs["litellm_params"].get("metadata") is None: pass else: + if "litellm_params" not in kwargs: + return model_group = kwargs["litellm_params"]["metadata"].get( "model_group", None ) diff --git a/litellm/secret_managers/get_azure_ad_token_provider.py b/litellm/secret_managers/get_azure_ad_token_provider.py index 5403675b979..c982856b5ed 100644 --- a/litellm/secret_managers/get_azure_ad_token_provider.py +++ b/litellm/secret_managers/get_azure_ad_token_provider.py @@ -1,5 +1,9 @@ import os -from typing import Callable +from typing import Any, Callable, Optional, Union + +from litellm.types.secret_managers.get_azure_ad_token_provider import ( + AzureCredentialType, +) def get_azure_ad_token_provider() -> Callable[[], str]: @@ -15,24 +19,45 @@ def get_azure_ad_token_provider() -> Callable[[], str]: Callable that returns a temporary authentication token. """ import azure.identity as identity - from azure.identity import get_bearer_token_provider + from azure.identity import ( + CertificateCredential, + ClientSecretCredential, + ManagedIdentityCredential, + get_bearer_token_provider, + ) azure_scope = os.environ.get( "AZURE_SCOPE", "https://cognitiveservices.azure.com/.default" ) - cred = os.environ.get("AZURE_CREDENTIAL", "ClientSecretCredential") - - cred_cls = getattr(identity, cred) - # ClientSecretCredential, DefaultAzureCredential, AzureCliCredential - if cred == "ClientSecretCredential": - credential = cred_cls( + cred: Union[AzureCredentialType, str] = AzureCredentialType( + os.environ.get("AZURE_CREDENTIAL", AzureCredentialType.ClientSecretCredential) + ) + credential: Optional[ + Union[ + ClientSecretCredential, + ManagedIdentityCredential, + CertificateCredential, + Any, + ] + ] = None + if cred == AzureCredentialType.ClientSecretCredential: + credential = ClientSecretCredential( client_id=os.environ["AZURE_CLIENT_ID"], client_secret=os.environ["AZURE_CLIENT_SECRET"], tenant_id=os.environ["AZURE_TENANT_ID"], ) - elif cred == "ManagedIdentityCredential": - credential = cred_cls(client_id=os.environ["AZURE_CLIENT_ID"]) + elif cred == AzureCredentialType.ManagedIdentityCredential: + credential = ManagedIdentityCredential(client_id=os.environ["AZURE_CLIENT_ID"]) + elif cred == AzureCredentialType.CertificateCredential: + credential = CertificateCredential( + client_id=os.environ["AZURE_CLIENT_ID"], + tenant_id=os.environ["AZURE_TENANT_ID"], + certificate_path=os.environ["AZURE_CERTIFICATE_PATH"], + ) else: + cred_cls = getattr(identity, cred) credential = cred_cls() + if credential is None: + raise ValueError("No credential provided") return get_bearer_token_provider(credential, azure_scope) diff --git a/litellm/types/llms/openai.py b/litellm/types/llms/openai.py index 0d880a4b1cf..367f86fd3f4 100644 --- a/litellm/types/llms/openai.py +++ b/litellm/types/llms/openai.py @@ -878,7 +878,7 @@ class FineTuningJobCreate(BaseModel): class LiteLLMFineTuningJobCreate(FineTuningJobCreate): - custom_llm_provider: Literal["openai", "azure", "vertex_ai"] + custom_llm_provider: Optional[Literal["openai", "azure", "vertex_ai"]] = None model_config = { "extra": "allow" @@ -1000,7 +1000,7 @@ class ResponsesAPIResponse(BaseLiteLLMOpenAIResponseObject): model: Optional[str] object: Optional[str] output: Union[ - List[ResponseOutputItem], + List[Union[ResponseOutputItem, Dict]], List[Union[GenericResponseOutputItem, OutputFunctionToolCall]], ] parallel_tool_calls: bool diff --git a/litellm/types/secret_managers/get_azure_ad_token_provider.py b/litellm/types/secret_managers/get_azure_ad_token_provider.py new file mode 100644 index 00000000000..f318b4333bd --- /dev/null +++ b/litellm/types/secret_managers/get_azure_ad_token_provider.py @@ -0,0 +1,7 @@ +from enum import Enum + + +class AzureCredentialType(str, Enum): + ClientSecretCredential = "ClientSecretCredential" + ManagedIdentityCredential = "ManagedIdentityCredential" + CertificateCredential = "CertificateCredential" diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 2b230d089c3..310a4332f08 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -33,6 +33,7 @@ from pydantic import BaseModel, ConfigDict, Field, PrivateAttr, model_validator from typing_extensions import Callable, Dict, Required, TypedDict, override import litellm +from litellm.types.llms.base import BaseLiteLLMOpenAIResponseObject from ..litellm_core_utils.core_helpers import map_finish_reason from .guardrails import GuardrailEventHooks @@ -44,6 +45,7 @@ from .llms.openai import ( ChatCompletionToolCallChunk, ChatCompletionUsageBlock, FileSearchTool, + FineTuningJob, OpenAIChatCompletionChunk, OpenAIFileObject, OpenAIRealtimeStreamList, @@ -1541,19 +1543,46 @@ class ImageObject(OpenAIImage): return self.dict() +class ImageUsageInputTokensDetails(BaseLiteLLMOpenAIResponseObject): + image_tokens: int + """The number of image tokens in the input prompt.""" + + text_tokens: int + """The number of text tokens in the input prompt.""" + + +class ImageUsage(BaseLiteLLMOpenAIResponseObject): + input_tokens: int + """The number of tokens (images and text) in the input prompt.""" + + input_tokens_details: ImageUsageInputTokensDetails + """The input tokens detailed information for the image generation.""" + + output_tokens: int + """The number of image tokens in the output image.""" + + total_tokens: int + """The total number of tokens (images and text) used for the image generation.""" + + from openai.types.images_response import ImagesResponse as OpenAIImageResponse -class ImageResponse(OpenAIImageResponse): +class ImageResponse(OpenAIImageResponse, BaseLiteLLMOpenAIResponseObject): _hidden_params: dict = {} - usage: Usage + + usage: Optional[ImageUsage] = None # type: ignore + """ + Users might use litellm with older python versions, we don't want this to break for them. + Happens when their OpenAIImageResponse has the old OpenAI usage class. + """ def __init__( self, created: Optional[int] = None, data: Optional[List[ImageObject]] = None, response_ms=None, - usage: Optional[Usage] = None, + usage: Optional[ImageUsage] = None, hidden_params: Optional[dict] = None, ): if response_ms: @@ -1576,9 +1605,14 @@ class ImageResponse(OpenAIImageResponse): _data.append(ImageObject(**d)) elif isinstance(d, BaseModel): _data.append(ImageObject(**d.model_dump())) - _usage = usage or Usage( - prompt_tokens=0, - completion_tokens=0, + + _usage = usage or ImageUsage( + input_tokens=0, + input_tokens_details=ImageUsageInputTokensDetails( + image_tokens=0, + text_tokens=0, + ), + output_tokens=0, total_tokens=0, ) super().__init__(created=created, data=_data, usage=_usage) # type: ignore @@ -2083,6 +2117,7 @@ all_litellm_params = [ "allowed_openai_params", "litellm_session_id", "use_litellm_proxy", + "prompt_label", ] + list(StandardCallbackDynamicParams.__annotations__.keys()) @@ -2256,6 +2291,19 @@ class SelectTokenizerResponse(TypedDict): tokenizer: Any +class LiteLLMFineTuningJob(FineTuningJob): + _hidden_params: dict = {} + seed: Optional[int] = None # type: ignore + + def __init__(self, **kwargs): + if "error" in kwargs and kwargs["error"] is not None: + # check if error is all None - if so, set error to None + if all(value is None for value in kwargs["error"].values()): + kwargs["error"] = None + super().__init__(**kwargs) + self._hidden_params = kwargs.get("_hidden_params", {}) + + class LiteLLMBatch(Batch): _hidden_params: dict = {} usage: Optional[Usage] = None @@ -2360,9 +2408,16 @@ class SpecialEnums(Enum): LITELLM_MANAGED_BATCH_COMPLETE_STR = "litellm_proxy;model_id:{};llm_batch_id:{}" + LITELLM_MANAGED_GENERIC_RESPONSE_COMPLETE_STR = "litellm_proxy;model_id:{};generic_response_id:{}" # generic implementation of 'managed batches' - used for finetuning and any future work. + LLMResponseTypes = Union[ - ModelResponse, EmbeddingResponse, ImageResponse, OpenAIFileObject, LiteLLMBatch + ModelResponse, + EmbeddingResponse, + ImageResponse, + OpenAIFileObject, + LiteLLMBatch, + LiteLLMFineTuningJob, ] diff --git a/litellm/utils.py b/litellm/utils.py index 773196077d1..65d825c979c 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -6835,6 +6835,14 @@ def _add_path_to_api_base(api_base: str, ending_path: str) -> str: return str(modified_url.copy_with(params=original_url.params)) +def get_standard_openai_params(params: dict) -> dict: + return { + k: v + for k, v in params.items() + if k in litellm.OPENAI_CHAT_COMPLETION_PARAMS and v is not None + } + + def get_non_default_completion_params(kwargs: dict) -> dict: openai_params = litellm.OPENAI_CHAT_COMPLETION_PARAMS default_params = openai_params + all_litellm_params diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 8d7d062b818..0a37f9fb99e 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -3117,6 +3117,18 @@ "supports_function_calling": true, "source": "https://azuremarketplace.microsoft.com/en/marketplace/apps/000-000.mistral-nemo-12b-2407?tab=PlansAndPrice" }, + "azure_ai/mistral-medium-2505": { + "max_tokens": 8191, + "max_input_tokens": 131072, + "max_output_tokens": 8191, + "input_cost_per_token": 0.0000004, + "output_cost_per_token": 0.000002, + "litellm_provider": "azure_ai", + "mode": "chat", + "supports_function_calling": true, + "supports_assistant_prefill": true, + "supports_tool_choice": true + }, "azure_ai/mistral-large": { "max_tokens": 8191, "max_input_tokens": 32000, @@ -4684,6 +4696,58 @@ "deprecation_date": "2025-06-01", "supports_tool_choice": true }, + "claude-opus-4-20250514": { + "max_tokens": 32000, + "max_input_tokens": 200000, + "max_output_tokens": 32000, + "input_cost_per_token": 15e-6, + "output_cost_per_token": 75e-6, + "search_context_cost_per_query": { + "search_context_size_low": 1e-2, + "search_context_size_medium": 1e-2, + "search_context_size_high": 1e-2 + }, + "cache_creation_input_token_cost": 18.75e-6, + "cache_read_input_token_cost": 1.5e-6, + "litellm_provider": "anthropic", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "tool_use_system_prompt_tokens": 159, + "supports_assistant_prefill": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_computer_use": true + }, + "claude-sonnet-4-20250514": { + "max_tokens": 64000, + "max_input_tokens": 200000, + "max_output_tokens": 64000, + "input_cost_per_token": 3e-6, + "output_cost_per_token": 15e-6, + "search_context_cost_per_query": { + "search_context_size_low": 1e-2, + "search_context_size_medium": 1e-2, + "search_context_size_high": 1e-2 + }, + "cache_creation_input_token_cost": 3.75e-6, + "cache_read_input_token_cost": 0.3e-6, + "litellm_provider": "anthropic", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "tool_use_system_prompt_tokens": 159, + "supports_assistant_prefill": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_computer_use": true + }, "claude-3-7-sonnet-latest": { "supports_computer_use": true, "max_tokens": 128000, @@ -6753,6 +6817,58 @@ "supports_reasoning": true, "supports_tool_choice": true }, + "vertex_ai/claude-opus-4@20250514": { + "max_tokens": 32000, + "max_input_tokens": 200000, + "max_output_tokens": 32000, + "input_cost_per_token": 15e-6, + "output_cost_per_token": 75e-6, + "search_context_cost_per_query": { + "search_context_size_low": 1e-2, + "search_context_size_medium": 1e-2, + "search_context_size_high": 1e-2 + }, + "cache_creation_input_token_cost": 18.75e-6, + "cache_read_input_token_cost": 1.5e-6, + "litellm_provider": "vertex_ai-anthropic_models", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "tool_use_system_prompt_tokens": 159, + "supports_assistant_prefill": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_computer_use": true + }, + "vertex_ai/claude-sonnet-4@20250514": { + "max_tokens": 64000, + "max_input_tokens": 200000, + "max_output_tokens": 64000, + "input_cost_per_token": 3e-6, + "output_cost_per_token": 15e-6, + "search_context_cost_per_query": { + "search_context_size_low": 1e-2, + "search_context_size_medium": 1e-2, + "search_context_size_high": 1e-2 + }, + "cache_creation_input_token_cost": 3.75e-6, + "cache_read_input_token_cost": 0.3e-6, + "litellm_provider": "vertex_ai-anthropic_models", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "tool_use_system_prompt_tokens": 159, + "supports_assistant_prefill": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_computer_use": true + }, "vertex_ai/claude-3-haiku": { "max_tokens": 4096, "max_input_tokens": 200000, @@ -9332,6 +9448,58 @@ "supports_pdf_input": true, "supports_tool_choice": true }, + "anthropic.claude-opus-4-20250514-v1:0": { + "max_tokens": 32000, + "max_input_tokens": 200000, + "max_output_tokens": 32000, + "input_cost_per_token": 15e-6, + "output_cost_per_token": 75e-6, + "search_context_cost_per_query": { + "search_context_size_low": 1e-2, + "search_context_size_medium": 1e-2, + "search_context_size_high": 1e-2 + }, + "cache_creation_input_token_cost": 18.75e-6, + "cache_read_input_token_cost": 1.5e-6, + "litellm_provider": "bedrock_converse", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "tool_use_system_prompt_tokens": 159, + "supports_assistant_prefill": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_computer_use": true + }, + "anthropic.claude-sonnet-4-20250514-v1:0": { + "max_tokens": 64000, + "max_input_tokens": 200000, + "max_output_tokens": 64000, + "input_cost_per_token": 3e-6, + "output_cost_per_token": 15e-6, + "search_context_cost_per_query": { + "search_context_size_low": 1e-2, + "search_context_size_medium": 1e-2, + "search_context_size_high": 1e-2 + }, + "cache_creation_input_token_cost": 3.75e-6, + "cache_read_input_token_cost": 0.3e-6, + "litellm_provider": "bedrock_converse", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "tool_use_system_prompt_tokens": 159, + "supports_assistant_prefill": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_computer_use": true + }, "anthropic.claude-3-7-sonnet-20250219-v1:0": { "supports_computer_use": true, "max_tokens": 8192, @@ -9482,6 +9650,58 @@ "supports_tool_choice": true, "supports_reasoning": true }, + "us.anthropic.claude-opus-4-20250514-v1:0": { + "max_tokens": 32000, + "max_input_tokens": 200000, + "max_output_tokens": 32000, + "input_cost_per_token": 15e-6, + "output_cost_per_token": 75e-6, + "search_context_cost_per_query": { + "search_context_size_low": 1e-2, + "search_context_size_medium": 1e-2, + "search_context_size_high": 1e-2 + }, + "cache_creation_input_token_cost": 18.75e-6, + "cache_read_input_token_cost": 1.5e-6, + "litellm_provider": "bedrock_converse", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "tool_use_system_prompt_tokens": 159, + "supports_assistant_prefill": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_computer_use": true + }, + "us.anthropic.claude-sonnet-4-20250514-v1:0": { + "max_tokens": 64000, + "max_input_tokens": 200000, + "max_output_tokens": 64000, + "input_cost_per_token": 3e-6, + "output_cost_per_token": 15e-6, + "search_context_cost_per_query": { + "search_context_size_low": 1e-2, + "search_context_size_medium": 1e-2, + "search_context_size_high": 1e-2 + }, + "cache_creation_input_token_cost": 3.75e-6, + "cache_read_input_token_cost": 0.3e-6, + "litellm_provider": "bedrock_converse", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "tool_use_system_prompt_tokens": 159, + "supports_assistant_prefill": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_computer_use": true + }, "us.anthropic.claude-3-haiku-20240307-v1:0": { "max_tokens": 4096, "max_input_tokens": 200000, @@ -9603,6 +9823,58 @@ "supports_pdf_input": true, "supports_tool_choice": true }, + "eu.anthropic.claude-opus-4-20250514-v1:0": { + "max_tokens": 32000, + "max_input_tokens": 200000, + "max_output_tokens": 32000, + "input_cost_per_token": 15e-6, + "output_cost_per_token": 75e-6, + "search_context_cost_per_query": { + "search_context_size_low": 1e-2, + "search_context_size_medium": 1e-2, + "search_context_size_high": 1e-2 + }, + "cache_creation_input_token_cost": 18.75e-6, + "cache_read_input_token_cost": 1.5e-6, + "litellm_provider": "bedrock_converse", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "tool_use_system_prompt_tokens": 159, + "supports_assistant_prefill": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_computer_use": true + }, + "eu.anthropic.claude-sonnet-4-20250514-v1:0": { + "max_tokens": 64000, + "max_input_tokens": 200000, + "max_output_tokens": 64000, + "input_cost_per_token": 3e-6, + "output_cost_per_token": 15e-6, + "search_context_cost_per_query": { + "search_context_size_low": 1e-2, + "search_context_size_medium": 1e-2, + "search_context_size_high": 1e-2 + }, + "cache_creation_input_token_cost": 3.75e-6, + "cache_read_input_token_cost": 0.3e-6, + "litellm_provider": "bedrock_converse", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "tool_use_system_prompt_tokens": 159, + "supports_assistant_prefill": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_computer_use": true + }, "eu.anthropic.claude-3-5-haiku-20241022-v1:0": { "max_tokens": 8192, "max_input_tokens": 200000, @@ -12297,7 +12569,7 @@ "max_output_tokens": 128000, "input_cost_per_token": 0.0000025, "input_dbu_cost_per_token": 0.00003571, - "output_cost_per_token": 0.00017857, + "output_cost_per_token": 0.000017857, "output_db_cost_per_token": 0.000214286, "litellm_provider": "databricks", "mode": "chat", @@ -12350,6 +12622,20 @@ "metadata": {"notes": "Input/output cost per token is dbu cost * $0.070, based on databricks Llama 3.1 70B conversion. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation."}, "supports_tool_choice": true }, + "databricks/databricks-llama-4-maverick": { + "max_tokens": 128000, + "max_input_tokens": 128000, + "max_output_tokens": 128000, + "input_cost_per_token": 0.000005, + "input_dbu_cost_per_token": 0.00007143, + "output_cost_per_token": 0.000015, + "output_dbu_cost_per_token": 0.00021429, + "litellm_provider": "databricks", + "mode": "chat", + "source": "https://www.databricks.com/product/pricing/foundation-model-serving", + "metadata": {"notes": "Databricks documentation now provides both DBU costs (_dbu_cost_per_token) and dollar costs(_cost_per_token)."}, + "supports_tool_choice": true + }, "databricks/databricks-dbrx-instruct": { "max_tokens": 32768, "max_input_tokens": 32768, @@ -12976,4 +13262,4 @@ "litellm_provider": "featherless_ai", "mode": "chat" } -} \ No newline at end of file +} diff --git a/poetry.lock b/poetry.lock index 9c361b597d5..cadc7cd6d76 100644 --- a/poetry.lock +++ b/poetry.lock @@ -258,10 +258,9 @@ tests-mypy = ["mypy (>=1.11.1) ; platform_python_implementation == \"CPython\" a name = "azure-core" version = "1.33.0" description = "Microsoft Azure Core Library for Python" -optional = true +optional = false python-versions = ">=3.8" -groups = ["main"] -markers = "extra == \"extra-proxy\"" +groups = ["main", "proxy-dev"] files = [ {file = "azure_core-1.33.0-py3-none-any.whl", hash = "sha256:9b5b6d0223a1d38c37500e6971118c1e0f13f54951e6893968b38910bc9cda8f"}, {file = "azure_core-1.33.0.tar.gz", hash = "sha256:f367aa07b5e3005fec2c1e184b882b0b039910733907d001c20fb08ebb8c0eb9"}, @@ -280,10 +279,9 @@ tracing = ["opentelemetry-api (>=1.26,<2.0)"] name = "azure-identity" version = "1.21.0" description = "Microsoft Azure Identity Library for Python" -optional = true +optional = false python-versions = ">=3.8" -groups = ["main"] -markers = "extra == \"extra-proxy\"" +groups = ["main", "proxy-dev"] files = [ {file = "azure_identity-1.21.0-py3-none-any.whl", hash = "sha256:258ea6325537352440f71b35c3dffe9d240eae4a5126c1b7ce5efd5766bd9fd9"}, {file = "azure_identity-1.21.0.tar.gz", hash = "sha256:ea22ce6e6b0f429bc1b8d9212d5b9f9877bd4c82f1724bfa910760612c07a9a6"}, @@ -499,7 +497,7 @@ version = "1.17.1" description = "Foreign Function Interface for Python calling C code." optional = false python-versions = ">=3.8" -groups = ["main", "dev"] +groups = ["main", "dev", "proxy-dev"] files = [ {file = "cffi-1.17.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:df8b1c11f177bc2313ec4b2d46baec87a5f3e71fc8b45dab2ee7cae86d9aba14"}, {file = "cffi-1.17.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8f2cdc858323644ab277e9bb925ad72ae0e67f69e804f4898c070998d50b1a67"}, @@ -569,7 +567,7 @@ files = [ {file = "cffi-1.17.1-cp39-cp39-win_amd64.whl", hash = "sha256:d016c76bdd850f3c626af19b0542c9677ba156e4ee4fccfdd7848803533ef662"}, {file = "cffi-1.17.1.tar.gz", hash = "sha256:1c39c6016c32bc48dd54561950ebd6836e1670f2ae46128f67cf49e789c52824"}, ] -markers = {main = "(extra == \"proxy\" or extra == \"extra-proxy\") and (platform_python_implementation != \"PyPy\" or extra == \"proxy\")", dev = "platform_python_implementation != \"PyPy\""} +markers = {main = "platform_python_implementation != \"PyPy\" or extra == \"proxy\"", dev = "platform_python_implementation != \"PyPy\"", proxy-dev = "platform_python_implementation != \"PyPy\""} [package.dependencies] pycparser = "*" @@ -729,7 +727,7 @@ version = "43.0.3" description = "cryptography is a package which provides cryptographic recipes and primitives to Python developers." optional = false python-versions = ">=3.7" -groups = ["main", "dev"] +groups = ["main", "dev", "proxy-dev"] files = [ {file = "cryptography-43.0.3-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:bf7a1932ac4176486eab36a19ed4c0492da5d97123f1406cf15e41b05e787d2e"}, {file = "cryptography-43.0.3-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:63efa177ff54aec6e1c0aefaa1a241232dcd37413835a9b674b6e3f0ae2bfd3e"}, @@ -759,7 +757,6 @@ files = [ {file = "cryptography-43.0.3-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:2ce6fae5bdad59577b44e4dfed356944fbf1d925269114c28be377692643b4ff"}, {file = "cryptography-43.0.3.tar.gz", hash = "sha256:315b9001266a492a6ff443b61238f956b214dbec9910a081ba5b6646a055a805"}, ] -markers = {main = "extra == \"proxy\" or extra == \"extra-proxy\""} [package.dependencies] cffi = {version = ">=1.12", markers = "platform_python_implementation != \"PyPy\""} @@ -1986,10 +1983,9 @@ dev = ["absl-py", "pyink", "pylint (>=2.6.0)", "pytest", "pytest-xdist"] name = "msal" version = "1.32.3" description = "The Microsoft Authentication Library (MSAL) for Python library enables your app to access the Microsoft Cloud by supporting authentication of users with Microsoft Azure Active Directory accounts (AAD) and Microsoft Accounts (MSA) using industry standard OAuth2 and OpenID Connect." -optional = true +optional = false python-versions = ">=3.7" -groups = ["main"] -markers = "extra == \"extra-proxy\"" +groups = ["main", "proxy-dev"] files = [ {file = "msal-1.32.3-py3-none-any.whl", hash = "sha256:b2798db57760b1961b142f027ffb7c8169536bf77316e99a0df5c4aaebb11569"}, {file = "msal-1.32.3.tar.gz", hash = "sha256:5eea038689c78a5a70ca8ecbe1245458b55a857bd096efb6989c69ba15985d35"}, @@ -2007,10 +2003,9 @@ broker = ["pymsalruntime (>=0.14,<0.18) ; python_version >= \"3.6\" and platform name = "msal-extensions" version = "1.3.0" description = "Microsoft Authentication Library extensions (MSAL EX) provides a persistence API that can save your data on disk, encrypted on Windows, macOS and Linux. Concurrent data access will be coordinated by a file lock mechanism." -optional = true +optional = false python-versions = ">=3.7" -groups = ["main"] -markers = "extra == \"extra-proxy\"" +groups = ["main", "proxy-dev"] files = [ {file = "msal_extensions-1.3.0-py3-none-any.whl", hash = "sha256:105328ddcbdd342016c9949d8f89e3917554740c8ab26669c0fa0e069e730a0e"}, {file = "msal_extensions-1.3.0.tar.gz", hash = "sha256:96918996642b38c78cd59b55efa0f06fd1373c90e0949be8615697c048fba62c"}, @@ -2937,12 +2932,12 @@ version = "2.22" description = "C parser in Python" optional = false python-versions = ">=3.8" -groups = ["main", "dev"] +groups = ["main", "dev", "proxy-dev"] files = [ {file = "pycparser-2.22-py3-none-any.whl", hash = "sha256:c3702b6d3dd8c7abc1afa565d7e63d53a1d0bd86cdc24edd75470f4de499cfcc"}, {file = "pycparser-2.22.tar.gz", hash = "sha256:491c8be9c040f5390f5bf44a5b07752bd07f56edf992381b05c701439eec10f6"}, ] -markers = {main = "(extra == \"proxy\" or extra == \"extra-proxy\") and (platform_python_implementation != \"PyPy\" or extra == \"proxy\")", dev = "platform_python_implementation != \"PyPy\""} +markers = {main = "platform_python_implementation != \"PyPy\" or extra == \"proxy\"", dev = "platform_python_implementation != \"PyPy\"", proxy-dev = "platform_python_implementation != \"PyPy\""} [[package]] name = "pydantic" @@ -3136,10 +3131,9 @@ windows-terminal = ["colorama (>=0.4.6)"] name = "pyjwt" version = "2.9.0" description = "JSON Web Token implementation in Python" -optional = true +optional = false python-versions = ">=3.8" -groups = ["main"] -markers = "extra == \"extra-proxy\" or extra == \"proxy\"" +groups = ["main", "proxy-dev"] files = [ {file = "PyJWT-2.9.0-py3-none-any.whl", hash = "sha256:3b02fb0f44517787776cf48f2ae25d8e14f300e6d7545a4315cee571a415e850"}, {file = "pyjwt-2.9.0.tar.gz", hash = "sha256:7e1e5b56cc735432a7369cbfa0efe50fa113ebecdc04ae6922deba8b84582d0c"}, @@ -3879,10 +3873,9 @@ crt = ["botocore[crt] (>=1.33.2,<2.0a.0)"] name = "six" version = "1.17.0" description = "Python 2 and 3 compatibility utilities" -optional = true +optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" -groups = ["main"] -markers = "extra == \"extra-proxy\" or extra == \"proxy\"" +groups = ["main", "proxy-dev"] files = [ {file = "six-1.17.0-py2.py3-none-any.whl", hash = "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274"}, {file = "six-1.17.0.tar.gz", hash = "sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81"}, @@ -4935,4 +4928,4 @@ utils = ["numpydoc"] [metadata] lock-version = "2.1" python-versions = ">=3.8.1,<4.0, !=3.9.7" -content-hash = "4c385d4e27013d9cacf9573f5532c5e13d22fc27af6344e87ba2417340c9be93" +content-hash = "fe1bc122aaeae89043f8099bfe7fda082110b06ddee5147cbfcdd085f072ef72" diff --git a/pyproject.toml b/pyproject.toml index 2607937e7b1..0cc0c4a826e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "litellm" -version = "1.70.3" +version = "1.70.4" description = "Library to easily interface with LLM API providers" authors = ["BerriAI"] license = "MIT" @@ -21,7 +21,7 @@ Documentation = "https://docs.litellm.ai" [tool.poetry.dependencies] python = ">=3.8.1,<4.0, !=3.9.7" httpx = ">=0.23.0" -openai = ">=1.68.2, <1.76.0" +openai = ">=1.68.2" python-dotenv = ">=0.2.0" tiktoken = ">=0.7.0" importlib-metadata = ">=6.8.0" @@ -130,13 +130,14 @@ prometheus-client = "0.20.0" opentelemetry-api = "1.25.0" opentelemetry-sdk = "1.25.0" opentelemetry-exporter-otlp = "1.25.0" +azure-identity = "^1.15.0" [build-system] requires = ["poetry-core", "wheel"] build-backend = "poetry.core.masonry.api" [tool.commitizen] -version = "1.70.3" +version = "1.70.4" version_files = [ "pyproject.toml:^version" ] diff --git a/requirements.txt b/requirements.txt index db040686f1e..e3aae6f90fc 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,7 @@ # LITELLM PROXY DEPENDENCIES # anyio==4.5.0 # openai + http req. httpx==0.27.0 # Pin Httpx dependency -openai==1.68.2 # openai req. +openai==1.81.0 # openai req. fastapi==0.115.5 # server dep backoff==2.2.1 # server dep pyyaml==6.0.2 # server dep diff --git a/schema.prisma b/schema.prisma index 1d6f3b52118..b415a777359 100644 --- a/schema.prisma +++ b/schema.prisma @@ -453,13 +453,30 @@ model LiteLLM_ManagedFileTable { id String @id @default(uuid()) unified_file_id String @unique // The base64 encoded unified file ID file_object Json // Stores the OpenAIFileObject - model_mappings Json // Stores the mapping of model_id -> provider_file_id + model_mappings Json + flat_model_file_ids String[] @default([]) // Flat list of model file id's - for faster querying of model id -> unified file id created_at DateTime @default(now()) + created_by String? updated_at DateTime @updatedAt + updated_by String? @@index([unified_file_id]) } +model LiteLLM_ManagedObjectTable { // for batches or finetuning jobs which use managed files + id String @id @default(uuid()) + unified_object_id String @unique // The base64 encoded unified object ID + model_object_id String @unique // the id returned by the backend API provider + file_object Json // Stores the OpenAIFileObject + file_purpose String // either 'batch' or 'fine-tune' + created_at DateTime @default(now()) + created_by String? + updated_at DateTime @updatedAt + updated_by String? + + @@index([unified_object_id]) + @@index([model_object_id]) +} model LiteLLM_ManagedVectorStoresTable { vector_store_id String @id diff --git a/tests/batches_tests/test_fine_tuning_api.py b/tests/batches_tests/test_fine_tuning_api.py index e6c20e129e7..3561d99d0f6 100644 --- a/tests/batches_tests/test_fine_tuning_api.py +++ b/tests/batches_tests/test_fine_tuning_api.py @@ -525,9 +525,12 @@ async def test_mock_openai_cancel_fine_tune_job(): client = AsyncOpenAI(api_key="fake-api-key") with patch.object(client.fine_tuning.jobs, "cancel") as mock_cancel: - await litellm.acancel_fine_tuning_job( - fine_tuning_job_id="ft-123", client=client - ) + try: + await litellm.acancel_fine_tuning_job( + fine_tuning_job_id="ft-123", client=client + ) + except Exception as e: + print("error=", e) # Only verify that the client was called with correct parameters mock_cancel.assert_called_once_with(fine_tuning_job_id="ft-123") @@ -541,10 +544,13 @@ async def test_mock_openai_retrieve_fine_tune_job(): client = AsyncOpenAI(api_key="fake-api-key") with patch.object(client.fine_tuning.jobs, "retrieve") as mock_retrieve: + try: + response = await litellm.aretrieve_fine_tuning_job( + fine_tuning_job_id="ft-123", client=client + ) + except Exception as e: + print("error=", e) - response = await litellm.aretrieve_fine_tuning_job( - fine_tuning_job_id="ft-123", client=client - ) # Verify the request mock_retrieve.assert_called_once_with(fine_tuning_job_id="ft-123") diff --git a/tests/enterprise/enterprise_hooks/test_managed_files.py b/tests/enterprise/enterprise_hooks/test_managed_files.py index e84034ed099..81e27d941fa 100644 --- a/tests/enterprise/enterprise_hooks/test_managed_files.py +++ b/tests/enterprise/enterprise_hooks/test_managed_files.py @@ -3,16 +3,20 @@ import os import sys import pytest +from fastapi import HTTPException from fastapi.testclient import TestClient sys.path.insert( 0, os.path.abspath("../../..") ) # Adds the parent directory to the system path -from unittest.mock import MagicMock +from unittest.mock import AsyncMock, MagicMock, patch from enterprise.enterprise_hooks.managed_files import _PROXY_LiteLLMManagedFiles from litellm.caching import DualCache +from litellm.proxy.openai_files_endpoints.common_utils import ( + _is_base64_encoded_unified_file_id, +) from litellm.types.utils import SpecialEnums @@ -42,11 +46,19 @@ def test_get_file_ids_from_messages(): @pytest.mark.asyncio async def test_async_pre_call_hook_batch_retrieve(): + from litellm.proxy._types import UserAPIKeyAuth + + prisma_client = AsyncMock() + return_value = MagicMock() + return_value.created_by = "123" + prisma_client.db.litellm_managedobjecttable.find_first.return_value = return_value proxy_managed_files = _PROXY_LiteLLMManagedFiles( - DualCache(), prisma_client=MagicMock() + DualCache(), prisma_client=prisma_client ) data = { - "user_api_key_dict": {"parent_otel_span": MagicMock()}, + "user_api_key_dict": UserAPIKeyAuth( + user_id="123", parent_otel_span=MagicMock() + ), "data": { "batch_id": "bGl0ZWxsbV9wcm94eTttb2RlbF9pZDpteS1nZW5lcmFsLWF6dXJlLWRlcGxveW1lbnQ7bGxtX2JhdGNoX2lkOmJhdGNoX2EzMjJiNmJhLWFjN2UtNDg4OC05MjljLTFhZDM0NDJmMDZlZA", }, @@ -161,3 +173,99 @@ async def test_async_pre_call_hook_batch_retrieve(): # assert len(batch_files) == 1 # assert assistant_files[0].id == file1.id # assert batch_files[0].id == file2.id + + +@pytest.mark.asyncio +async def test_async_post_call_success_hook_for_unified_finetuning_job(): + from litellm.types.utils import LiteLLMFineTuningJob + + unified_file_id = "bGl0ZWxsbV9wcm94eTphcHBsaWNhdGlvbi9vY3RldC1zdHJlYW07dW5pZmllZF9pZCxiZTQ0ZDVlYi1mNDU3LTRiNzktOWM4My01N2QxMTMxYWM0YzY7dGFyZ2V0X21vZGVsX25hbWVzLGdwdC00LjEtb3BlbmFpO2xsbV9vdXRwdXRfZmlsZV9pZCxmaWxlLURKMnQ0OWZlQ2NTQk5vNG9oekZ6NGc7bGxtX291dHB1dF9maWxlX21vZGVsX2lkLGRiNjY5ODcwNzdkZTdmYzZjNzAzY2Y1MDczMGU2MmNkOWQ3YTU1N2NlNjVmMDUzNTFkYTM4YTA3ZjBlZDEyNzQ" + provider_ft_job = LiteLLMFineTuningJob( + object="fine_tuning.job", + id="ftjob-0kEBV5b4sPrFcMnuzmYSzU1G", + model="gpt-3.5-turbo-0613", + created_at=1692779769, + finished_at=None, + fine_tuned_model=None, + organization_id="org-dUVLhaAQ37YCGwVC2QVY8sdB", + result_files=[], + status="validating_files", + validation_file=None, + training_file="file-azQuKMLAmiFdEjxpCcbI11zF", + hyperparameters={"n_epochs": 8}, + trained_tokens=None, + seed=0, + ) + provider_ft_job._hidden_params = { + "unified_file_id": unified_file_id, + "model_id": "gpt-3.5-turbo-0613", + } + proxy_managed_files = _PROXY_LiteLLMManagedFiles( + DualCache(), prisma_client=MagicMock() + ) + data = { + "user_api_key_dict": {"parent_otel_span": MagicMock()}, + } + + response = await proxy_managed_files.async_post_call_success_hook( + data=data, + user_api_key_dict=MagicMock(), + response=provider_ft_job, + ) + + assert isinstance(response, LiteLLMFineTuningJob) + assert _is_base64_encoded_unified_file_id(response.id) + + +@pytest.mark.asyncio +async def test_async_pre_call_hook_for_unified_finetuning_job(): + from litellm.proxy._types import UserAPIKeyAuth + + prisma_client = AsyncMock() + return_value = MagicMock() + return_value.created_by = "123" + prisma_client.db.litellm_managedobjecttable.find_first.return_value = return_value + proxy_managed_files = _PROXY_LiteLLMManagedFiles( + DualCache(), prisma_client=prisma_client + ) + data = { + "user_api_key_dict": UserAPIKeyAuth( + user_id="123", parent_otel_span=MagicMock() + ), + "data": { + "fine_tuning_job_id": "bGl0ZWxsbV9wcm94eTttb2RlbF9pZDo0OTIxODU4MWY3OGViZTllZjE4NDE0ZmE0ZjdmYjlmYTc0YzA5NWVkMTEyY2E4NDBkZDU2ZGZmZTliZDMwZGQxO2dlbmVyaWNfcmVzcG9uc2VfaWQ6ZnRqb2ItalRCeXM3YlZzYnlaRE93TDlHbHBZcVhS", + }, + "call_type": "acancel_fine_tuning_job", + "cache": MagicMock(), + } + + response = await proxy_managed_files.async_pre_call_hook(**data) + assert response["fine_tuning_job_id"] == "ftjob-jTBys7bVsbyZDOwL9GlpYqXR" + + +@pytest.mark.asyncio +@pytest.mark.parametrize("call_type", ["afile_content", "afile_delete"]) +async def test_can_user_call_unified_file_id(call_type): + """ + Test that on file retrieve, delete we check if the user has access to the file + """ + from litellm.proxy._types import UserAPIKeyAuth + + prisma_client = AsyncMock() + return_value = MagicMock() + return_value.created_by = "123" + prisma_client.db.litellm_managedfiletable.find_first.return_value = return_value + proxy_managed_files = _PROXY_LiteLLMManagedFiles( + MagicMock(), prisma_client=prisma_client + ) + unified_file_id = "bGl0ZWxsbV9wcm94eTphcHBsaWNhdGlvbi9vY3RldC1zdHJlYW07dW5pZmllZF9pZCxmMTNlNDAzZS01YWM3LTRhZjktOGQzNS0wNDgwZDMxOTgyYTg7dGFyZ2V0X21vZGVsX25hbWVzLGdwdC00by1taW5pLW9wZW5haTtsbG1fb3V0cHV0X2ZpbGVfaWQsZmlsZS1Ib3UxZDFXc3c1SDNKcjFMYllpZDJiO2xsbV9vdXRwdXRfZmlsZV9tb2RlbF9pZCxmODBiNWU2NzQ1NzdkNjkyMjM4YmVhNTIxZDdiMGI5ZGYyY2FmMTEwMTU2YmU5YzBjM2NjMmNkNTBjOTM1ZDI0" + + with pytest.raises(HTTPException) as e: + await proxy_managed_files.async_pre_call_hook( + user_api_key_dict=UserAPIKeyAuth( + user_id="456", parent_otel_span=MagicMock() + ), + cache=MagicMock(), + data={"file_id": unified_file_id}, + call_type=call_type, + ) diff --git a/tests/image_gen_tests/base_image_generation_test.py b/tests/image_gen_tests/base_image_generation_test.py index df64d353c71..c3a5cfb2251 100644 --- a/tests/image_gen_tests/base_image_generation_test.py +++ b/tests/image_gen_tests/base_image_generation_test.py @@ -68,10 +68,17 @@ class BaseImageGenTest(ABC): assert logged_standard_logging_payload is not None assert logged_standard_logging_payload["response_cost"] is not None assert logged_standard_logging_payload["response_cost"] > 0 - + import openai from openai.types.images_response import ImagesResponse - ImagesResponse.model_validate(response.model_dump()) + # print openai version + print("openai version=", openai.__version__) + + response_dict = dict(response) + if "usage" in response_dict: + response_dict["usage"] = dict(response_dict["usage"]) + print("response usage=", response_dict.get("usage")) + ImagesResponse.model_validate(response_dict) for d in response.data: assert isinstance(d, Image) diff --git a/tests/litellm/integrations/SlackAlerting/test_slack_alerting_utils.py b/tests/litellm/integrations/SlackAlerting/test_slack_alerting_utils.py new file mode 100644 index 00000000000..027fed1b5ff --- /dev/null +++ b/tests/litellm/integrations/SlackAlerting/test_slack_alerting_utils.py @@ -0,0 +1,39 @@ +import json +import os +import sys +from typing import Optional +from unittest.mock import MagicMock + +import pytest + +# Adds the grandparent directory to sys.path to allow importing project modules +sys.path.insert(0, os.path.abspath("../..")) + +import litellm +from litellm.integrations.langfuse.langfuse_prompt_management import ( + LangfusePromptManagement, +) +from litellm.integrations.SlackAlerting.utils import _add_langfuse_trace_id_to_alert +from litellm.litellm_core_utils.logging_callback_manager import LoggingCallbackManager + + +@pytest.mark.asyncio +async def test_langfuse_not_initialized_returns_none_early(): + """ + Test that when no LangfusePromptManagement is initialized, + the function returns None immediately without executing further logic + """ + # Ensure no Langfuse logger is in the callback manager + litellm.logging_callback_manager = LoggingCallbackManager() + + # Create request data that would normally trigger processing + request_data = {"litellm_logging_obj": MagicMock(), "trace_id": "test-trace-id"} + + # Call the function + result = await _add_langfuse_trace_id_to_alert(request_data) + + # Should return None early without processing request_data + assert result is None + + # Verify the litellm_logging_obj was never accessed (early return) + request_data["litellm_logging_obj"].assert_not_called() diff --git a/tests/litellm/integrations/test_custom_prompt_management.py b/tests/litellm/integrations/test_custom_prompt_management.py index 09ba32b2033..f5855abf71e 100644 --- a/tests/litellm/integrations/test_custom_prompt_management.py +++ b/tests/litellm/integrations/test_custom_prompt_management.py @@ -33,6 +33,7 @@ class TestCustomPromptManagement(CustomPromptManagement): prompt_id: Optional[str], prompt_variables: Optional[dict], dynamic_callback_params: StandardCallbackDynamicParams, + prompt_label: Optional[str], ) -> Tuple[str, List[AllMessageValues], dict]: print( "TestCustomPromptManagement: running get_chat_completion_prompt for prompt_id: ", diff --git a/tests/litellm/integrations/test_langfuse.py b/tests/litellm/integrations/test_langfuse.py index 2aa363e5f94..e7e4ca74845 100644 --- a/tests/litellm/integrations/test_langfuse.py +++ b/tests/litellm/integrations/test_langfuse.py @@ -1,8 +1,13 @@ import unittest +import asyncio from unittest.mock import patch, MagicMock +from typing import Optional import sys import os import datetime +import json +import pytest +import litellm sys.path.insert(0, os.path.abspath("../..")) from litellm.integrations.langfuse.langfuse import LangFuseLogger @@ -210,3 +215,44 @@ class TestLangfuseUsageDetails(unittest.TestCase): self.assertEqual(usage_details["output"], 25) self.assertEqual(usage_details["cache_creation_input_tokens"], 7) self.assertEqual(usage_details["cache_read_input_tokens"], 4) + +def test_max_langfuse_clients_limit(): + """ + Test that the max langfuse clients limit is respected when initializing multiple clients + """ + # Set max clients to 2 for testing + with patch( + "litellm.integrations.langfuse.langfuse.MAX_LANGFUSE_INITIALIZED_CLIENTS", 2 + ): + # Reset the counter + litellm.initialized_langfuse_clients = 0 + + # First client should succeed + logger1 = LangFuseLogger( + langfuse_public_key="test_key_1", + langfuse_secret="test_secret_1", + langfuse_host="https://test1.langfuse.com", + ) + assert litellm.initialized_langfuse_clients == 1 + + # Second client should succeed + logger2 = LangFuseLogger( + langfuse_public_key="test_key_2", + langfuse_secret="test_secret_2", + langfuse_host="https://test2.langfuse.com", + ) + assert litellm.initialized_langfuse_clients == 2 + + # Third client should fail with exception + with pytest.raises(Exception) as exc_info: + logger3 = LangFuseLogger( + langfuse_public_key="test_key_3", + langfuse_secret="test_secret_3", + langfuse_host="https://test3.langfuse.com", + ) + + # Verify the error message contains the expected text + assert "Max langfuse clients reached" in str(exc_info.value) + + # Counter should still be 2 (third client failed to initialize) + assert litellm.initialized_langfuse_clients == 2 \ No newline at end of file diff --git a/tests/litellm/llms/azure/test_azure_common_utils.py b/tests/litellm/llms/azure/test_azure_common_utils.py index abdd1cb9294..34f0dc3a973 100644 --- a/tests/litellm/llms/azure/test_azure_common_utils.py +++ b/tests/litellm/llms/azure/test_azure_common_utils.py @@ -391,6 +391,11 @@ def test_select_azure_base_url_called(setup_mocks): "add_message", "arun_thread_stream", "aresponses", + "acreate_fine_tuning_job", + "acancel_fine_tuning_job", + "alist_fine_tuning_jobs", + "aretrieve_fine_tuning_job", + "afile_list", ] ], ) diff --git a/tests/litellm/llms/openrouter/chat/test_openrouter_chat_transformation.py b/tests/litellm/llms/openrouter/chat/test_openrouter_chat_transformation.py index de0b284f0a3..ecdcadfb735 100644 --- a/tests/litellm/llms/openrouter/chat/test_openrouter_chat_transformation.py +++ b/tests/litellm/llms/openrouter/chat/test_openrouter_chat_transformation.py @@ -26,6 +26,11 @@ class TestOpenRouterChatCompletionStreamingHandler: "id": "test_id", "created": 1234567890, "model": "test_model", + "usage": { + "prompt_tokens": 10, + "completion_tokens": 20, + "total_tokens": 30 + }, "choices": [ {"delta": {"content": "test content", "reasoning": "test reasoning"}} ], @@ -39,6 +44,9 @@ class TestOpenRouterChatCompletionStreamingHandler: assert result.object == "chat.completion.chunk" assert result.created == 1234567890 assert result.model == "test_model" + assert result.usage.prompt_tokens == chunk["usage"]["prompt_tokens"] + assert result.usage.completion_tokens == chunk["usage"]["completion_tokens"] + assert result.usage.total_tokens == chunk["usage"]["total_tokens"] assert len(result.choices) == 1 assert result.choices[0]["delta"]["reasoning_content"] == "test reasoning" diff --git a/tests/litellm/proxy/anthropic_endpoints/__init__.py b/tests/litellm/proxy/anthropic_endpoints/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/litellm/proxy/anthropic_endpoints/test_endpoints.py b/tests/litellm/proxy/anthropic_endpoints/test_endpoints.py new file mode 100644 index 00000000000..2dbdf345042 --- /dev/null +++ b/tests/litellm/proxy/anthropic_endpoints/test_endpoints.py @@ -0,0 +1,61 @@ +""" +Test for anthropic_endpoints/endpoints.py, focusing on handling dictionary objects in streaming responses +""" + +import json +import unittest +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest + +from litellm.proxy.anthropic_endpoints.endpoints import async_data_generator_anthropic + + +class TestAnthropicEndpoints(unittest.TestCase): + @patch("litellm.litellm_core_utils.safe_json_dumps.safe_dumps") + @pytest.mark.asyncio + async def test_async_data_generator_anthropic_dict_handling(self, mock_safe_dumps): + """Test async_data_generator_anthropic handles dictionary chunks properly""" + # Setup + mock_response = AsyncMock() + mock_response.__aiter__.return_value = [ + {"type": "message_start", "message": {"id": "msg_123"}}, + "text chunk data", + {"type": "content_block_delta", "delta": {"text": "more data"}}, + "text chunk data again", + ] + + mock_user_api_key_dict = MagicMock() + mock_request_data = {} + mock_proxy_logging_obj = MagicMock() + mock_proxy_logging_obj.async_post_call_streaming_hook = AsyncMock(side_effect=lambda **kwargs: kwargs["response"]) + + # Configure safe_dumps to return a properly formatted JSON string + mock_safe_dumps.side_effect = lambda chunk: json.dumps(chunk) + + # Execute + result = [chunk async for chunk in async_data_generator_anthropic( + response=mock_response, + user_api_key_dict=mock_user_api_key_dict, + request_data=mock_request_data, + proxy_logging_obj=mock_proxy_logging_obj, + )] + + # Verify + expected_result = [ + 'data: {"type": "message_start", "message": {"id": "msg_123"}}\n\n', + 'text chunk data', + 'data: {"type": "content_block_delta", "delta": {"text": "more data"}}\n\n', + 'text chunk data again', + ] + + self.assertEqual(result, expected_result) + + # Assert safe_dumps was called for dictionary objects + mock_safe_dumps.assert_any_call({"type": "message_start", "message": {"id": "msg_123"}}) + mock_safe_dumps.assert_any_call({"type": "content_block_delta", "delta": {"text": "more data"}}) + assert mock_safe_dumps.call_count == 2 # Called twice, once for each dict object + + +if __name__ == "__main__": + unittest.main() \ No newline at end of file diff --git a/tests/litellm/proxy/hooks/test_parallel_request_limiter_v2.py b/tests/litellm/proxy/hooks/test_parallel_request_limiter_v2.py index f0a3497a12d..d97e83b7dbb 100644 --- a/tests/litellm/proxy/hooks/test_parallel_request_limiter_v2.py +++ b/tests/litellm/proxy/hooks/test_parallel_request_limiter_v2.py @@ -19,6 +19,7 @@ from litellm.proxy.hooks.parallel_request_limiter_v2 import ( from litellm.proxy.utils import InternalUsageCache, ProxyLogging, hash_token +@pytest.mark.flaky(reruns=3) @pytest.mark.asyncio async def test_normal_router_call_v2(monkeypatch): """ @@ -340,6 +341,7 @@ async def test_normal_router_call_rpm(monkeypatch, rate_limit_object): ) +@pytest.mark.flaky(reruns=3) @pytest.mark.asyncio async def test_streaming_router_call_v2(monkeypatch): """ diff --git a/tests/litellm/proxy/management_endpoints/test_customer_endpoints.py b/tests/litellm/proxy/management_endpoints/test_customer_endpoints.py new file mode 100644 index 00000000000..6382976c361 --- /dev/null +++ b/tests/litellm/proxy/management_endpoints/test_customer_endpoints.py @@ -0,0 +1,87 @@ +from unittest.mock import AsyncMock, patch + +import pytest +from fastapi import FastAPI, HTTPException +from fastapi.testclient import TestClient + +from litellm.proxy._types import ( + LiteLLM_BudgetTable, + LiteLLM_EndUserTable, + LitellmUserRoles, +) +from litellm.proxy.auth.user_api_key_auth import UserAPIKeyAuth +from litellm.proxy.management_endpoints.customer_endpoints import router +from litellm.proxy.proxy_server import ProxyException + +app = FastAPI() +app.include_router(router) +client = TestClient(app) + + +@pytest.fixture +def mock_prisma_client(): + with patch("litellm.proxy.proxy_server.prisma_client") as mock: + yield mock + + +@pytest.fixture +def mock_user_api_key_auth(): + with patch("litellm.proxy.proxy_server.user_api_key_auth") as mock: + mock.return_value = UserAPIKeyAuth( + user_id="test-user", user_role=LitellmUserRoles.PROXY_ADMIN + ) + yield mock + + +def test_update_customer_success(mock_prisma_client, mock_user_api_key_auth): + # Mock the database responses + mock_end_user = LiteLLM_EndUserTable( + user_id="test-user-1", alias="Test User", blocked=False + ) + updated_mock_end_user = LiteLLM_EndUserTable( + user_id="test-user-1", alias="Updated Test User", blocked=False + ) + + # Mock the find_first response + mock_prisma_client.db.litellm_endusertable.find_first = AsyncMock( + return_value=mock_end_user + ) + + # Mock the update response + mock_prisma_client.db.litellm_endusertable.update = AsyncMock( + return_value=updated_mock_end_user + ) + + # Test data + test_data = {"user_id": "test-user-1", "alias": "Updated Test User"} + + # Make the request + response = client.post( + "/customer/update", json=test_data, headers={"Authorization": "Bearer test-key"} + ) + + # Assert response + assert response.status_code == 200 + assert response.json()["user_id"] == "test-user-1" + assert response.json()["alias"] == "Updated Test User" + + +def test_update_customer_not_found(mock_prisma_client, mock_user_api_key_auth): + # Mock the database response to return None (user not found) + mock_prisma_client.db.litellm_endusertable.find_first = AsyncMock(return_value=None) + + # Test data + test_data = {"user_id": "non-existent-user", "alias": "Test User"} + + # Make the request + try: + response = client.post( + "/customer/update", + json=test_data, + headers={"Authorization": "Bearer test-key"}, + ) + except Exception as e: + print(e, type(e)) + assert isinstance(e, ProxyException) + assert int(e.code) == 400 + assert "End User Id=non-existent-user does not exist in db" in e.message diff --git a/tests/litellm/proxy/test_team_member_update.py b/tests/litellm/proxy/test_team_member_update.py new file mode 100644 index 00000000000..7f534b0b618 --- /dev/null +++ b/tests/litellm/proxy/test_team_member_update.py @@ -0,0 +1,39 @@ +import pytest +from fastapi import HTTPException +from starlette.requests import Request + +from litellm.proxy.management_endpoints.team_endpoints import team_member_update +from litellm.proxy._types import TeamMemberUpdateRequest +import litellm.proxy.proxy_server as proxy_server + +@pytest.mark.asyncio +async def test_team_member_update_admin_requires_premium(monkeypatch): + # Arrange: patch prisma_client and premium_user + monkeypatch.setattr(proxy_server, 'prisma_client', object()) + monkeypatch.setattr(proxy_server, 'premium_user', False) + + # Create a request body that tries to set role=admin + data = TeamMemberUpdateRequest( + team_id="team-1234", + user_id="user-1", + user_email=None, + role="admin", + max_budget_in_team=None, + ) + scope = {"type": "http", "method": "POST", "path": "/team/member_update"} + request = Request(scope) + + # We don't need a full auth object since premium check happens before auth is used + auth = object() + + # Act & Assert: expect HTTPException 400 with the exact premium feature message + with pytest.raises(HTTPException) as exc_info: + await team_member_update(data, request, auth) + + assert exc_info.value.status_code == 400 + expected_msg = ( + "Assigning team admins is a premium feature. You must be a LiteLLM Enterprise user to use this feature. " + "If you have a license please set `LITELLM_LICENSE` in your env. Get a 7 day trial key here: https://www.litellm.ai/#trial. " + "Pricing: https://www.litellm.ai/#pricing" + ) + assert exc_info.value.detail == expected_msg diff --git a/tests/litellm/secret_managers/test_get_azure_ad_token_provider.py b/tests/litellm/secret_managers/test_get_azure_ad_token_provider.py new file mode 100644 index 00000000000..7fd427dae48 --- /dev/null +++ b/tests/litellm/secret_managers/test_get_azure_ad_token_provider.py @@ -0,0 +1,138 @@ +import json +import os +import sys +from typing import Optional +from unittest.mock import MagicMock, patch + +# Adds the grandparent directory to sys.path to allow importing project modules +sys.path.insert(0, os.path.abspath("../..")) + +import pytest + +from litellm.secret_managers.get_azure_ad_token_provider import ( + get_azure_ad_token_provider, +) + + +class TestGetAzureAdTokenProvider: + @patch.dict( + os.environ, + { + "AZURE_CLIENT_ID": "test-client-id", + "AZURE_CLIENT_SECRET": "test-client-secret", + "AZURE_TENANT_ID": "test-tenant-id", + "AZURE_SCOPE": "https://cognitiveservices.azure.com/.default", + "AZURE_CREDENTIAL": "ClientSecretCredential", + }, + ) + @patch("azure.identity.get_bearer_token_provider") + @patch("azure.identity.ClientSecretCredential") + def test_get_azure_ad_token_provider_client_secret_credential( + self, mock_client_secret_credential, mock_get_bearer_token_provider + ): + """Test get_azure_ad_token_provider with ClientSecretCredential.""" + # Mock the Azure identity credential instance + mock_credential_instance = MagicMock() + mock_client_secret_credential.return_value = mock_credential_instance + + # Mock the bearer token provider + mock_token_provider = MagicMock(return_value="mock-token") + mock_get_bearer_token_provider.return_value = mock_token_provider + + # Call the function + result = get_azure_ad_token_provider() + + # Assertions + assert callable(result) + mock_client_secret_credential.assert_called_once_with( + client_id="test-client-id", + client_secret="test-client-secret", + tenant_id="test-tenant-id", + ) + mock_get_bearer_token_provider.assert_called_once_with( + mock_credential_instance, "https://cognitiveservices.azure.com/.default" + ) + + # Test that the returned callable works + token = result() + assert token == "mock-token" + + @patch.dict( + os.environ, + { + "AZURE_CLIENT_ID": "test-client-id", + "AZURE_SCOPE": "https://cognitiveservices.azure.com/.default", + "AZURE_CREDENTIAL": "ManagedIdentityCredential", + }, + ) + @patch("azure.identity.get_bearer_token_provider") + @patch("azure.identity.ManagedIdentityCredential") + def test_get_azure_ad_token_provider_managed_identity_credential( + self, mock_managed_identity_credential, mock_get_bearer_token_provider + ): + """Test get_azure_ad_token_provider with ManagedIdentityCredential.""" + # Mock the Azure identity credential instance + mock_credential_instance = MagicMock() + mock_managed_identity_credential.return_value = mock_credential_instance + + # Mock the bearer token provider + mock_token_provider = MagicMock(return_value="mock-managed-identity-token") + mock_get_bearer_token_provider.return_value = mock_token_provider + + # Call the function + result = get_azure_ad_token_provider() + + # Assertions + assert callable(result) + mock_managed_identity_credential.assert_called_once_with( + client_id="test-client-id" + ) + mock_get_bearer_token_provider.assert_called_once_with( + mock_credential_instance, "https://cognitiveservices.azure.com/.default" + ) + + # Test that the returned callable works + token = result() + assert token == "mock-managed-identity-token" + + @patch.dict( + os.environ, + { + "AZURE_CLIENT_ID": "test-client-id", + "AZURE_TENANT_ID": "test-tenant-id", + "AZURE_CERTIFICATE_PATH": "/path/to/cert.pem", + "AZURE_SCOPE": "https://cognitiveservices.azure.com/.default", + "AZURE_CREDENTIAL": "CertificateCredential", + }, + ) + @patch("azure.identity.get_bearer_token_provider") + @patch("azure.identity.CertificateCredential") + def test_get_azure_ad_token_provider_certificate_credential( + self, mock_certificate_credential, mock_get_bearer_token_provider + ): + """Test get_azure_ad_token_provider with CertificateCredential.""" + # Mock the Azure identity credential instance + mock_credential_instance = MagicMock() + mock_certificate_credential.return_value = mock_credential_instance + + # Mock the bearer token provider + mock_token_provider = MagicMock(return_value="mock-certificate-token") + mock_get_bearer_token_provider.return_value = mock_token_provider + + # Call the function + result = get_azure_ad_token_provider() + + # Assertions + assert callable(result) + mock_certificate_credential.assert_called_once_with( + client_id="test-client-id", + tenant_id="test-tenant-id", + certificate_path="/path/to/cert.pem", + ) + mock_get_bearer_token_provider.assert_called_once_with( + mock_credential_instance, "https://cognitiveservices.azure.com/.default" + ) + + # Test that the returned callable works + token = result() + assert token == "mock-certificate-token" diff --git a/tests/llm_responses_api_testing/test_openai_responses_api.py b/tests/llm_responses_api_testing/test_openai_responses_api.py index 333f99f9d46..4c3998e0c31 100644 --- a/tests/llm_responses_api_testing/test_openai_responses_api.py +++ b/tests/llm_responses_api_testing/test_openai_responses_api.py @@ -2,7 +2,7 @@ import os import sys import pytest import asyncio -from typing import Optional +from typing import Optional, cast from unittest.mock import patch, AsyncMock sys.path.insert(0, os.path.abspath("../..")) @@ -1032,4 +1032,51 @@ def test_basic_computer_use_preview_tool_call(): # Validate the input format assert isinstance(request_body["input"], str) assert request_body["input"] == "Check the latest OpenAI news on bing.com." - \ No newline at end of file + + + +def test_mcp_tools_with_responses_api(): + litellm._turn_on_debug() + MCP_TOOLS = [ + { + "type": "mcp", + "server_label": "deepwiki", + "server_url": "https://mcp.deepwiki.com/mcp", + "allowed_tools": ["ask_question"] + } + ] + MODEL = "openai/gpt-4.1" + USER_QUERY = "What transport protocols does the 2025-03-26 version of the MCP spec (modelcontextprotocol/modelcontextprotocol) support?" + ######################################################### + # Step 1: OpenAI will use MCP LIST, and return a list of MCP calls for our approval + response = litellm.responses( + model=MODEL, + tools=MCP_TOOLS, + input=USER_QUERY + ) + print(response) + + response = cast(ResponsesAPIResponse, response) + + mcp_approval_id: Optional[str] + for output in response.output: + if output.type == "mcp_approval_request": + mcp_approval_id = output.id + break + + # Step 2: Send followup with approval for the MCP call + response_with_mcp_call = litellm.responses( + model=MODEL, + tools=MCP_TOOLS, + input=[ + { + "type": "mcp_approval_response", + "approve": True, + "approval_request_id": mcp_approval_id + } + ], + previous_response_id=response.id, + ) + print(response_with_mcp_call) + + diff --git a/tests/local_testing/test_custom_llm.py b/tests/local_testing/test_custom_llm.py index beb1e3332dd..77f4544afa1 100644 --- a/tests/local_testing/test_custom_llm.py +++ b/tests/local_testing/test_custom_llm.py @@ -44,7 +44,7 @@ from litellm import ( image_generation, ) from litellm.utils import ModelResponseIterator -from litellm.types.utils import ImageResponse, ImageObject +from litellm.types.utils import ImageResponse, ImageObject, EmbeddingResponse from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler @@ -257,6 +257,53 @@ class MyCustomLLM(CustomLLM): response_ms=1000, ) + def embedding( + self, + model: str, + input: list, + model_response: EmbeddingResponse, + print_verbose: Callable, + logging_obj: Any, + optional_params: dict, + litellm_params=None, + aembedding=None, + ) -> EmbeddingResponse: + model_response.model = model + + model_response.data = [ + { + "object": "embedding", + "embedding": [0.1, 0.2, 0.3], + "index": i, + } + for i, _ in enumerate(input) + ] + + return model_response + + async def aembedding( + self, + model: str, + input: list, + model_response: EmbeddingResponse, + print_verbose: Callable, + logging_obj: Any, + optional_params: dict, + litellm_params=None, + ) -> EmbeddingResponse: + model_response.model = model + + model_response.data = [ + { + "object": "embedding", + "embedding": [0.1, 0.2, 0.3], + "index": i, + } + for i, _ in enumerate(input) + ] + + return model_response + def test_get_llm_provider(): """""" @@ -452,3 +499,36 @@ def test_get_supported_openai_params(): response = get_supported_openai_params(model="my-custom-llm/my-fake-model") assert response is not None + +def test_simple_embedding(): + my_custom_llm = MyCustomLLM() + litellm.custom_provider_map = [ + {"provider": "custom_llm", "custom_handler": my_custom_llm} + ] + resp = litellm.embedding( + model="custom_llm/my-fake-model", + input=["good morning from litellm", "good night from litellm"] + ) + + assert resp.data[1] == { + "object": "embedding", + "embedding": [0.1, 0.2, 0.3], + "index": 1, + } + +@pytest.mark.asyncio +async def test_simple_aembedding(): + my_custom_llm = MyCustomLLM() + litellm.custom_provider_map = [ + {"provider": "custom_llm", "custom_handler": my_custom_llm} + ] + resp = await litellm.aembedding( + model="custom_llm/my-fake-model", + input=["good morning from litellm", "good night from litellm"] + ) + + assert resp.data[1] == { + "object": "embedding", + "embedding": [0.1, 0.2, 0.3], + "index": 1, + } \ No newline at end of file diff --git a/tests/local_testing/test_stream_chunk_builder.py b/tests/local_testing/test_stream_chunk_builder.py index 14ff8278505..63907eb7d5e 100644 --- a/tests/local_testing/test_stream_chunk_builder.py +++ b/tests/local_testing/test_stream_chunk_builder.py @@ -931,9 +931,6 @@ def execute_completion(opts: dict): print("\n\n") assembly = litellm.stream_chunk_builder(partial_streaming_chunks) print(f"assembly.choices[0].message.tool_calls: {assembly.choices[0].message.tool_calls}") - assert len(assembly.choices[0].message.tool_calls) == 3, ( - assembly.choices[0].message.tool_calls[0].function.arguments[0] - ) print(assembly.choices[0].message.tool_calls) for tool_call in assembly.choices[0].message.tool_calls: json.loads(tool_call.function.arguments) # assert valid json - https://github.com/BerriAI/litellm/issues/10034 diff --git a/tests/logging_callback_tests/langfuse_expected_request_body/completion.json b/tests/logging_callback_tests/langfuse_expected_request_body/completion.json index 4dfe9630ff9..b2a2c83b51a 100644 --- a/tests/logging_callback_tests/langfuse_expected_request_body/completion.json +++ b/tests/logging_callback_tests/langfuse_expected_request_body/completion.json @@ -62,9 +62,7 @@ "endTime": "2025-01-16T11:28:55.124353-08:00", "completionStartTime": "2025-01-16T11:28:55.124353-08:00", "model": "gpt-3.5-turbo", - "modelParameters": { - "extra_body": "{}" - }, + "modelParameters": {}, "usage": { "input": 10, "output": 20, diff --git a/tests/logging_callback_tests/langfuse_expected_request_body/completion_with_complex_metadata.json b/tests/logging_callback_tests/langfuse_expected_request_body/completion_with_complex_metadata.json index 4c5f345eaa5..9d30a82b8d2 100644 --- a/tests/logging_callback_tests/langfuse_expected_request_body/completion_with_complex_metadata.json +++ b/tests/logging_callback_tests/langfuse_expected_request_body/completion_with_complex_metadata.json @@ -103,9 +103,7 @@ "endTime": "2025-01-22T09:27:51.702048-08:00", "completionStartTime": "2025-01-22T09:27:51.702048-08:00", "model": "gpt-3.5-turbo", - "modelParameters": { - "extra_body": "{}" - }, + "modelParameters": {}, "usage": { "input": 10, "output": 20, diff --git a/tests/logging_callback_tests/langfuse_expected_request_body/completion_with_langfuse_metadata.json b/tests/logging_callback_tests/langfuse_expected_request_body/completion_with_langfuse_metadata.json index d4882c962d8..7c2fc6c5f35 100644 --- a/tests/logging_callback_tests/langfuse_expected_request_body/completion_with_langfuse_metadata.json +++ b/tests/logging_callback_tests/langfuse_expected_request_body/completion_with_langfuse_metadata.json @@ -81,9 +81,7 @@ "endTime": "2025-01-22T09:19:11.234200-08:00", "completionStartTime": "2025-01-22T09:19:11.234200-08:00", "model": "gpt-3.5-turbo", - "modelParameters": { - "extra_body": "{}" - }, + "modelParameters": {}, "usage": { "input": 10, "output": 20, diff --git a/tests/logging_callback_tests/langfuse_expected_request_body/completion_with_no_choices.json b/tests/logging_callback_tests/langfuse_expected_request_body/completion_with_no_choices.json index 0683ff9ba9f..cb9f007c2d5 100644 --- a/tests/logging_callback_tests/langfuse_expected_request_body/completion_with_no_choices.json +++ b/tests/logging_callback_tests/langfuse_expected_request_body/completion_with_no_choices.json @@ -52,9 +52,7 @@ "endTime": "2025-02-06T16:23:27.644253-08:00", "completionStartTime": "2025-02-06T16:23:27.644253-08:00", "model": "gpt-3.5-turbo", - "modelParameters": { - "extra_body": "{}" - }, + "modelParameters": {}, "usage": { "input": 10, "output": 10, diff --git a/tests/logging_callback_tests/langfuse_expected_request_body/completion_with_tags.json b/tests/logging_callback_tests/langfuse_expected_request_body/completion_with_tags.json index 3a87c0ad739..c4cbe1e68af 100644 --- a/tests/logging_callback_tests/langfuse_expected_request_body/completion_with_tags.json +++ b/tests/logging_callback_tests/langfuse_expected_request_body/completion_with_tags.json @@ -71,9 +71,7 @@ "endTime": "2025-01-22T07:31:28.962389-08:00", "completionStartTime": "2025-01-22T07:31:28.962389-08:00", "model": "gpt-3.5-turbo", - "modelParameters": { - "extra_body": "{}" - }, + "modelParameters": {}, "usage": { "input": 10, "output": 20, diff --git a/tests/logging_callback_tests/langfuse_expected_request_body/completion_with_tags_stream.json b/tests/logging_callback_tests/langfuse_expected_request_body/completion_with_tags_stream.json index 6495ed947d6..cd882af614d 100644 --- a/tests/logging_callback_tests/langfuse_expected_request_body/completion_with_tags_stream.json +++ b/tests/logging_callback_tests/langfuse_expected_request_body/completion_with_tags_stream.json @@ -71,9 +71,7 @@ "endTime": "2025-01-22T08:38:26.015666-08:00", "completionStartTime": "2025-01-22T08:38:26.015666-08:00", "model": "gpt-3.5-turbo", - "modelParameters": { - "extra_body": "{}" - }, + "modelParameters": {}, "usage": { "input": 10, "output": 20, diff --git a/tests/logging_callback_tests/langfuse_expected_request_body/complex_metadata.json b/tests/logging_callback_tests/langfuse_expected_request_body/complex_metadata.json index 01dcd264883..5c8d5c5b88d 100644 --- a/tests/logging_callback_tests/langfuse_expected_request_body/complex_metadata.json +++ b/tests/logging_callback_tests/langfuse_expected_request_body/complex_metadata.json @@ -78,9 +78,7 @@ "endTime": "2025-01-22T09:59:39.365756-08:00", "completionStartTime": "2025-01-22T09:59:39.365756-08:00", "model": "gpt-3.5-turbo", - "modelParameters": { - "extra_body": "{}" - }, + "modelParameters": {}, "usage": { "input": 10, "output": 20, diff --git a/tests/logging_callback_tests/langfuse_expected_request_body/complex_metadata_2.json b/tests/logging_callback_tests/langfuse_expected_request_body/complex_metadata_2.json index 1b7b91930e9..4533262ef42 100644 --- a/tests/logging_callback_tests/langfuse_expected_request_body/complex_metadata_2.json +++ b/tests/logging_callback_tests/langfuse_expected_request_body/complex_metadata_2.json @@ -70,9 +70,7 @@ "endTime": "2025-01-22T10:06:50.958374-08:00", "completionStartTime": "2025-01-22T10:06:50.958374-08:00", "model": "gpt-3.5-turbo", - "modelParameters": { - "extra_body": "{}" - }, + "modelParameters": {}, "usage": { "input": 10, "output": 20, diff --git a/tests/logging_callback_tests/langfuse_expected_request_body/empty_metadata.json b/tests/logging_callback_tests/langfuse_expected_request_body/empty_metadata.json index 8c1711ee98e..39a88320bbf 100644 --- a/tests/logging_callback_tests/langfuse_expected_request_body/empty_metadata.json +++ b/tests/logging_callback_tests/langfuse_expected_request_body/empty_metadata.json @@ -64,9 +64,7 @@ "endTime": "2025-01-22T09:59:32.880691-08:00", "completionStartTime": "2025-01-22T09:59:32.880691-08:00", "model": "gpt-3.5-turbo", - "modelParameters": { - "extra_body": "{}" - }, + "modelParameters": {}, "usage": { "input": 10, "output": 20, diff --git a/tests/logging_callback_tests/langfuse_expected_request_body/metadata_with_function.json b/tests/logging_callback_tests/langfuse_expected_request_body/metadata_with_function.json index 0b1309425e3..e73ef0d9ed6 100644 --- a/tests/logging_callback_tests/langfuse_expected_request_body/metadata_with_function.json +++ b/tests/logging_callback_tests/langfuse_expected_request_body/metadata_with_function.json @@ -64,9 +64,7 @@ "endTime": "2025-01-22T09:59:36.161959-08:00", "completionStartTime": "2025-01-22T09:59:36.161959-08:00", "model": "gpt-3.5-turbo", - "modelParameters": { - "extra_body": "{}" - }, + "modelParameters": {}, "usage": { "input": 10, "output": 20, diff --git a/tests/logging_callback_tests/langfuse_expected_request_body/metadata_with_lock.json b/tests/logging_callback_tests/langfuse_expected_request_body/metadata_with_lock.json index 8c1711ee98e..39a88320bbf 100644 --- a/tests/logging_callback_tests/langfuse_expected_request_body/metadata_with_lock.json +++ b/tests/logging_callback_tests/langfuse_expected_request_body/metadata_with_lock.json @@ -64,9 +64,7 @@ "endTime": "2025-01-22T09:59:32.880691-08:00", "completionStartTime": "2025-01-22T09:59:32.880691-08:00", "model": "gpt-3.5-turbo", - "modelParameters": { - "extra_body": "{}" - }, + "modelParameters": {}, "usage": { "input": 10, "output": 20, diff --git a/tests/logging_callback_tests/langfuse_expected_request_body/nested_metadata.json b/tests/logging_callback_tests/langfuse_expected_request_body/nested_metadata.json index bb24688aa5c..efd3bbae323 100644 --- a/tests/logging_callback_tests/langfuse_expected_request_body/nested_metadata.json +++ b/tests/logging_callback_tests/langfuse_expected_request_body/nested_metadata.json @@ -70,9 +70,7 @@ "endTime": "2025-01-22T09:55:28.853979-08:00", "completionStartTime": "2025-01-22T09:55:28.853979-08:00", "model": "gpt-3.5-turbo", - "modelParameters": { - "extra_body": "{}" - }, + "modelParameters": {}, "usage": { "input": 10, "output": 20, diff --git a/tests/logging_callback_tests/langfuse_expected_request_body/simple_metadata.json b/tests/logging_callback_tests/langfuse_expected_request_body/simple_metadata.json index d40ec6bafca..8cb1cced89d 100644 --- a/tests/logging_callback_tests/langfuse_expected_request_body/simple_metadata.json +++ b/tests/logging_callback_tests/langfuse_expected_request_body/simple_metadata.json @@ -70,9 +70,7 @@ "endTime": "2025-01-22T09:53:53.753431-08:00", "completionStartTime": "2025-01-22T09:53:53.753431-08:00", "model": "gpt-3.5-turbo", - "modelParameters": { - "extra_body": "{}" - }, + "modelParameters": {}, "usage": { "input": 10, "output": 20, diff --git a/tests/logging_callback_tests/langfuse_expected_request_body/simple_metadata2.json b/tests/logging_callback_tests/langfuse_expected_request_body/simple_metadata2.json index 610bc461a13..0de688644b9 100644 --- a/tests/logging_callback_tests/langfuse_expected_request_body/simple_metadata2.json +++ b/tests/logging_callback_tests/langfuse_expected_request_body/simple_metadata2.json @@ -74,9 +74,7 @@ "endTime": "2025-01-22T09:56:35.476236-08:00", "completionStartTime": "2025-01-22T09:56:35.476236-08:00", "model": "gpt-3.5-turbo", - "modelParameters": { - "extra_body": "{}" - }, + "modelParameters": {}, "usage": { "input": 10, "output": 20, diff --git a/tests/logging_callback_tests/langfuse_expected_request_body/simple_metadata3.json b/tests/logging_callback_tests/langfuse_expected_request_body/simple_metadata3.json index d21c58fdee4..f0ad3e9e712 100644 --- a/tests/logging_callback_tests/langfuse_expected_request_body/simple_metadata3.json +++ b/tests/logging_callback_tests/langfuse_expected_request_body/simple_metadata3.json @@ -78,9 +78,7 @@ "endTime": "2025-01-22T09:56:38.785762-08:00", "completionStartTime": "2025-01-22T09:56:38.785762-08:00", "model": "gpt-3.5-turbo", - "modelParameters": { - "extra_body": "{}" - }, + "modelParameters": {}, "usage": { "input": 10, "output": 20, diff --git a/ui/litellm-dashboard/src/components/common_components/filter.tsx b/ui/litellm-dashboard/src/components/common_components/filter.tsx index 067fadfc6ba..8351ea791ad 100644 --- a/ui/litellm-dashboard/src/components/common_components/filter.tsx +++ b/ui/litellm-dashboard/src/components/common_components/filter.tsx @@ -88,6 +88,7 @@ const FilterComponent: React.FC = ({ "Key Alias", "User ID", "Key Hash", + "Model" ]; return ( diff --git a/ui/litellm-dashboard/src/components/networking.tsx b/ui/litellm-dashboard/src/components/networking.tsx index 86d63cfc17b..ba9d1cb34a5 100644 --- a/ui/litellm-dashboard/src/components/networking.tsx +++ b/ui/litellm-dashboard/src/components/networking.tsx @@ -3358,14 +3358,15 @@ export interface Member { export const teamMemberAddCall = async ( accessToken: string, teamId: string, - formValues: Member // Assuming formValues is an object + formValues: Member ) => { try { - console.log("Form Values in teamMemberAddCall:", formValues); // Log the form values before making the API call + console.log("Form Values in teamMemberAddCall:", formValues); const url = proxyBaseUrl ? `${proxyBaseUrl}/team/member_add` : `/team/member_add`; + const response = await fetch(url, { method: "POST", headers: { @@ -3374,21 +3375,30 @@ export const teamMemberAddCall = async ( }, body: JSON.stringify({ team_id: teamId, - member: formValues, // Include formValues in the request body + member: formValues, }), }); if (!response.ok) { - const errorData = await response.text(); - handleError(errorData); - console.error("Error response from the server:", errorData); - throw new Error("Network response was not ok"); + // Read and parse JSON error body + const errorText = await response.text(); + let parsedError: any = {}; + + try { + parsedError = JSON.parse(errorText); + } catch (e) { + console.warn("Failed to parse error body as JSON:", errorText); + } + + const rawMessage = parsedError?.detail?.error || "Failed to add team member"; + const err = new Error(rawMessage); + (err as any).raw = parsedError; + throw err; } const data = await response.json(); console.log("API Response:", data); return data; - // Handle success - you might want to update some state or UI based on the created key } catch (error) { console.error("Failed to create key:", error); throw error; @@ -3401,7 +3411,7 @@ export const teamMemberUpdateCall = async ( formValues: Member // Assuming formValues is an object ) => { try { - console.log("Form Values in teamMemberAddCall:", formValues); // Log the form values before making the API call + console.log("Form Values in teamMemberUpdateCall:", formValues); const url = proxyBaseUrl ? `${proxyBaseUrl}/team/member_update` @@ -3420,21 +3430,30 @@ export const teamMemberUpdateCall = async ( }); if (!response.ok) { - const errorData = await response.text(); - handleError(errorData); - console.error("Error response from the server:", errorData); - throw new Error("Network response was not ok"); + // Read and parse JSON error body + const errorText = await response.text(); + let parsedError: any = {}; + + try { + parsedError = JSON.parse(errorText); + } catch (e) { + console.warn("Failed to parse error body as JSON:", errorText); + } + + const rawMessage = parsedError?.detail?.error || "Failed to add team member"; + const err = new Error(rawMessage); + (err as any).raw = parsedError; + throw err; } const data = await response.json(); console.log("API Response:", data); return data; - // Handle success - you might want to update some state or UI based on the created key } catch (error) { - console.error("Failed to create key:", error); + console.error("Failed to update team member:", error); throw error; } -} +}; export const teamMemberDeleteCall = async ( accessToken: string, diff --git a/ui/litellm-dashboard/src/components/organizations.tsx b/ui/litellm-dashboard/src/components/organizations.tsx index f9ee248b82d..86a62ac42c5 100644 --- a/ui/litellm-dashboard/src/components/organizations.tsx +++ b/ui/litellm-dashboard/src/components/organizations.tsx @@ -288,7 +288,7 @@ const OrganizationsTable: React.FC = ({ {org.created_at ? new Date(org.created_at).toLocaleDateString() : "N/A"} - {org.spend} + {org.spend.toFixed(4)} {org.litellm_budget_table?.max_budget !== null && org.litellm_budget_table?.max_budget !== undefined ? org.litellm_budget_table?.max_budget : "No limit"} @@ -353,13 +353,86 @@ const OrganizationsTable: React.FC = ({ + {(userRole === "Admin" || userRole === "Org Admin") && ( + + + +
+ + + + + + + All Proxy Models + + {userModels && userModels.length > 0 && userModels.map((model) => ( + + {getModelDisplayName(model)} + + ))} + + + + + + + + + daily + weekly + monthly + + + + + + + + + + + + + +
+ +
+
+ + + )} - {isDeleteModalOpen ? (
@@ -396,6 +469,7 @@ const OrganizationsTable: React.FC = ({
) : <>} + ); }; diff --git a/ui/litellm-dashboard/src/components/team/edit_membership.tsx b/ui/litellm-dashboard/src/components/team/edit_membership.tsx index 27e64193e35..5ffc1dd5945 100644 --- a/ui/litellm-dashboard/src/components/team/edit_membership.tsx +++ b/ui/litellm-dashboard/src/components/team/edit_membership.tsx @@ -78,9 +78,9 @@ const MemberModal = ({ onSubmit(formData); form.resetFields(); - message.success(`Successfully ${mode === 'add' ? 'added' : 'updated'} member`); + // message.success(`Successfully ${mode === 'add' ? 'added' : 'updated'} member`); } catch (error) { - message.error('Failed to submit form'); + // message.error('Failed to submit form'); console.error('Form submission error:', error); } }; diff --git a/ui/litellm-dashboard/src/components/team/team_info.tsx b/ui/litellm-dashboard/src/components/team/team_info.tsx index f195e0845df..d502097b8f5 100644 --- a/ui/litellm-dashboard/src/components/team/team_info.tsx +++ b/ui/litellm-dashboard/src/components/team/team_info.tsx @@ -115,26 +115,36 @@ const TeamInfoView: React.FC = ({ const handleMemberCreate = async (values: any) => { try { - if (accessToken == null) { - return; - } - + if (accessToken == null) return; + const member: Member = { user_email: values.user_email, user_id: values.user_id, role: values.role, - } - const response = await teamMemberAddCall(accessToken, teamId, member); - + }; + + await teamMemberAddCall(accessToken, teamId, member); + message.success("Team member added successfully"); setIsAddMemberModalVisible(false); form.resetFields(); fetchTeamInfo(); - } catch (error) { - message.error("Failed to add team member"); + } catch (error: any) { + let errMsg = "Failed to add team member"; + + if (error?.raw?.detail?.error?.includes("Assigning team admins is a premium feature")) { + errMsg = "Assigning admins is an enterprise-only feature. Please upgrade your LiteLLM plan to enable this."; + } else if (error?.message) { + errMsg = error.message; + } + + message.error(errMsg); console.error("Error adding team member:", error); } }; + + + const handleMemberUpdate = async (values: any) => { try { @@ -147,17 +157,29 @@ const TeamInfoView: React.FC = ({ user_id: values.user_id, role: values.role, } + message.destroy(); // Remove all existing toasts - const response = await teamMemberUpdateCall(accessToken, teamId, member); + await teamMemberUpdateCall(accessToken, teamId, member); message.success("Team member updated successfully"); setIsEditMemberModalVisible(false); fetchTeamInfo(); - } catch (error) { - message.error("Failed to update team member"); + } catch (error: any) { + let errMsg = "Failed to update team member"; + if (error?.raw?.detail?.includes("Assigning team admins is a premium feature")) { + errMsg = "Assigning admins is an enterprise-only feature. Please upgrade your LiteLLM plan to enable this."; + } else if (error?.message) { + errMsg = error.message; + } + setIsEditMemberModalVisible(false); + + message.destroy(); // Remove all existing toasts + + message.error(errMsg); console.error("Error updating team member:", error); } }; + const handleMemberDelete = async (member: Member) => { try { diff --git a/ui/litellm-dashboard/src/components/view_logs/index.tsx b/ui/litellm-dashboard/src/components/view_logs/index.tsx index 2c05c3138d7..b6be331aba5 100644 --- a/ui/litellm-dashboard/src/components/view_logs/index.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/index.tsx @@ -226,7 +226,6 @@ export default function SpendLogsTable({ filteredLogs, allTeams: hookAllTeams, allKeyAliases, - allModels, handleFilterChange, handleFilterReset } = useLogFilterLogic({ @@ -408,17 +407,7 @@ export default function SpendLogsTable({ { name: 'Model', label: 'Model', - isSearchable: true, - searchFn: async (searchText: string) => { - if (!allModels || allModels.length === 0) return []; - const filtered = allModels.filter((model: string) => { - return model.toLowerCase().includes(searchText.toLowerCase()); - }); - return filtered.map((model: string) => ({ - label: model, - value: model - })); - } + isSearchable: false, }, { name: 'Key Alias', @@ -440,7 +429,7 @@ export default function SpendLogsTable({ name: 'Key Hash', label: 'Key Hash', isSearchable: false, - } + }, ] // When a session is selected, render the SessionView component diff --git a/ui/litellm-dashboard/src/components/view_logs/log_filter_logic.tsx b/ui/litellm-dashboard/src/components/view_logs/log_filter_logic.tsx index 1f2b765ded7..e61bfbf109b 100644 --- a/ui/litellm-dashboard/src/components/view_logs/log_filter_logic.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/log_filter_logic.tsx @@ -198,24 +198,6 @@ export function useLogFilterLogic({ enabled: !!accessToken, }); - const { data: allModels = [] } = useQuery({ - queryKey: ['allModels', accessToken, userID, userRole], - queryFn: async () => { - if (!accessToken || !userID || !userRole) return []; - - const response = await modelAvailableCall( - accessToken, - userID, - userRole, - false, // return_wildcard_routes - null // teamID - ); - - return response.data.map((model: { id: string }) => model.id); - }, - enabled: !!accessToken && !!userID && !!userRole, - }); - // Update filters state const handleFilterChange = (newFilters: Partial) => { setFilters(prev => { @@ -252,7 +234,6 @@ export function useLogFilterLogic({ filteredLogs, allKeyAliases, allTeams, - allModels, handleFilterChange, handleFilterReset, }; diff --git a/ui/litellm-dashboard/src/components/view_users/columns.tsx b/ui/litellm-dashboard/src/components/view_users/columns.tsx index 6abf7680344..3a5f9c3cbb6 100644 --- a/ui/litellm-dashboard/src/components/view_users/columns.tsx +++ b/ui/litellm-dashboard/src/components/view_users/columns.tsx @@ -39,7 +39,7 @@ export const columns = ( accessorKey: "spend", cell: ({ row }) => ( - {row.original.spend ? row.original.spend.toFixed(2) : "-"} + {row.original.spend ? row.original.spend.toFixed(4) : "-"} ), },