diff --git a/.circleci/config.yml b/.circleci/config.yml index 867accaf05a..476f138b1d4 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1465,7 +1465,7 @@ jobs: - run: name: Run core tests command: | - python -m pytest tests/test_litellm --ignore=tests/test_litellm/proxy --ignore=tests/test_litellm/llms --cov=litellm --cov-report=xml --junitxml=test-results/junit-core.xml --durations=10 -n 16 --maxfail=5 --timeout=300 -vv --log-cli-level=WARNING + python -m pytest tests/test_litellm --ignore=tests/test_litellm/proxy --ignore=tests/test_litellm/llms --ignore=tests/test_litellm/integrations --ignore=tests/test_litellm/litellm_core_utils --cov=litellm --cov-report=xml --junitxml=test-results/junit-core.xml --durations=10 -n 16 --maxfail=5 --timeout=300 -vv --log-cli-level=WARNING no_output_timeout: 120m - run: name: Rename the coverage files @@ -1479,6 +1479,60 @@ jobs: paths: - litellm_core_tests_coverage.xml - litellm_core_tests_coverage + litellm_mapped_tests_litellm_core_utils: + docker: + - image: cimg/python:3.11 + auth: + username: ${DOCKERHUB_USERNAME} + password: ${DOCKERHUB_PASSWORD} + working_directory: ~/project + resource_class: xlarge + steps: + - setup_litellm_test_deps + - run: + name: Run litellm_core_utils tests + command: | + python -m pytest tests/test_litellm/litellm_core_utils --cov=litellm --cov-report=xml --junitxml=test-results/junit-litellm-core-utils.xml --durations=10 -n 16 --maxfail=5 --timeout=300 -vv --log-cli-level=WARNING + no_output_timeout: 120m + - run: + name: Rename the coverage files + command: | + mv coverage.xml litellm_core_utils_tests_coverage.xml + mv .coverage litellm_core_utils_tests_coverage + - store_test_results: + path: test-results + - persist_to_workspace: + root: . + paths: + - litellm_core_utils_tests_coverage.xml + - litellm_core_utils_tests_coverage + litellm_mapped_tests_integrations: + docker: + - image: cimg/python:3.11 + auth: + username: ${DOCKERHUB_USERNAME} + password: ${DOCKERHUB_PASSWORD} + working_directory: ~/project + resource_class: xlarge + steps: + - setup_litellm_test_deps + - run: + name: Run integrations tests + command: | + python -m pytest tests/test_litellm/integrations --cov=litellm --cov-report=xml --junitxml=test-results/junit-integrations.xml --durations=10 -n 16 --maxfail=5 --timeout=300 -vv --log-cli-level=WARNING + no_output_timeout: 120m + - run: + name: Rename the coverage files + command: | + mv coverage.xml litellm_integrations_tests_coverage.xml + mv .coverage litellm_integrations_tests_coverage + - store_test_results: + path: test-results + - persist_to_workspace: + root: . + paths: + - litellm_integrations_tests_coverage.xml + - litellm_integrations_tests_coverage litellm_mapped_enterprise_tests: docker: - image: cimg/python:3.11 @@ -1960,6 +2014,7 @@ jobs: - run: ruff check ./litellm # - run: python ./tests/documentation_tests/test_general_setting_keys.py - run: python ./tests/code_coverage_tests/check_licenses.py + - run: python ./tests/code_coverage_tests/check_provider_folders_documented.py - run: python ./tests/code_coverage_tests/router_code_coverage.py - run: python ./tests/code_coverage_tests/test_chat_completion_imports.py - run: python ./tests/code_coverage_tests/info_log_check.py @@ -3871,6 +3926,18 @@ workflows: only: - main - /litellm_.*/ + - litellm_mapped_tests_integrations: + filters: + branches: + only: + - main + - /litellm_.*/ + - litellm_mapped_tests_litellm_core_utils: + filters: + branches: + only: + - main + - /litellm_.*/ - batches_testing: filters: branches: @@ -3919,6 +3986,8 @@ workflows: - litellm_mapped_tests_proxy - litellm_mapped_tests_llms - litellm_mapped_tests_core + - litellm_mapped_tests_integrations + - litellm_mapped_tests_litellm_core_utils - litellm_mapped_enterprise_tests - batches_testing - litellm_utils_testing @@ -3990,6 +4059,8 @@ workflows: - litellm_mapped_tests_proxy - litellm_mapped_tests_llms - litellm_mapped_tests_core + - litellm_mapped_tests_integrations + - litellm_mapped_tests_litellm_core_utils - litellm_mapped_enterprise_tests - batches_testing - litellm_utils_testing diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml index 39b46cba999..905ebd3dba4 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.yml +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -27,6 +27,7 @@ body: attributes: label: What part of LiteLLM is this about? options: + - '' - "SDK (litellm Python package)" - "Proxy" - "UI Dashboard" diff --git a/.github/ISSUE_TEMPLATE/feature_request.yml b/.github/ISSUE_TEMPLATE/feature_request.yml index 96b95cc7f02..e575db7302a 100644 --- a/.github/ISSUE_TEMPLATE/feature_request.yml +++ b/.github/ISSUE_TEMPLATE/feature_request.yml @@ -27,6 +27,7 @@ body: attributes: label: What part of LiteLLM is this about? options: + - '' - "SDK (litellm Python package)" - "Proxy" - "UI Dashboard" diff --git a/.github/workflows/label-component.yml b/.github/workflows/label-component.yml index c0f9436288c..9a547c162a6 100644 --- a/.github/workflows/label-component.yml +++ b/.github/workflows/label-component.yml @@ -12,7 +12,7 @@ jobs: issues: write steps: - name: Add SDK label - if: contains(github.event.issue.body, 'SDK (litellm Python package)') + if: contains(github.event.issue.body, 'What part of LiteLLM is this about?\n\nSDK (litellm Python package)') uses: actions/github-script@v7 with: github-token: ${{ secrets.GITHUB_TOKEN }} @@ -45,7 +45,7 @@ jobs: }); - name: Add Proxy label - if: contains(github.event.issue.body, 'Proxy') + if: contains(github.event.issue.body, 'What part of LiteLLM is this about?\n\nProxy') uses: actions/github-script@v7 with: github-token: ${{ secrets.GITHUB_TOKEN }} @@ -78,7 +78,7 @@ jobs: }); - name: Add UI Dashboard label - if: contains(github.event.issue.body, 'UI Dashboard') + if: contains(github.event.issue.body, 'What part of LiteLLM is this about?\n\nUI Dashboard') uses: actions/github-script@v7 with: github-token: ${{ secrets.GITHUB_TOKEN }} @@ -111,7 +111,7 @@ jobs: }); - name: Add Docs label - if: contains(github.event.issue.body, 'Docs') + if: contains(github.event.issue.body, 'What part of LiteLLM is this about?\n\nDocs') uses: actions/github-script@v7 with: github-token: ${{ secrets.GITHUB_TOKEN }} diff --git a/Dockerfile b/Dockerfile index d8397ec4811..0e7a8412bbc 100644 --- a/Dockerfile +++ b/Dockerfile @@ -20,7 +20,8 @@ RUN python -m pip install build COPY . . # Build Admin UI -RUN chmod +x docker/build_admin_ui.sh && ./docker/build_admin_ui.sh +# Convert Windows line endings to Unix and make executable +RUN sed -i 's/\r$//' docker/build_admin_ui.sh && chmod +x docker/build_admin_ui.sh && ./docker/build_admin_ui.sh # Build the package RUN rm -rf dist/* && python -m build @@ -65,12 +66,14 @@ RUN find /usr/lib -type f -path "*/tornado/test/*" -delete && \ find /usr/lib -type d -path "*/tornado/test" -delete # Install semantic_router and aurelio-sdk using script -RUN chmod +x docker/install_auto_router.sh && ./docker/install_auto_router.sh +# Convert Windows line endings to Unix and make executable +RUN sed -i 's/\r$//' docker/install_auto_router.sh && chmod +x docker/install_auto_router.sh && ./docker/install_auto_router.sh # Generate prisma client RUN prisma generate -RUN chmod +x docker/entrypoint.sh -RUN chmod +x docker/prod_entrypoint.sh +# Convert Windows line endings to Unix for entrypoint scripts +RUN sed -i 's/\r$//' docker/entrypoint.sh && chmod +x docker/entrypoint.sh +RUN sed -i 's/\r$//' docker/prod_entrypoint.sh && chmod +x docker/prod_entrypoint.sh EXPOSE 4000/tcp diff --git a/ci_cd/security_scans.sh b/ci_cd/security_scans.sh index 276f5abe330..be9167adda2 100755 --- a/ci_cd/security_scans.sh +++ b/ci_cd/security_scans.sh @@ -34,47 +34,47 @@ install_ggshield() { echo "ggshield installed successfully" } -# Function to run secret detection scans -run_secret_detection() { - echo "Running secret detection scans..." +# # Function to run secret detection scans +# run_secret_detection() { +# echo "Running secret detection scans..." - if ! command -v ggshield &> /dev/null; then - install_ggshield - fi +# if ! command -v ggshield &> /dev/null; then +# install_ggshield +# fi - # Check if GITGUARDIAN_API_KEY is set (required for CI/CD) - if [ -z "$GITGUARDIAN_API_KEY" ]; then - echo "Warning: GITGUARDIAN_API_KEY environment variable is not set." - echo "ggshield requires a GitGuardian API key to scan for secrets." - echo "Please set GITGUARDIAN_API_KEY in your CI/CD environment variables." - exit 1 - fi +# # Check if GITGUARDIAN_API_KEY is set (required for CI/CD) +# if [ -z "$GITGUARDIAN_API_KEY" ]; then +# echo "Warning: GITGUARDIAN_API_KEY environment variable is not set." +# echo "ggshield requires a GitGuardian API key to scan for secrets." +# echo "Please set GITGUARDIAN_API_KEY in your CI/CD environment variables." +# exit 1 +# fi - echo "Scanning codebase for secrets..." - echo "Note: Large codebases may take several minutes due to API rate limits (50 requests/minute on free plan)" - echo "ggshield will automatically handle rate limits and retry as needed." - echo "Binary files, cache files, and build artifacts are excluded via .gitguardian.yaml" +# echo "Scanning codebase for secrets..." +# echo "Note: Large codebases may take several minutes due to API rate limits (50 requests/minute on free plan)" +# echo "ggshield will automatically handle rate limits and retry as needed." +# echo "Binary files, cache files, and build artifacts are excluded via .gitguardian.yaml" - # Use --recursive for directory scanning and auto-confirm if prompted - # .gitguardian.yaml will automatically exclude binary files, wheel files, etc. - # GITGUARDIAN_API_KEY environment variable will be used for authentication - echo y | ggshield secret scan path . --recursive || { - echo "" - echo "==========================================" - echo "ERROR: Secret Detection Failed" - echo "==========================================" - echo "ggshield has detected secrets in the codebase." - echo "Please review discovered secrets above, revoke any actively used secrets" - echo "from underlying systems and make changes to inject secrets dynamically at runtime." - echo "" - echo "For more information, see: https://docs.gitguardian.com/secrets-detection/" - echo "==========================================" - echo "" - exit 1 - } +# # Use --recursive for directory scanning and auto-confirm if prompted +# # .gitguardian.yaml will automatically exclude binary files, wheel files, etc. +# # GITGUARDIAN_API_KEY environment variable will be used for authentication +# echo y | ggshield secret scan path . --recursive || { +# echo "" +# echo "==========================================" +# echo "ERROR: Secret Detection Failed" +# echo "==========================================" +# echo "ggshield has detected secrets in the codebase." +# echo "Please review discovered secrets above, revoke any actively used secrets" +# echo "from underlying systems and make changes to inject secrets dynamically at runtime." +# echo "" +# echo "For more information, see: https://docs.gitguardian.com/secrets-detection/" +# echo "==========================================" +# echo "" +# exit 1 +# } - echo "Secret detection scans completed successfully" -} +# echo "Secret detection scans completed successfully" +# } # Function to run Trivy scans run_trivy_scans() { @@ -209,8 +209,8 @@ main() { install_trivy install_grype - echo "Running secret detection scans..." - run_secret_detection + # echo "Running secret detection scans..." + # run_secret_detection echo "Running filesystem vulnerability scans..." run_trivy_scans diff --git a/deploy/Dockerfile.ghcr_base b/deploy/Dockerfile.ghcr_base index dbfe0a5a206..69b08a5893c 100644 --- a/deploy/Dockerfile.ghcr_base +++ b/deploy/Dockerfile.ghcr_base @@ -8,7 +8,8 @@ WORKDIR /app COPY config.yaml . # Make sure your docker/entrypoint.sh is executable -RUN chmod +x docker/entrypoint.sh +# Convert Windows line endings to Unix +RUN sed -i 's/\r$//' docker/entrypoint.sh && chmod +x docker/entrypoint.sh # Expose the necessary port EXPOSE 4000/tcp diff --git a/docker/Dockerfile.alpine b/docker/Dockerfile.alpine index ce83cfe653c..ef2bb98db6e 100644 --- a/docker/Dockerfile.alpine +++ b/docker/Dockerfile.alpine @@ -46,8 +46,9 @@ COPY --from=builder /wheels/ /wheels/ # Install the built wheel using pip; again using a wildcard if it's the only file RUN pip install *.whl /wheels/* --no-index --find-links=/wheels/ && rm -f *.whl && rm -rf /wheels -RUN chmod +x docker/entrypoint.sh -RUN chmod +x docker/prod_entrypoint.sh +# Convert Windows line endings to Unix for entrypoint scripts +RUN sed -i 's/\r$//' docker/entrypoint.sh && chmod +x docker/entrypoint.sh +RUN sed -i 's/\r$//' docker/prod_entrypoint.sh && chmod +x docker/prod_entrypoint.sh EXPOSE 4000/tcp diff --git a/docker/Dockerfile.custom_ui b/docker/Dockerfile.custom_ui index 5a313142112..c437929a27e 100644 --- a/docker/Dockerfile.custom_ui +++ b/docker/Dockerfile.custom_ui @@ -32,8 +32,9 @@ RUN rm -rf /app/litellm/proxy/_experimental/out/* && \ WORKDIR /app # Make sure your docker/entrypoint.sh is executable -RUN chmod +x docker/entrypoint.sh -RUN chmod +x docker/prod_entrypoint.sh +# Convert Windows line endings to Unix for entrypoint scripts +RUN sed -i 's/\r$//' docker/entrypoint.sh && chmod +x docker/entrypoint.sh +RUN sed -i 's/\r$//' docker/prod_entrypoint.sh && chmod +x docker/prod_entrypoint.sh # Expose the necessary port EXPOSE 4000/tcp diff --git a/docker/Dockerfile.database b/docker/Dockerfile.database index 9a4e9a315ea..49655129506 100644 --- a/docker/Dockerfile.database +++ b/docker/Dockerfile.database @@ -27,7 +27,8 @@ RUN python -m pip install build COPY . . # Build Admin UI -RUN chmod +x docker/build_admin_ui.sh && ./docker/build_admin_ui.sh +# Convert Windows line endings to Unix and make executable +RUN sed -i 's/\r$//' docker/build_admin_ui.sh && chmod +x docker/build_admin_ui.sh && ./docker/build_admin_ui.sh # Build the package RUN rm -rf dist/* && python -m build @@ -63,20 +64,23 @@ COPY --from=builder /wheels/ /wheels/ RUN pip install *.whl /wheels/* --no-index --find-links=/wheels/ && rm -f *.whl && rm -rf /wheels # Install semantic_router and aurelio-sdk using script -RUN chmod +x docker/install_auto_router.sh && ./docker/install_auto_router.sh +# Convert Windows line endings to Unix and make executable +RUN sed -i 's/\r$//' docker/install_auto_router.sh && chmod +x docker/install_auto_router.sh && ./docker/install_auto_router.sh # ensure pyjwt is used, not jwt RUN pip uninstall jwt -y RUN pip uninstall PyJWT -y RUN pip install PyJWT==2.9.0 --no-cache-dir -# Build Admin UI -RUN chmod +x docker/build_admin_ui.sh && ./docker/build_admin_ui.sh +# Build Admin UI (runtime stage) +# Convert Windows line endings to Unix and make executable +RUN sed -i 's/\r$//' docker/build_admin_ui.sh && chmod +x docker/build_admin_ui.sh && ./docker/build_admin_ui.sh # Generate prisma client RUN prisma generate -RUN chmod +x docker/entrypoint.sh -RUN chmod +x docker/prod_entrypoint.sh +# Convert Windows line endings to Unix for entrypoint scripts +RUN sed -i 's/\r$//' docker/entrypoint.sh && chmod +x docker/entrypoint.sh +RUN sed -i 's/\r$//' docker/prod_entrypoint.sh && chmod +x docker/prod_entrypoint.sh EXPOSE 4000/tcp RUN apk add --no-cache supervisor diff --git a/docker/Dockerfile.dev b/docker/Dockerfile.dev index f95f540a7a5..67966f9c739 100644 --- a/docker/Dockerfile.dev +++ b/docker/Dockerfile.dev @@ -40,7 +40,8 @@ COPY enterprise/ ./enterprise/ COPY docker/ ./docker/ # Build Admin UI once -RUN chmod +x docker/build_admin_ui.sh && ./docker/build_admin_ui.sh +# Convert Windows line endings to Unix and make executable +RUN sed -i 's/\r$//' docker/build_admin_ui.sh && chmod +x docker/build_admin_ui.sh && ./docker/build_admin_ui.sh # Build the package RUN rm -rf dist/* && python -m build @@ -79,8 +80,12 @@ RUN pip install --no-cache-dir *.whl /wheels/* --no-index --find-links=/wheels/ rm -rf /wheels # Generate prisma client and set permissions +# Convert Windows line endings to Unix for entrypoint scripts RUN prisma generate && \ - chmod +x docker/entrypoint.sh docker/prod_entrypoint.sh + sed -i 's/\r$//' docker/entrypoint.sh && \ + sed -i 's/\r$//' docker/prod_entrypoint.sh && \ + chmod +x docker/entrypoint.sh && \ + chmod +x docker/prod_entrypoint.sh EXPOSE 4000/tcp diff --git a/docker/Dockerfile.non_root b/docker/Dockerfile.non_root index af1bb5b2022..86222bbc280 100644 --- a/docker/Dockerfile.non_root +++ b/docker/Dockerfile.non_root @@ -144,7 +144,10 @@ RUN pip install --no-index --find-links=/wheels/ -r requirements.txt && \ fi # Permissions, cleanup, and Prisma prep -RUN chmod +x docker/entrypoint.sh docker/prod_entrypoint.sh && \ +# Convert Windows line endings to Unix for entrypoint scripts +RUN sed -i 's/\r$//' docker/entrypoint.sh && \ + sed -i 's/\r$//' docker/prod_entrypoint.sh && \ + chmod +x docker/entrypoint.sh docker/prod_entrypoint.sh && \ mkdir -p /nonexistent /.npm /var/lib/litellm/assets /var/lib/litellm/ui && \ chown -R nobody:nogroup /app /var/lib/litellm/ui /var/lib/litellm/assets /nonexistent /.npm && \ pip uninstall jwt -y || true && \ diff --git a/docs/my-website/docs/container_files.md b/docs/my-website/docs/container_files.md index 25b58a043c8..1ef7687ea77 100644 --- a/docs/my-website/docs/container_files.md +++ b/docs/my-website/docs/container_files.md @@ -21,6 +21,7 @@ Looking for how to use Code Interpreter? See the [Code Interpreter Guide](/docs/ | Endpoint | Method | Description | |----------|--------|-------------| +| `/v1/containers/{container_id}/files` | POST | Upload file to container | | `/v1/containers/{container_id}/files` | GET | List files in container | | `/v1/containers/{container_id}/files/{file_id}` | GET | Get file metadata | | `/v1/containers/{container_id}/files/{file_id}/content` | GET | Download file content | @@ -28,6 +29,45 @@ Looking for how to use Code Interpreter? See the [Code Interpreter Guide](/docs/ ## LiteLLM Python SDK +### Upload Container File + +Upload files directly to a container session. This is useful when `/chat/completions` or `/responses` sends files to the container but the input file type is limited to PDF. This endpoint lets you work with other file types like CSV, Excel, Python scripts, etc. + +```python showLineNumbers title="upload_container_file.py" +from litellm import upload_container_file + +# Upload a CSV file +file = upload_container_file( + container_id="cntr_123...", + file=("data.csv", open("data.csv", "rb").read(), "text/csv"), + custom_llm_provider="openai" +) + +print(f"Uploaded: {file.id}") +print(f"Path: {file.path}") +``` + +**Async:** + +```python showLineNumbers title="aupload_container_file.py" +from litellm import aupload_container_file + +file = await aupload_container_file( + container_id="cntr_123...", + file=("script.py", b"print('hello world')", "text/x-python"), + custom_llm_provider="openai" +) +``` + +**Supported file formats:** +- CSV (`.csv`) +- Excel (`.xlsx`) +- Python scripts (`.py`) +- JSON (`.json`) +- Markdown (`.md`) +- Text files (`.txt`) +- And more... + ### List Container Files ```python showLineNumbers title="list_container_files.py" @@ -103,6 +143,40 @@ print(f"Deleted: {result.deleted}") import Tabs from '@theme/Tabs'; import TabItem from '@theme/TabItem'; +### Upload File + + + + +```python showLineNumbers title="upload_file.py" +from openai import OpenAI + +client = OpenAI( + api_key="sk-1234", + base_url="http://localhost:4000" +) + +file = client.containers.files.create( + container_id="cntr_123...", + file=open("data.csv", "rb") +) + +print(f"Uploaded: {file.id}") +print(f"Path: {file.path}") +``` + + + + +```bash showLineNumbers title="upload_file.sh" +curl "http://localhost:4000/v1/containers/cntr_123.../files" \ + -H "Authorization: Bearer sk-1234" \ + -F file="@data.csv" +``` + + + + ### List Files @@ -236,6 +310,13 @@ curl -X DELETE "http://localhost:4000/v1/containers/cntr_123.../files/cfile_456. ## Parameters +### Upload File + +| Parameter | Type | Required | Description | +|-----------|------|----------|-------------| +| `container_id` | string | Yes | Container ID | +| `file` | FileTypes | Yes | File to upload. Can be a tuple of (filename, content, content_type), file-like object, or bytes | + ### List Files | Parameter | Type | Required | Description | diff --git a/docs/my-website/docs/interactions.md b/docs/my-website/docs/interactions.md index 1cd0f7be867..32c82a1589c 100644 --- a/docs/my-website/docs/interactions.md +++ b/docs/my-website/docs/interactions.md @@ -8,7 +8,7 @@ import TabItem from '@theme/TabItem'; | Logging | ✅ | Works across all integrations | | Streaming | ✅ | | | Loadbalancing | ✅ | Between supported models | -| Supported LLM providers | **All LiteLLM supported providers** | `openai`, `anthropic`, `bedrock`, `vertex_ai`, `gemini`, `azure`, `azure_ai` etc. | +| Supported LLM providers | **All LiteLLM supported CHAT COMPLETION providers** | `openai`, `anthropic`, `bedrock`, `vertex_ai`, `gemini`, `azure`, `azure_ai` etc. | ## **LiteLLM Python SDK Usage** diff --git a/docs/my-website/docs/observability/arize_integration.md b/docs/my-website/docs/observability/arize_integration.md index 0b457f08687..b3ccf98ea3b 100644 --- a/docs/my-website/docs/observability/arize_integration.md +++ b/docs/my-website/docs/observability/arize_integration.md @@ -68,6 +68,7 @@ environment_variables: ARIZE_API_KEY: "141a****" ARIZE_ENDPOINT: "https://otlp.arize.com/v1" # OPTIONAL - your custom arize GRPC api endpoint ARIZE_HTTP_ENDPOINT: "https://otlp.arize.com/v1" # OPTIONAL - your custom arize HTTP api endpoint. Set either this or ARIZE_ENDPOINT or Neither (defaults to https://otlp.arize.com/v1 on grpc) + ARIZE_PROJECT_NAME: "my-litellm-project" # OPTIONAL - sets the arize project name ``` 2. Start the proxy diff --git a/docs/my-website/docs/providers/azure_ai_img.md b/docs/my-website/docs/providers/azure_ai_img.md index 8e2f5226866..513bbe858d0 100644 --- a/docs/my-website/docs/providers/azure_ai_img.md +++ b/docs/my-website/docs/providers/azure_ai_img.md @@ -1,7 +1,7 @@ import Tabs from '@theme/Tabs'; import TabItem from '@theme/TabItem'; -# Azure AI Image Generation +# Azure AI Image Generation (Black Forest Labs - Flux) Azure AI provides powerful image generation capabilities using FLUX models from Black Forest Labs to create high-quality images from text descriptions. @@ -12,7 +12,7 @@ Azure AI provides powerful image generation capabilities using FLUX models from | Description | Azure AI Image Generation uses FLUX models to generate high-quality images from text descriptions. | | Provider Route on LiteLLM | `azure_ai/` | | Provider Doc | [Azure AI FLUX Models ↗](https://techcommunity.microsoft.com/blog/azure-ai-foundry-blog/black-forest-labs-flux-1-kontext-pro-and-flux1-1-pro-now-available-in-azure-ai-f/4434659) | -| Supported Operations | [`/images/generations`](#image-generation) | +| Supported Operations | [`/images/generations`](#image-generation), [`/images/edits`](#image-editing) | ## Setup @@ -33,6 +33,7 @@ Get your API key and endpoint from [Azure AI Studio](https://ai.azure.com/). |------------|-------------|----------------| | `azure_ai/FLUX-1.1-pro` | Latest FLUX 1.1 Pro model for high-quality image generation | $0.04 | | `azure_ai/FLUX.1-Kontext-pro` | FLUX 1 Kontext Pro model with enhanced context understanding | $0.04 | +| `azure_ai/flux.2-pro` | FLUX 2 Pro model for next-generation image generation | $0.04 | ## Image Generation @@ -85,6 +86,32 @@ print(response.data[0].url) + + +```python showLineNumbers title="FLUX 2 Pro Image Generation" +import litellm +import os + +# Set your API credentials +os.environ["AZURE_AI_API_KEY"] = "your-api-key-here" +os.environ["AZURE_AI_API_BASE"] = "your-azure-ai-endpoint" # e.g., https://litellm-ci-cd-prod.services.ai.azure.com + +# Generate image with FLUX 2 Pro +response = litellm.image_generation( + model="azure_ai/flux.2-pro", + prompt="A photograph of a red fox in an autumn forest", + api_base=os.environ["AZURE_AI_API_BASE"], + api_key=os.environ["AZURE_AI_API_KEY"], + api_version="preview", + size="1024x1024", + n=1 +) + +print(response.data[0].b64_json) # FLUX 2 returns base64 encoded images +``` + + + ```python showLineNumbers title="Async Image Generation" @@ -165,6 +192,15 @@ model_list: model_info: mode: image_generation + - model_name: azure-flux-2-pro + litellm_params: + model: azure_ai/flux.2-pro + api_key: os.environ/AZURE_AI_API_KEY + api_base: os.environ/AZURE_AI_API_BASE + api_version: preview + model_info: + mode: image_generation + general_settings: master_key: sk-1234 ``` @@ -239,6 +275,103 @@ curl --location 'http://localhost:4000/v1/images/generations' \ +## Image Editing + +FLUX 2 Pro supports image editing by passing an input image along with a prompt describing the desired modifications. + +### Usage - LiteLLM Python SDK + + + + +```python showLineNumbers title="Basic Image Editing with FLUX 2 Pro" +import litellm +import os + +# Set your API credentials +os.environ["AZURE_AI_API_KEY"] = "your-api-key-here" +os.environ["AZURE_AI_API_BASE"] = "your-azure-ai-endpoint" # e.g., https://litellm-ci-cd-prod.services.ai.azure.com + +# Edit an existing image +response = litellm.image_edit( + model="azure_ai/flux.2-pro", + prompt="Add a red hat to the subject", + image=open("input_image.png", "rb"), + api_base=os.environ["AZURE_AI_API_BASE"], + api_key=os.environ["AZURE_AI_API_KEY"], + api_version="preview", +) + +print(response.data[0].b64_json) # FLUX 2 returns base64 encoded images +``` + + + + + +```python showLineNumbers title="Async Image Editing" +import litellm +import asyncio +import os + +async def edit_image(): + os.environ["AZURE_AI_API_KEY"] = "your-api-key-here" + os.environ["AZURE_AI_API_BASE"] = "your-azure-ai-endpoint" + + response = await litellm.aimage_edit( + model="azure_ai/flux.2-pro", + prompt="Change the background to a sunset beach", + image=open("input_image.png", "rb"), + api_base=os.environ["AZURE_AI_API_BASE"], + api_key=os.environ["AZURE_AI_API_KEY"], + api_version="preview", + ) + + return response + +asyncio.run(edit_image()) +``` + + + + +### Usage - LiteLLM Proxy Server + + + + +```bash showLineNumbers title="Image Edit via Proxy - cURL" +curl --location 'http://localhost:4000/v1/images/edits' \ +--header 'Authorization: Bearer sk-1234' \ +--form 'model="azure-flux-2-pro"' \ +--form 'prompt="Add sunglasses to the person"' \ +--form 'image=@"input_image.png"' +``` + + + + + +```python showLineNumbers title="Image Edit via Proxy - OpenAI SDK" +from openai import OpenAI + +client = OpenAI( + base_url="http://localhost:4000", + api_key="sk-1234" +) + +response = client.images.edit( + model="azure-flux-2-pro", + prompt="Make the sky more dramatic with storm clouds", + image=open("input_image.png", "rb"), +) + +print(response.data[0].b64_json) +``` + + + + ## Supported Parameters Azure AI Image Generation supports the following OpenAI-compatible parameters: diff --git a/docs/my-website/docs/proxy/config_settings.md b/docs/my-website/docs/proxy/config_settings.md index 5772fbaa487..dfc0efd37ad 100644 --- a/docs/my-website/docs/proxy/config_settings.md +++ b/docs/my-website/docs/proxy/config_settings.md @@ -498,6 +498,7 @@ router_settings: | DD_VERSION | Version identifier for Datadog logs. Defaults to "unknown" | DEBUG_OTEL | Enable debug mode for OpenTelemetry | DEFAULT_ALLOWED_FAILS | Maximum failures allowed before cooling down a model. Default is 3 +| DEFAULT_A2A_AGENT_TIMEOUT | Default timeout in seconds for A2A (Agent-to-Agent) protocol requests. Default is 6000 | DEFAULT_ANTHROPIC_CHAT_MAX_TOKENS | Default maximum tokens for Anthropic chat completions. Default is 4096 | DEFAULT_BATCH_SIZE | Default batch size for operations. Default is 512 | DEFAULT_CHUNK_OVERLAP | Default chunk overlap for RAG text splitters. Default is 200 diff --git a/docs/my-website/docs/proxy/configs.md b/docs/my-website/docs/proxy/configs.md index bc2f6a13362..a5674bf2bc5 100644 --- a/docs/my-website/docs/proxy/configs.md +++ b/docs/my-website/docs/proxy/configs.md @@ -576,10 +576,31 @@ custom_tokenizer: ```yaml general_settings: - database_connection_pool_limit: 10 # sets connection pool for prisma client to postgres db (default: 10, recommended: 10-20) + database_connection_pool_limit: 10 # sets connection pool per worker for prisma client to postgres db (default: 10, recommended: 10-20) database_connection_timeout: 60 # sets a 60s timeout for any connection call to the db ``` +**How to calculate the right value:** + +The connection limit is applied **per worker process**, not per instance. This means if you have multiple workers, each worker will create its own connection pool. + +**Formula:** +``` +database_connection_pool_limit = MAX_DB_CONNECTIONS ÷ (number_of_instances × number_of_workers_per_instance) +``` + +**Example:** +- Your database allows a maximum of **100 connections** +- You're running **1 instance** of LiteLLM +- Each instance has **8 workers** (set via `--num_workers 8`) + +Calculation: `100 ÷ (1 × 8) = 12.5` + +Since you shouldn't use 12.5, round down to **10** to leave a safety buffer. This means: +- Each of the 8 workers will have a connection pool limit of 10 +- Total maximum connections: 8 workers × 10 connections = 80 connections +- This stays safely under your database's 100 connection limit + ## Extras diff --git a/docs/my-website/docs/proxy/prod.md b/docs/my-website/docs/proxy/prod.md index 71f0317cedf..9216b0fbf30 100644 --- a/docs/my-website/docs/proxy/prod.md +++ b/docs/my-website/docs/proxy/prod.md @@ -19,7 +19,11 @@ general_settings: master_key: sk-1234 # enter your own master key, ensure it starts with 'sk-' alerting: ["slack"] # Setup slack alerting - get alerts on LLM exceptions, Budget Alerts, Slow LLM Responses proxy_batch_write_at: 60 # Batch write spend updates every 60s - database_connection_pool_limit: 10 # limit the number of database connections to = MAX Number of DB Connections/Number of instances of litellm proxy (Around 10-20 is good number) + database_connection_pool_limit: 10 # connection pool limit per worker process. Total connections = limit × workers × instances. Calculate: MAX_DB_CONNECTIONS / (instances × workers). Default: 10. + +:::warning +**Multiple instances:** If running multiple LiteLLM instances (e.g., Kubernetes pods), remember each instance multiplies your total connections. Example: 3 instances × 4 workers × 10 connections = 120 total connections. +::: # OPTIONAL Best Practices disable_error_logs: True # turn off writing LLM Exceptions to DB @@ -54,8 +58,8 @@ For optimal performance in production, we recommend the following minimum machin | Resource | Recommended Value | |----------|------------------| -| CPU | 2 vCPU | -| Memory | 4 GB RAM | +| CPU | 4 vCPU | +| Memory | 8 GB RAM | These specifications provide: - Sufficient compute power for handling concurrent requests diff --git a/docs/my-website/docs/realtime.md b/docs/my-website/docs/realtime.md index 7a6143dd028..0b3c823f5db 100644 --- a/docs/my-website/docs/realtime.md +++ b/docs/my-website/docs/realtime.md @@ -5,6 +5,12 @@ import TabItem from '@theme/TabItem'; Use this to loadbalance across Azure + OpenAI. +Supported Providers: +- OpenAI +- Azure +- Google AI Studio (Gemini) +- Vertex AI + ## Proxy Usage ### Add model to config diff --git a/docs/my-website/sidebars.js b/docs/my-website/sidebars.js index 5d2f096156e..482d855082e 100644 --- a/docs/my-website/sidebars.js +++ b/docs/my-website/sidebars.js @@ -420,14 +420,8 @@ const sidebars = { ], }, "assistants", - { - type: "category", - label: "/audio", - items: [ - "audio_transcription", - "text_to_speech", - ] - }, + "audio_transcription", + "text_to_speech", { type: "category", label: "/batches", @@ -477,17 +471,13 @@ const sidebars = { "apply_guardrail", "bedrock_invoke", "interactions", - { - type: "category", - label: "/images", - items: [ - "image_edits", - "image_generation", - "image_variations", - ] - }, + "image_edits", + "image_generation", + "image_variations", "videos", "vector_store_files", + "vector_stores/create", + "vector_stores/search", { type: "category", label: "/mcp - Model Context Protocol", @@ -531,24 +521,12 @@ const sidebars = { "proxy/pass_through_guardrails" ] }, - { - type: "category", - label: "/rag", - items: [ - "rag_ingest", - "rag_query", - ] - }, + "rag_ingest", + "rag_query", "realtime", "rerank", - { - type: "category", - label: "/responses", - items: [ - "response_api", - "response_api_compact", - ] - }, + "response_api", + "response_api_compact", { type: "category", label: "/search", @@ -566,14 +544,7 @@ const sidebars = { ] }, "skills", - { - type: "category", - label: "/vector_stores", - items: [ - "vector_stores/create", - "vector_stores/search", - ] - }, + ], }, { diff --git a/flux2_test_image.png b/flux2_test_image.png new file mode 100644 index 00000000000..d40fa1a65f2 Binary files /dev/null and b/flux2_test_image.png differ diff --git a/litellm-proxy-extras/dist/litellm_proxy_extras-0.4.17-py3-none-any.whl b/litellm-proxy-extras/dist/litellm_proxy_extras-0.4.17-py3-none-any.whl new file mode 100644 index 00000000000..9f8a8b03931 Binary files /dev/null and b/litellm-proxy-extras/dist/litellm_proxy_extras-0.4.17-py3-none-any.whl differ diff --git a/litellm-proxy-extras/dist/litellm_proxy_extras-0.4.17.tar.gz b/litellm-proxy-extras/dist/litellm_proxy_extras-0.4.17.tar.gz new file mode 100644 index 00000000000..37c3d3f2638 Binary files /dev/null and b/litellm-proxy-extras/dist/litellm_proxy_extras-0.4.17.tar.gz differ diff --git a/litellm-proxy-extras/dist/litellm_proxy_extras-0.4.18-py3-none-any.whl b/litellm-proxy-extras/dist/litellm_proxy_extras-0.4.18-py3-none-any.whl new file mode 100644 index 00000000000..9d23c4f66a5 Binary files /dev/null and b/litellm-proxy-extras/dist/litellm_proxy_extras-0.4.18-py3-none-any.whl differ diff --git a/litellm-proxy-extras/dist/litellm_proxy_extras-0.4.18.tar.gz b/litellm-proxy-extras/dist/litellm_proxy_extras-0.4.18.tar.gz new file mode 100644 index 00000000000..0adba14c025 Binary files /dev/null and b/litellm-proxy-extras/dist/litellm_proxy_extras-0.4.18.tar.gz differ diff --git a/litellm-proxy-extras/dist/litellm_proxy_extras-0.4.19-py3-none-any.whl b/litellm-proxy-extras/dist/litellm_proxy_extras-0.4.19-py3-none-any.whl new file mode 100644 index 00000000000..471ddce912c Binary files /dev/null and b/litellm-proxy-extras/dist/litellm_proxy_extras-0.4.19-py3-none-any.whl differ diff --git a/litellm-proxy-extras/dist/litellm_proxy_extras-0.4.19.tar.gz b/litellm-proxy-extras/dist/litellm_proxy_extras-0.4.19.tar.gz new file mode 100644 index 00000000000..290c4bfeef5 Binary files /dev/null and b/litellm-proxy-extras/dist/litellm_proxy_extras-0.4.19.tar.gz differ diff --git a/litellm-proxy-extras/dist/litellm_proxy_extras-0.4.20-py3-none-any.whl b/litellm-proxy-extras/dist/litellm_proxy_extras-0.4.20-py3-none-any.whl new file mode 100644 index 00000000000..d62330de7be Binary files /dev/null and b/litellm-proxy-extras/dist/litellm_proxy_extras-0.4.20-py3-none-any.whl differ diff --git a/litellm-proxy-extras/dist/litellm_proxy_extras-0.4.20.tar.gz b/litellm-proxy-extras/dist/litellm_proxy_extras-0.4.20.tar.gz new file mode 100644 index 00000000000..7e509f12082 Binary files /dev/null and b/litellm-proxy-extras/dist/litellm_proxy_extras-0.4.20.tar.gz differ diff --git a/litellm-proxy-extras/litellm_proxy_extras/migrations/20260106155622_add_endpoint_to_daily_activity_tables/migration.sql b/litellm-proxy-extras/litellm_proxy_extras/migrations/20260106155622_add_endpoint_to_daily_activity_tables/migration.sql new file mode 100644 index 00000000000..4ed7feb9ca0 --- /dev/null +++ b/litellm-proxy-extras/litellm_proxy_extras/migrations/20260106155622_add_endpoint_to_daily_activity_tables/migration.sql @@ -0,0 +1,72 @@ +-- DropIndex +DROP INDEX "LiteLLM_DailyAgentSpend_agent_id_date_api_key_model_custom__key"; + +-- DropIndex +DROP INDEX "LiteLLM_DailyEndUserSpend_end_user_id_date_api_key_model_cu_key"; + +-- DropIndex +DROP INDEX "LiteLLM_DailyOrganizationSpend_organization_id_date_api_key_key"; + +-- DropIndex +DROP INDEX "LiteLLM_DailyTagSpend_tag_date_api_key_model_custom_llm_pro_key"; + +-- DropIndex +DROP INDEX "LiteLLM_DailyTeamSpend_team_id_date_api_key_model_custom_ll_key"; + +-- DropIndex +DROP INDEX "LiteLLM_DailyUserSpend_user_id_date_api_key_model_custom_ll_key"; + +-- AlterTable +ALTER TABLE "LiteLLM_DailyAgentSpend" ADD COLUMN "endpoint" TEXT; + +-- AlterTable +ALTER TABLE "LiteLLM_DailyEndUserSpend" ADD COLUMN "endpoint" TEXT; + +-- AlterTable +ALTER TABLE "LiteLLM_DailyOrganizationSpend" ADD COLUMN "endpoint" TEXT; + +-- AlterTable +ALTER TABLE "LiteLLM_DailyTagSpend" ADD COLUMN "endpoint" TEXT; + +-- AlterTable +ALTER TABLE "LiteLLM_DailyTeamSpend" ADD COLUMN "endpoint" TEXT; + +-- AlterTable +ALTER TABLE "LiteLLM_DailyUserSpend" ADD COLUMN "endpoint" TEXT; + +-- CreateIndex +CREATE INDEX "LiteLLM_DailyAgentSpend_endpoint_idx" ON "LiteLLM_DailyAgentSpend"("endpoint"); + +-- CreateIndex +CREATE UNIQUE INDEX "LiteLLM_DailyAgentSpend_agent_id_date_api_key_model_custom__key" ON "LiteLLM_DailyAgentSpend"("agent_id", "date", "api_key", "model", "custom_llm_provider", "mcp_namespaced_tool_name", "endpoint"); + +-- CreateIndex +CREATE INDEX "LiteLLM_DailyEndUserSpend_endpoint_idx" ON "LiteLLM_DailyEndUserSpend"("endpoint"); + +-- CreateIndex +CREATE UNIQUE INDEX "LiteLLM_DailyEndUserSpend_end_user_id_date_api_key_model_cu_key" ON "LiteLLM_DailyEndUserSpend"("end_user_id", "date", "api_key", "model", "custom_llm_provider", "mcp_namespaced_tool_name", "endpoint"); + +-- CreateIndex +CREATE INDEX "LiteLLM_DailyOrganizationSpend_endpoint_idx" ON "LiteLLM_DailyOrganizationSpend"("endpoint"); + +-- CreateIndex +CREATE UNIQUE INDEX "LiteLLM_DailyOrganizationSpend_organization_id_date_api_key_key" ON "LiteLLM_DailyOrganizationSpend"("organization_id", "date", "api_key", "model", "custom_llm_provider", "mcp_namespaced_tool_name", "endpoint"); + +-- CreateIndex +CREATE INDEX "LiteLLM_DailyTagSpend_endpoint_idx" ON "LiteLLM_DailyTagSpend"("endpoint"); + +-- CreateIndex +CREATE UNIQUE INDEX "LiteLLM_DailyTagSpend_tag_date_api_key_model_custom_llm_pro_key" ON "LiteLLM_DailyTagSpend"("tag", "date", "api_key", "model", "custom_llm_provider", "mcp_namespaced_tool_name", "endpoint"); + +-- CreateIndex +CREATE INDEX "LiteLLM_DailyTeamSpend_endpoint_idx" ON "LiteLLM_DailyTeamSpend"("endpoint"); + +-- CreateIndex +CREATE UNIQUE INDEX "LiteLLM_DailyTeamSpend_team_id_date_api_key_model_custom_ll_key" ON "LiteLLM_DailyTeamSpend"("team_id", "date", "api_key", "model", "custom_llm_provider", "mcp_namespaced_tool_name", "endpoint"); + +-- CreateIndex +CREATE INDEX "LiteLLM_DailyUserSpend_endpoint_idx" ON "LiteLLM_DailyUserSpend"("endpoint"); + +-- CreateIndex +CREATE UNIQUE INDEX "LiteLLM_DailyUserSpend_user_id_date_api_key_model_custom_ll_key" ON "LiteLLM_DailyUserSpend"("user_id", "date", "api_key", "model", "custom_llm_provider", "mcp_namespaced_tool_name", "endpoint"); + diff --git a/litellm-proxy-extras/litellm_proxy_extras/migrations/20260107111013_add_router_settings_to_keys_teams/migration.sql b/litellm-proxy-extras/litellm_proxy_extras/migrations/20260107111013_add_router_settings_to_keys_teams/migration.sql new file mode 100644 index 00000000000..95566950118 --- /dev/null +++ b/litellm-proxy-extras/litellm_proxy_extras/migrations/20260107111013_add_router_settings_to_keys_teams/migration.sql @@ -0,0 +1,6 @@ +-- AlterTable +ALTER TABLE "LiteLLM_TeamTable" ADD COLUMN "router_settings" JSONB DEFAULT '{}'; + +-- AlterTable +ALTER TABLE "LiteLLM_VerificationToken" ADD COLUMN "router_settings" JSONB DEFAULT '{}'; + diff --git a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma index e565135bbc4..56fe093a8bc 100644 --- a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma +++ b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma @@ -124,6 +124,7 @@ model LiteLLM_TeamTable { updated_at DateTime @default(now()) @updatedAt @map("updated_at") model_spend Json @default("{}") model_max_budget Json @default("{}") + router_settings Json? @default("{}") team_member_permissions String[] @default([]) model_id Int? @unique // id for LiteLLM_ModelTable -> stores team-level model aliases litellm_organization_table LiteLLM_OrganizationTable? @relation(fields: [organization_id], references: [organization_id]) @@ -225,6 +226,7 @@ model LiteLLM_VerificationToken { models String[] aliases Json @default("{}") config Json @default("{}") + router_settings Json? @default("{}") user_id String? team_id String? permissions Json @default("{}") @@ -422,6 +424,7 @@ model LiteLLM_DailyUserSpend { model_group String? custom_llm_provider String? mcp_namespaced_tool_name String? + endpoint String? prompt_tokens BigInt @default(0) completion_tokens BigInt @default(0) cache_read_input_tokens BigInt @default(0) @@ -433,12 +436,13 @@ model LiteLLM_DailyUserSpend { created_at DateTime @default(now()) updated_at DateTime @updatedAt - @@unique([user_id, date, api_key, model, custom_llm_provider, mcp_namespaced_tool_name]) + @@unique([user_id, date, api_key, model, custom_llm_provider, mcp_namespaced_tool_name, endpoint]) @@index([date]) @@index([user_id]) @@index([api_key]) @@index([model]) @@index([mcp_namespaced_tool_name]) + @@index([endpoint]) } // Track daily organization spend metrics per model and key @@ -451,6 +455,7 @@ model LiteLLM_DailyOrganizationSpend { model_group String? custom_llm_provider String? mcp_namespaced_tool_name String? + endpoint String? prompt_tokens BigInt @default(0) completion_tokens BigInt @default(0) cache_read_input_tokens BigInt @default(0) @@ -462,12 +467,13 @@ model LiteLLM_DailyOrganizationSpend { created_at DateTime @default(now()) updated_at DateTime @updatedAt - @@unique([organization_id, date, api_key, model, custom_llm_provider, mcp_namespaced_tool_name]) + @@unique([organization_id, date, api_key, model, custom_llm_provider, mcp_namespaced_tool_name, endpoint]) @@index([date]) @@index([organization_id]) @@index([api_key]) @@index([model]) @@index([mcp_namespaced_tool_name]) + @@index([endpoint]) } // Track daily end user (customer) spend metrics per model and key @@ -480,6 +486,7 @@ model LiteLLM_DailyEndUserSpend { model_group String? custom_llm_provider String? mcp_namespaced_tool_name String? + endpoint String? prompt_tokens BigInt @default(0) completion_tokens BigInt @default(0) cache_read_input_tokens BigInt @default(0) @@ -490,12 +497,13 @@ model LiteLLM_DailyEndUserSpend { failed_requests BigInt @default(0) created_at DateTime @default(now()) updated_at DateTime @updatedAt - @@unique([end_user_id, date, api_key, model, custom_llm_provider, mcp_namespaced_tool_name]) + @@unique([end_user_id, date, api_key, model, custom_llm_provider, mcp_namespaced_tool_name, endpoint]) @@index([date]) @@index([end_user_id]) @@index([api_key]) @@index([model]) @@index([mcp_namespaced_tool_name]) + @@index([endpoint]) } // Track daily agent spend metrics per model and key @@ -508,6 +516,7 @@ model LiteLLM_DailyAgentSpend { model_group String? custom_llm_provider String? mcp_namespaced_tool_name String? + endpoint String? prompt_tokens BigInt @default(0) completion_tokens BigInt @default(0) cache_read_input_tokens BigInt @default(0) @@ -518,12 +527,13 @@ model LiteLLM_DailyAgentSpend { failed_requests BigInt @default(0) created_at DateTime @default(now()) updated_at DateTime @updatedAt - @@unique([agent_id, date, api_key, model, custom_llm_provider, mcp_namespaced_tool_name]) + @@unique([agent_id, date, api_key, model, custom_llm_provider, mcp_namespaced_tool_name, endpoint]) @@index([date]) @@index([agent_id]) @@index([api_key]) @@index([model]) @@index([mcp_namespaced_tool_name]) + @@index([endpoint]) } // Track daily team spend metrics per model and key @@ -536,6 +546,7 @@ model LiteLLM_DailyTeamSpend { model_group String? custom_llm_provider String? mcp_namespaced_tool_name String? + endpoint String? prompt_tokens BigInt @default(0) completion_tokens BigInt @default(0) cache_read_input_tokens BigInt @default(0) @@ -547,12 +558,13 @@ model LiteLLM_DailyTeamSpend { created_at DateTime @default(now()) updated_at DateTime @updatedAt - @@unique([team_id, date, api_key, model, custom_llm_provider, mcp_namespaced_tool_name]) + @@unique([team_id, date, api_key, model, custom_llm_provider, mcp_namespaced_tool_name, endpoint]) @@index([date]) @@index([team_id]) @@index([api_key]) @@index([model]) @@index([mcp_namespaced_tool_name]) + @@index([endpoint]) } // Track daily team spend metrics per model and key @@ -566,6 +578,7 @@ model LiteLLM_DailyTagSpend { model_group String? custom_llm_provider String? mcp_namespaced_tool_name String? + endpoint String? prompt_tokens BigInt @default(0) completion_tokens BigInt @default(0) cache_read_input_tokens BigInt @default(0) @@ -577,12 +590,13 @@ model LiteLLM_DailyTagSpend { created_at DateTime @default(now()) updated_at DateTime @updatedAt - @@unique([tag, date, api_key, model, custom_llm_provider, mcp_namespaced_tool_name]) + @@unique([tag, date, api_key, model, custom_llm_provider, mcp_namespaced_tool_name, endpoint]) @@index([date]) @@index([tag]) @@index([api_key]) @@index([model]) @@index([mcp_namespaced_tool_name]) + @@index([endpoint]) } diff --git a/litellm-proxy-extras/pyproject.toml b/litellm-proxy-extras/pyproject.toml index 7c11a04fca8..7eccab254e3 100644 --- a/litellm-proxy-extras/pyproject.toml +++ b/litellm-proxy-extras/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "litellm-proxy-extras" -version = "0.4.16" +version = "0.4.20" description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package." authors = ["BerriAI"] readme = "README.md" @@ -22,7 +22,7 @@ requires = ["poetry-core"] build-backend = "poetry.core.masonry.api" [tool.commitizen] -version = "0.4.16" +version = "0.4.20" version_files = [ "pyproject.toml:version", "../requirements.txt:litellm-proxy-extras==", diff --git a/litellm/__init__.py b/litellm/__init__.py index 8af9a7d10a6..77e487fca24 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -486,6 +486,7 @@ vertex_mistral_models: Set = set() vertex_openai_models: Set = set() vertex_minimax_models: Set = set() vertex_moonshot_models: Set = set() +vertex_zai_models: Set = set() ai21_models: Set = set() ai21_chat_models: Set = set() nlp_cloud_models: Set = set() @@ -557,6 +558,8 @@ stability_models: Set = set() github_copilot_models: Set = set() minimax_models: Set = set() aws_polly_models: Set = set() +gigachat_models: Set = set() +llamagate_models: Set = set() def is_bedrock_pricing_only_model(key: str) -> bool: @@ -662,6 +665,9 @@ def add_known_models(): elif value.get("litellm_provider") == "vertex_ai-moonshot_models": key = key.replace("vertex_ai/", "") vertex_moonshot_models.add(key) + elif value.get("litellm_provider") == "vertex_ai-zai_models": + key = key.replace("vertex_ai/", "") + vertex_zai_models.add(key) elif value.get("litellm_provider") == "ai21": if value.get("mode") == "chat": ai21_chat_models.add(key) @@ -809,6 +815,10 @@ def add_known_models(): minimax_models.add(key) elif value.get("litellm_provider") == "aws_polly": aws_polly_models.add(key) + elif value.get("litellm_provider") == "gigachat": + gigachat_models.add(key) + elif value.get("litellm_provider") == "llamagate": + llamagate_models.add(key) add_known_models() @@ -944,7 +954,8 @@ models_by_provider: dict = { | vertex_language_models | vertex_deepseek_models | vertex_minimax_models - | vertex_moonshot_models, + | vertex_moonshot_models + | vertex_zai_models, "ai21": ai21_models, "bedrock": bedrock_models | bedrock_converse_models, "petals": petals_models, @@ -1015,6 +1026,8 @@ models_by_provider: dict = { "github_copilot": github_copilot_models, "minimax": minimax_models, "aws_polly": aws_polly_models, + "gigachat": gigachat_models, + "llamagate": llamagate_models, } # mapping for those models which have larger equivalents diff --git a/litellm/a2a_protocol/main.py b/litellm/a2a_protocol/main.py index f36f7d3ef5b..167aad7959a 100644 --- a/litellm/a2a_protocol/main.py +++ b/litellm/a2a_protocol/main.py @@ -12,6 +12,7 @@ import litellm from litellm._logging import verbose_logger from litellm.a2a_protocol.streaming_iterator import A2AStreamingIterator from litellm.a2a_protocol.utils import A2ARequestUtils +from litellm.constants import DEFAULT_A2A_AGENT_TIMEOUT from litellm.litellm_core_utils.litellm_logging import Logging from litellm.llms.custom_httpx.http_handler import ( get_async_httpx_client, @@ -494,7 +495,7 @@ async def create_a2a_client( async def aget_agent_card( base_url: str, - timeout: float = 60.0, + timeout: float = DEFAULT_A2A_AGENT_TIMEOUT, extra_headers: Optional[Dict[str, str]] = None, ) -> "AgentCard": """ diff --git a/litellm/constants.py b/litellm/constants.py index 1cd2da549ca..db9d0114118 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -278,6 +278,7 @@ MAX_SIZE_PER_ITEM_IN_MEMORY_CACHE_IN_KB = int( DEFAULT_MAX_TOKENS_FOR_TRITON = int(os.getenv("DEFAULT_MAX_TOKENS_FOR_TRITON", 2000)) #### Networking settings #### request_timeout: float = float(os.getenv("REQUEST_TIMEOUT", 6000)) # time in seconds +DEFAULT_A2A_AGENT_TIMEOUT: float = float(os.getenv("DEFAULT_A2A_AGENT_TIMEOUT", 6000)) # 10 minutes STREAM_SSE_DONE_STRING: str = "[DONE]" STREAM_SSE_DATA_PREFIX: str = "data: " ### SPEND TRACKING ### diff --git a/litellm/containers/endpoint_factory.py b/litellm/containers/endpoint_factory.py index 998b42a3abd..0b73a19b922 100644 --- a/litellm/containers/endpoint_factory.py +++ b/litellm/containers/endpoint_factory.py @@ -216,6 +216,8 @@ _generated_endpoints = generate_container_endpoints() # Export generated functions dynamically list_container_files = _generated_endpoints.get("list_container_files") alist_container_files = _generated_endpoints.get("alist_container_files") +upload_container_file = _generated_endpoints.get("upload_container_file") +aupload_container_file = _generated_endpoints.get("aupload_container_file") retrieve_container_file = _generated_endpoints.get("retrieve_container_file") aretrieve_container_file = _generated_endpoints.get("aretrieve_container_file") delete_container_file = _generated_endpoints.get("delete_container_file") diff --git a/litellm/containers/endpoints.json b/litellm/containers/endpoints.json index 4a23fc75c31..1ba61ee26e9 100644 --- a/litellm/containers/endpoints.json +++ b/litellm/containers/endpoints.json @@ -9,6 +9,16 @@ "query_params": ["after", "limit", "order"], "response_type": "ContainerFileListResponse" }, + { + "name": "upload_container_file", + "async_name": "aupload_container_file", + "path": "/containers/{container_id}/files", + "method": "POST", + "path_params": ["container_id"], + "query_params": [], + "response_type": "ContainerFileObject", + "is_multipart": true + }, { "name": "retrieve_container_file", "async_name": "aretrieve_container_file", diff --git a/litellm/containers/main.py b/litellm/containers/main.py index 1fe7a26c0a8..625a291fb55 100644 --- a/litellm/containers/main.py +++ b/litellm/containers/main.py @@ -13,11 +13,13 @@ from litellm.main import base_llm_http_handler from litellm.types.containers.main import ( ContainerCreateOptionalRequestParams, ContainerFileListResponse, + ContainerFileObject, ContainerListOptionalRequestParams, ContainerListResponse, ContainerObject, DeleteContainerResult, ) +from litellm.types.llms.openai import FileTypes from litellm.types.router import GenericLiteLLMParams from litellm.types.utils import CallTypes from litellm.utils import ProviderConfigManager, client @@ -28,11 +30,13 @@ __all__ = [ "alist_container_files", "alist_containers", "aretrieve_container", + "aupload_container_file", "create_container", "delete_container", "list_container_files", "list_containers", "retrieve_container", + "upload_container_file", ] ##### Container Create ####################### @@ -1011,3 +1015,236 @@ def list_container_files( extra_kwargs=kwargs, ) + +##### Container File Upload ####################### +@client +async def aupload_container_file( + container_id: str, + file: FileTypes, + timeout=600, # default to 10 minutes + custom_llm_provider: Literal["openai"] = "openai", + extra_headers: Optional[Dict[str, Any]] = None, + extra_query: Optional[Dict[str, Any]] = None, + extra_body: Optional[Dict[str, Any]] = None, + **kwargs, +) -> ContainerFileObject: + """Asynchronously upload a file to a container. + + This endpoint allows uploading files directly to a container session, + supporting various file types like CSV, Excel, Python scripts, etc. + + Parameters: + - `container_id` (str): The ID of the container to upload the file to + - `file` (FileTypes): The file to upload. Can be: + - A tuple of (filename, content, content_type) + - A tuple of (filename, content) + - A file-like object with read() method + - Bytes + - A string path to a file + - `timeout` (int): Request timeout in seconds + - `custom_llm_provider` (Literal["openai"]): The LLM provider to use + - `extra_headers` (Optional[Dict[str, Any]]): Additional headers + - `extra_query` (Optional[Dict[str, Any]]): Additional query parameters + - `extra_body` (Optional[Dict[str, Any]]): Additional body parameters + - `kwargs` (dict): Additional keyword arguments + + Returns: + - `response` (ContainerFileObject): The uploaded file object + + Example: + ```python + import litellm + + # Upload a CSV file + response = await litellm.aupload_container_file( + container_id="container_abc123", + file=("data.csv", open("data.csv", "rb").read(), "text/csv"), + custom_llm_provider="openai", + ) + print(response) + ``` + """ + local_vars = locals() + try: + loop = asyncio.get_event_loop() + kwargs["async_call"] = True + + func = partial( + upload_container_file, + container_id=container_id, + file=file, + timeout=timeout, + custom_llm_provider=custom_llm_provider, + extra_headers=extra_headers, + extra_query=extra_query, + extra_body=extra_body, + **kwargs, + ) + + ctx = contextvars.copy_context() + func_with_context = partial(ctx.run, func) + init_response = await loop.run_in_executor(None, func_with_context) + + if asyncio.iscoroutine(init_response): + response = await init_response + else: + response = init_response + + return response + except Exception as e: + raise litellm.exception_type( + model="", + custom_llm_provider=custom_llm_provider, + original_exception=e, + completion_kwargs=local_vars, + extra_kwargs=kwargs, + ) + + +# fmt: off + +@overload +def upload_container_file( + container_id: str, + file: FileTypes, + timeout=600, + api_key: Optional[str] = None, + api_base: Optional[str] = None, + api_version: Optional[str] = None, + custom_llm_provider: Literal["openai"] = "openai", + *, + aupload_container_file: Literal[True], + **kwargs, +) -> Coroutine[Any, Any, ContainerFileObject]: + ... + + +@overload +def upload_container_file( + container_id: str, + file: FileTypes, + timeout=600, + api_key: Optional[str] = None, + api_base: Optional[str] = None, + api_version: Optional[str] = None, + custom_llm_provider: Literal["openai"] = "openai", + *, + aupload_container_file: Literal[False] = False, + **kwargs, +) -> ContainerFileObject: + ... + +# fmt: on + + +@client +def upload_container_file( + container_id: str, + file: FileTypes, + timeout=600, # default to 10 minutes + api_key: Optional[str] = None, + api_base: Optional[str] = None, + api_version: Optional[str] = None, + custom_llm_provider: Literal["openai"] = "openai", + extra_headers: Optional[Dict[str, Any]] = None, + extra_query: Optional[Dict[str, Any]] = None, + extra_body: Optional[Dict[str, Any]] = None, + **kwargs, +) -> Union[ + ContainerFileObject, + Coroutine[Any, Any, ContainerFileObject], +]: + """Upload a file to a container using the OpenAI Container API. + + This endpoint allows uploading files directly to a container session, + supporting various file types like CSV, Excel, Python scripts, JSON, etc. + This is useful when /chat/completions or /responses sends files to the + container but the input file type is limited to PDF. This endpoint lets + you work with other file types. + + Currently supports OpenAI + + Example: + ```python + import litellm + + # Upload a CSV file + response = litellm.upload_container_file( + container_id="container_abc123", + file=("data.csv", open("data.csv", "rb").read(), "text/csv"), + custom_llm_provider="openai", + ) + print(response) + + # Upload a Python script + response = litellm.upload_container_file( + container_id="container_abc123", + file=("script.py", b"print('hello world')", "text/x-python"), + custom_llm_provider="openai", + ) + print(response) + ``` + """ + from litellm.llms.custom_httpx.container_handler import generic_container_handler + + local_vars = locals() + try: + litellm_logging_obj: LiteLLMLoggingObj = kwargs.pop("litellm_logging_obj") # type: ignore + litellm_call_id: Optional[str] = kwargs.get("litellm_call_id") + _is_async = kwargs.pop("async_call", False) is True + + # Check for mock response first + mock_response = kwargs.get("mock_response") + if mock_response is not None: + if isinstance(mock_response, str): + mock_response = json.loads(mock_response) + + response = ContainerFileObject(**mock_response) + return response + + # get llm provider logic + litellm_params = GenericLiteLLMParams(**kwargs) + # get provider config + container_provider_config: Optional[BaseContainerConfig] = ( + ProviderConfigManager.get_provider_container_config( + provider=litellm.LlmProviders(custom_llm_provider), + ) + ) + + if container_provider_config is None: + raise ValueError(f"Container provider config not found for provider: {custom_llm_provider}") + + # Pre Call logging + litellm_logging_obj.update_environment_variables( + model="", + optional_params={"container_id": container_id}, + litellm_params={ + "litellm_call_id": litellm_call_id, + }, + custom_llm_provider=custom_llm_provider, + ) + + # Set the correct call type + litellm_logging_obj.call_type = CallTypes.upload_container_file.value + + return generic_container_handler.handle( + endpoint_name="upload_container_file", + container_provider_config=container_provider_config, + litellm_params=litellm_params, + logging_obj=litellm_logging_obj, + extra_headers=extra_headers, + extra_query=extra_query, + timeout=timeout or DEFAULT_REQUEST_TIMEOUT, + _is_async=_is_async, + container_id=container_id, + file=file, + ) + + except Exception as e: + raise litellm.exception_type( + model="", + custom_llm_provider=custom_llm_provider, + original_exception=e, + completion_kwargs=local_vars, + extra_kwargs=kwargs, + ) diff --git a/litellm/integrations/arize/arize.py b/litellm/integrations/arize/arize.py index 4d1aa80dcce..9c2f0d95d4d 100644 --- a/litellm/integrations/arize/arize.py +++ b/litellm/integrations/arize/arize.py @@ -51,6 +51,7 @@ class ArizeLogger(OpenTelemetry): space_id = os.environ.get("ARIZE_SPACE_ID") space_key = os.environ.get("ARIZE_SPACE_KEY") api_key = os.environ.get("ARIZE_API_KEY") + project_name = os.environ.get("ARIZE_PROJECT_NAME") grpc_endpoint = os.environ.get("ARIZE_ENDPOINT") http_endpoint = os.environ.get("ARIZE_HTTP_ENDPOINT") @@ -74,6 +75,7 @@ class ArizeLogger(OpenTelemetry): api_key=api_key, protocol=protocol, endpoint=endpoint, + project_name=project_name, ) async def async_service_success_hook( diff --git a/litellm/integrations/opentelemetry.py b/litellm/integrations/opentelemetry.py index a7d2326d938..7e0cfab617b 100644 --- a/litellm/integrations/opentelemetry.py +++ b/litellm/integrations/opentelemetry.py @@ -54,38 +54,6 @@ RAW_REQUEST_SPAN_NAME = "raw_gen_ai_request" LITELLM_REQUEST_SPAN_NAME = "litellm_request" -def _get_litellm_resource(): - """ - Create a proper OpenTelemetry Resource that respects OTEL_RESOURCE_ATTRIBUTES - while maintaining backward compatibility with LiteLLM-specific environment variables. - """ - from opentelemetry.sdk.resources import OTELResourceDetector, Resource - - # Create base resource attributes with LiteLLM-specific defaults - # These will be overridden by OTEL_RESOURCE_ATTRIBUTES if present - base_attributes: Dict[str, Optional[str]] = { - "service.name": os.getenv("OTEL_SERVICE_NAME", "litellm"), - "deployment.environment": os.getenv("OTEL_ENVIRONMENT_NAME", "production"), - # Fix the model_id to use proper environment variable or default to service name - "model_id": os.getenv( - "OTEL_MODEL_ID", os.getenv("OTEL_SERVICE_NAME", "litellm") - ), - } - - # Create base resource with LiteLLM-specific defaults - base_resource = Resource.create(base_attributes) # type: ignore - - # Create resource from OTEL_RESOURCE_ATTRIBUTES using the detector - otel_resource_detector = OTELResourceDetector() - env_resource = otel_resource_detector.detect() - - # Merge the resources: env_resource takes precedence over base_resource - # This ensures OTEL_RESOURCE_ATTRIBUTES overrides LiteLLM defaults - merged_resource = base_resource.merge(env_resource) - - return merged_resource - - @dataclass class OpenTelemetryConfig: exporter: Union[str, SpanExporter] = "console" @@ -93,6 +61,19 @@ class OpenTelemetryConfig: headers: Optional[str] = None enable_metrics: bool = False enable_events: bool = False + service_name: Optional[str] = None + deployment_environment: Optional[str] = None + model_id: Optional[str] = None + + def __post_init__(self) -> None: + if not self.service_name: + self.service_name = os.getenv("OTEL_SERVICE_NAME", "litellm") + if not self.deployment_environment: + self.deployment_environment = os.getenv( + "OTEL_ENVIRONMENT_NAME", "production" + ) + if not self.model_id: + self.model_id = os.getenv("OTEL_MODEL_ID", self.service_name) @classmethod def from_env(cls): @@ -122,6 +103,9 @@ class OpenTelemetryConfig: os.getenv("LITELLM_OTEL_INTEGRATION_ENABLE_EVENTS", "false").lower() == "true" ) + service_name = os.getenv("OTEL_SERVICE_NAME", "litellm") + deployment_environment = os.getenv("OTEL_ENVIRONMENT_NAME", "production") + model_id = os.getenv("OTEL_MODEL_ID", service_name) if exporter == "in_memory": return cls(exporter=InMemorySpanExporter()) @@ -131,6 +115,9 @@ class OpenTelemetryConfig: headers=headers, # example: OTEL_HEADERS=x-honeycomb-team=B85YgLm96***" enable_metrics=enable_metrics, enable_events=enable_events, + service_name=service_name, + deployment_environment=deployment_environment, + model_id=model_id, ) @@ -174,6 +161,22 @@ class OpenTelemetry(CustomLogger): self._init_logs(logger_provider) self._init_otel_logger_on_litellm_proxy() + @staticmethod + def _get_litellm_resource(config: OpenTelemetryConfig): + """Create an OpenTelemetry Resource using config-driven defaults.""" + from opentelemetry.sdk.resources import OTELResourceDetector, Resource + + base_attributes: Dict[str, Optional[str]] = { + "service.name": config.service_name, + "deployment.environment": config.deployment_environment, + "model_id": config.model_id or config.service_name, + } + + base_resource = Resource.create(base_attributes) # type: ignore[arg-type] + otel_resource_detector = OTELResourceDetector() + env_resource = otel_resource_detector.detect() + return base_resource.merge(env_resource) + def _init_otel_logger_on_litellm_proxy(self): """ Initializes OpenTelemetry for litellm proxy server @@ -266,7 +269,7 @@ class OpenTelemetry(CustomLogger): from opentelemetry.trace import SpanKind def create_tracer_provider(): - provider = TracerProvider(resource=_get_litellm_resource()) + provider = TracerProvider(resource=self._get_litellm_resource(self.config)) provider.add_span_processor(self._get_span_processor()) return provider @@ -300,7 +303,8 @@ class OpenTelemetry(CustomLogger): def create_meter_provider(): metric_reader = self._get_metric_reader() return MeterProvider( - metric_readers=[metric_reader], resource=_get_litellm_resource() + metric_readers=[metric_reader], + resource=self._get_litellm_resource(self.config), ) meter_provider = self._get_or_create_provider( @@ -355,7 +359,9 @@ class OpenTelemetry(CustomLogger): from opentelemetry.sdk._logs.export import BatchLogRecordProcessor def create_logger_provider(): - provider = OTLoggerProvider(resource=_get_litellm_resource()) + provider = OTLoggerProvider( + resource=self._get_litellm_resource(self.config) + ) log_exporter = self._get_log_exporter() provider.add_log_record_processor( BatchLogRecordProcessor(log_exporter) # type: ignore[arg-type] @@ -606,7 +612,7 @@ class OpenTelemetry(CustomLogger): from opentelemetry.sdk.trace import TracerProvider # Create a temporary tracer provider with dynamic headers - temp_provider = TracerProvider(resource=_get_litellm_resource()) + temp_provider = TracerProvider(resource=self._get_litellm_resource(self.config)) temp_provider.add_span_processor( self._get_span_processor(dynamic_headers=dynamic_headers) ) @@ -987,9 +993,9 @@ class OpenTelemetry(CustomLogger): # Get the resource from the logger provider logger_provider = get_logger_provider() - resource = ( - getattr(logger_provider, "_resource", None) or _get_litellm_resource() - ) + resource = getattr( + logger_provider, "_resource", None + ) or self._get_litellm_resource(self.config) parent_ctx = span.get_span_context() provider = (kwargs.get("litellm_params") or {}).get( @@ -1910,7 +1916,9 @@ class OpenTelemetry(CustomLogger): ) _split_otel_headers = OpenTelemetry._get_headers_dictionary(self.OTEL_HEADERS) - normalized_endpoint = self._normalize_otel_endpoint(self.OTEL_ENDPOINT, "metrics") + normalized_endpoint = self._normalize_otel_endpoint( + self.OTEL_ENDPOINT, "metrics" + ) if self.OTEL_EXPORTER == "console": exporter = ConsoleMetricExporter() diff --git a/litellm/integrations/prometheus.py b/litellm/integrations/prometheus.py index c01f7481277..e4aca5ced04 100644 --- a/litellm/integrations/prometheus.py +++ b/litellm/integrations/prometheus.py @@ -14,6 +14,7 @@ from typing import ( Literal, Optional, Tuple, + Union, cast, ) @@ -791,6 +792,11 @@ class PrometheusLogger(CustomLogger): f"standard_logging_object is required, got={standard_logging_payload}" ) + if self._should_skip_metrics_for_invalid_key( + kwargs=kwargs, standard_logging_payload=standard_logging_payload + ): + return + model = kwargs.get("model", "") litellm_params = kwargs.get("litellm_params", {}) or {} _metadata = litellm_params.get("metadata", {}) @@ -1189,11 +1195,17 @@ class PrometheusLogger(CustomLogger): f"prometheus Logging - Enters failure logging function for kwargs {kwargs}" ) - # unpack kwargs - model = kwargs.get("model", "") standard_logging_payload: StandardLoggingPayload = kwargs.get( "standard_logging_object", {} ) + + if self._should_skip_metrics_for_invalid_key( + kwargs=kwargs, standard_logging_payload=standard_logging_payload + ): + return + + model = kwargs.get("model", "") + litellm_params = kwargs.get("litellm_params", {}) or {} get_end_user_id_for_cost_tracking = _get_cached_end_user_id_for_cost_tracking() @@ -1207,7 +1219,6 @@ class PrometheusLogger(CustomLogger): user_api_team_alias = standard_logging_payload["metadata"][ "user_api_key_team_alias" ] - kwargs.get("exception", None) try: self.litellm_llm_api_failed_requests_metric.labels( @@ -1227,6 +1238,139 @@ class PrometheusLogger(CustomLogger): pass pass + def _extract_status_code( + self, + kwargs: Optional[dict] = None, + enum_values: Optional[Any] = None, + exception: Optional[Exception] = None, + ) -> Optional[int]: + """ + Extract HTTP status code from various input formats for validation. + + This is a centralized helper to extract status code from different + callback function signatures. Handles both ProxyException (uses 'code') + and standard exceptions (uses 'status_code'). + + Args: + kwargs: Dictionary potentially containing 'exception' key + enum_values: Object with 'status_code' attribute + exception: Exception object to extract status code from directly + + Returns: + Status code as integer if found, None otherwise + """ + status_code = None + + # Try from enum_values first (most common in our callbacks) + if enum_values and hasattr(enum_values, "status_code") and enum_values.status_code: + try: + status_code = int(enum_values.status_code) + except (ValueError, TypeError): + pass + + if not status_code and exception: + # ProxyException uses 'code' attribute, other exceptions may use 'status_code' + status_code = getattr(exception, "status_code", None) or getattr(exception, "code", None) + if status_code is not None: + try: + status_code = int(status_code) + except (ValueError, TypeError): + status_code = None + + if not status_code and kwargs: + exception_in_kwargs = kwargs.get("exception") + if exception_in_kwargs: + status_code = getattr(exception_in_kwargs, "status_code", None) or getattr(exception_in_kwargs, "code", None) + if status_code is not None: + try: + status_code = int(status_code) + except (ValueError, TypeError): + status_code = None + + return status_code + + def _is_invalid_api_key_request( + self, + status_code: Optional[int], + exception: Optional[Exception] = None, + ) -> bool: + """ + Determine if a request has an invalid API key based on status code and exception. + + This method prevents invalid authentication attempts from being recorded in + Prometheus metrics. A 401 status code is the definitive indicator of authentication + failure. Additionally, we check exception messages for authentication error patterns + to catch cases where the exception hasn't been converted to a ProxyException yet. + + Args: + status_code: HTTP status code (401 indicates authentication error) + exception: Exception object to check for auth-related error messages + + Returns: + True if the request has an invalid API key and metrics should be skipped, + False otherwise + """ + if status_code == 401: + return True + + # Handle cases where AssertionError is raised before conversion to ProxyException + if exception is not None: + exception_str = str(exception).lower() + auth_error_patterns = [ + "virtual key expected", + "expected to start with 'sk-'", + "authentication error", + "invalid api key", + "api key not valid", + ] + if any(pattern in exception_str for pattern in auth_error_patterns): + return True + + return False + + def _should_skip_metrics_for_invalid_key( + self, + kwargs: Optional[dict] = None, + user_api_key_dict: Optional[Any] = None, + enum_values: Optional[Any] = None, + standard_logging_payload: Optional[Union[dict, StandardLoggingPayload]] = None, + exception: Optional[Exception] = None, + ) -> bool: + """ + Determine if Prometheus metrics should be skipped for invalid API key requests. + + This is a centralized validation method that extracts status code and exception + information from various callback function signatures and determines if the request + represents an invalid API key attempt that should be filtered from metrics. + + Args: + kwargs: Dictionary potentially containing exception and other data + user_api_key_dict: User API key authentication object (currently unused) + enum_values: Object with status_code attribute + standard_logging_payload: Standard logging payload dictionary + exception: Exception object to check directly + + Returns: + True if metrics should be skipped (invalid key detected), False otherwise + """ + status_code = self._extract_status_code( + kwargs=kwargs, + enum_values=enum_values, + exception=exception, + ) + + if exception is None and kwargs: + exception = kwargs.get("exception") + + if self._is_invalid_api_key_request(status_code, exception=exception): + verbose_logger.debug( + "Skipping Prometheus metrics for invalid API key request: " + f"status_code={status_code}, exception={type(exception).__name__ if exception else None}" + ) + return True + + return False + async def async_post_call_failure_hook( self, request_data: dict, @@ -1252,6 +1396,14 @@ class PrometheusLogger(CustomLogger): StandardLoggingPayloadSetup, ) + if self._should_skip_metrics_for_invalid_key( + user_api_key_dict=user_api_key_dict, + exception=original_exception, + ): + return + + status_code = self._extract_status_code(exception=original_exception) + try: _tags = StandardLoggingPayloadSetup._get_request_tags( litellm_params=request_data, @@ -1266,8 +1418,8 @@ class PrometheusLogger(CustomLogger): team=user_api_key_dict.team_id, team_alias=user_api_key_dict.team_alias, requested_model=request_data.get("model", ""), - status_code=str(getattr(original_exception, "status_code", None)), - exception_status=str(getattr(original_exception, "status_code", None)), + status_code=str(status_code), + exception_status=str(status_code), exception_class=self._get_exception_class_name(original_exception), tags=_tags, route=user_api_key_dict.request_route, @@ -1305,6 +1457,11 @@ class PrometheusLogger(CustomLogger): StandardLoggingPayloadSetup, ) + if self._should_skip_metrics_for_invalid_key( + user_api_key_dict=user_api_key_dict + ): + return + enum_values = UserAPIKeyLabelValues( end_user=user_api_key_dict.end_user_id, hashed_api_key=user_api_key_dict.api_key, @@ -1360,6 +1517,15 @@ class PrometheusLogger(CustomLogger): exception = request_kwargs.get("exception", None) llm_provider = _litellm_params.get("custom_llm_provider", None) + + if self._should_skip_metrics_for_invalid_key( + kwargs=request_kwargs, + standard_logging_payload=standard_logging_payload, + ): + return + hashed_api_key = standard_logging_payload.get("metadata", {}).get( + "user_api_key_hash" + ) # Create enum_values for the label factory (always create for use in different metrics) enum_values = UserAPIKeyLabelValues( @@ -1374,9 +1540,7 @@ class PrometheusLogger(CustomLogger): self._get_exception_class_name(exception) if exception else None ), requested_model=model_group, - hashed_api_key=standard_logging_payload["metadata"][ - "user_api_key_hash" - ], + hashed_api_key=hashed_api_key, api_key_alias=standard_logging_payload["metadata"][ "user_api_key_alias" ], @@ -1441,6 +1605,14 @@ class PrometheusLogger(CustomLogger): if standard_logging_payload is None: return + # Skip recording metrics for invalid API key requests + if self._should_skip_metrics_for_invalid_key( + kwargs=request_kwargs, + enum_values=enum_values, + standard_logging_payload=standard_logging_payload, + ): + return + api_base = standard_logging_payload["api_base"] _litellm_params = request_kwargs.get("litellm_params", {}) or {} _metadata = _litellm_params.get("metadata", {}) diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index cd324935562..5448fe7c771 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -3630,6 +3630,7 @@ def _init_custom_logger_compatible_class( # noqa: PLR0915 otel_config = OpenTelemetryConfig( exporter=arize_config.protocol, endpoint=arize_config.endpoint, + service_name=arize_config.project_name, ) os.environ[ diff --git a/litellm/litellm_core_utils/prompt_templates/factory.py b/litellm/litellm_core_utils/prompt_templates/factory.py index 12570a02de7..0c331e43038 100644 --- a/litellm/litellm_core_utils/prompt_templates/factory.py +++ b/litellm/litellm_core_utils/prompt_templates/factory.py @@ -2137,6 +2137,14 @@ def anthropic_messages_pt( # noqa: PLR0915 assistant_content.append( cast(AnthropicMessagesTextParam, _cached_message) ) + # handle server_tool_use blocks (tool search, web search, etc.) + # Pass through as-is since these are Anthropic-native content types + elif m.get("type", "") == "server_tool_use": + assistant_content.append(m) # type: ignore + # handle tool_search_tool_result blocks + # Pass through as-is since these are Anthropic-native content types + elif m.get("type", "") == "tool_search_tool_result": + assistant_content.append(m) # type: ignore elif ( "content" in assistant_content_block and isinstance(assistant_content_block["content"], str) @@ -3168,6 +3176,11 @@ def _convert_to_bedrock_tool_call_invoke( id = tool["id"] name = tool["function"].get("name", "") arguments = tool["function"].get("arguments", "") + arguments_dict = json.loads(arguments) if arguments else {} + # Ensure arguments_dict is always a dict (Bedrock requires toolUse.input to be an object) + # When some providers return arguments: '""' (JSON-encoded empty string), json.loads returns "" + if not isinstance(arguments_dict, dict): + arguments_dict = {} if not arguments or not arguments.strip(): arguments_dict = {} else: diff --git a/litellm/llms/azure/azure.py b/litellm/llms/azure/azure.py index 994afa26e9c..ec4553fac4f 100644 --- a/litellm/llms/azure/azure.py +++ b/litellm/llms/azure/azure.py @@ -990,6 +990,10 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM): def create_azure_base_url( self, azure_client_params: dict, model: Optional[str] ) -> str: + from litellm.llms.azure_ai.image_generation import ( + AzureFoundryFluxImageGenerationConfig, + ) + api_base: str = azure_client_params.get( "azure_endpoint", "" ) # "https://example-endpoint.openai.azure.com" @@ -999,6 +1003,15 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM): if model is None: model = "" + # Handle FLUX 2 models on Azure AI which use a different URL pattern + # e.g., /providers/blackforestlabs/v1/flux-2-pro instead of /openai/deployments/{model}/images/generations + if AzureFoundryFluxImageGenerationConfig.is_flux2_model(model): + return AzureFoundryFluxImageGenerationConfig.get_flux2_image_generation_url( + api_base=api_base, + model=model, + api_version=api_version, + ) + if "/openai/deployments/" in api_base: base_url_with_deployment = api_base else: diff --git a/litellm/llms/azure_ai/image_edit/__init__.py b/litellm/llms/azure_ai/image_edit/__init__.py index e0e57bec403..e3acd610446 100644 --- a/litellm/llms/azure_ai/image_edit/__init__.py +++ b/litellm/llms/azure_ai/image_edit/__init__.py @@ -1,15 +1,28 @@ +from litellm.llms.azure_ai.image_generation.flux_transformation import ( + AzureFoundryFluxImageGenerationConfig, +) from litellm.llms.base_llm.image_edit.transformation import BaseImageEditConfig +from .flux2_transformation import AzureFoundryFlux2ImageEditConfig from .transformation import AzureFoundryFluxImageEditConfig -__all__ = ["AzureFoundryFluxImageEditConfig"] +__all__ = ["AzureFoundryFluxImageEditConfig", "AzureFoundryFlux2ImageEditConfig"] def get_azure_ai_image_edit_config(model: str) -> BaseImageEditConfig: - model = model.lower() - model = model.replace("-", "") - model = model.replace("_", "") - if model == "" or "flux" in model: # empty model is flux + """ + Get the appropriate image edit config for an Azure AI model. + + - FLUX 2 models use JSON with base64 image + - FLUX 1 models use multipart/form-data + """ + # Check if it's a FLUX 2 model + if AzureFoundryFluxImageGenerationConfig.is_flux2_model(model): + return AzureFoundryFlux2ImageEditConfig() + + # Default to FLUX 1 config for other FLUX models + model_normalized = model.lower().replace("-", "").replace("_", "") + if model_normalized == "" or "flux" in model_normalized: return AzureFoundryFluxImageEditConfig() - else: - raise ValueError(f"Model {model} is not supported for Azure AI image editing.") + + raise ValueError(f"Model {model} is not supported for Azure AI image editing.") diff --git a/litellm/llms/azure_ai/image_edit/flux2_transformation.py b/litellm/llms/azure_ai/image_edit/flux2_transformation.py new file mode 100644 index 00000000000..caa39056675 --- /dev/null +++ b/litellm/llms/azure_ai/image_edit/flux2_transformation.py @@ -0,0 +1,167 @@ +import base64 +from io import BufferedReader +from typing import Any, Dict, Optional, Tuple + +from httpx._types import RequestFiles + +import litellm +from litellm.llms.azure_ai.common_utils import AzureFoundryModelInfo +from litellm.llms.azure_ai.image_generation.flux_transformation import ( + AzureFoundryFluxImageGenerationConfig, +) +from litellm.llms.openai.image_edit.transformation import OpenAIImageEditConfig +from litellm.secret_managers.main import get_secret_str +from litellm.types.images.main import ImageEditOptionalRequestParams +from litellm.types.llms.openai import FileTypes +from litellm.types.router import GenericLiteLLMParams + + +class AzureFoundryFlux2ImageEditConfig(OpenAIImageEditConfig): + """ + Azure AI Foundry FLUX 2 image edit config + + Supports FLUX 2 models (e.g., flux.2-pro) for image editing. + Uses the same /providers/blackforestlabs/v1/flux-2-pro endpoint as image generation, + with the image passed as base64 in JSON body. + """ + + def get_supported_openai_params(self, model: str) -> list: + """ + FLUX 2 supports a subset of OpenAI image edit params + """ + return [ + "prompt", + "image", + "model", + "n", + "size", + ] + + def map_openai_params( + self, + image_edit_optional_params: ImageEditOptionalRequestParams, + model: str, + drop_params: bool, + ) -> Dict: + """ + Map OpenAI params to FLUX 2 params. + FLUX 2 uses the same param names as OpenAI for supported params. + """ + mapped_params: Dict[str, Any] = {} + supported_params = self.get_supported_openai_params(model) + + for key, value in dict(image_edit_optional_params).items(): + if key in supported_params and value is not None: + mapped_params[key] = value + + return mapped_params + + def use_multipart_form_data(self) -> bool: + """FLUX 2 uses JSON requests, not multipart/form-data.""" + return False + + def validate_environment( + self, + headers: dict, + model: str, + api_key: Optional[str] = None, + ) -> dict: + """ + Validate Azure AI Foundry environment and set up authentication + """ + api_key = AzureFoundryModelInfo.get_api_key(api_key) + + if not api_key: + raise ValueError( + f"Azure AI API key is required for model {model}. Set AZURE_AI_API_KEY environment variable or pass api_key parameter." + ) + + headers.update( + { + "Api-Key": api_key, + "Content-Type": "application/json", + } + ) + return headers + + def transform_image_edit_request( + self, + model: str, + prompt: str, + image: FileTypes, + image_edit_optional_request_params: Dict, + litellm_params: GenericLiteLLMParams, + headers: dict, + ) -> Tuple[Dict, RequestFiles]: + """ + Transform image edit request for FLUX 2. + + FLUX 2 uses the same endpoint for generation and editing, + with the image passed as base64 in the JSON body. + """ + image_b64 = self._convert_image_to_base64(image) + + # Build request body with required params + request_body: Dict[str, Any] = { + "prompt": prompt, + "image": image_b64, + "model": model, + } + + # Add mapped optional params (already filtered by map_openai_params) + request_body.update(image_edit_optional_request_params) + + # Return JSON body and empty files list (FLUX 2 doesn't use multipart) + return request_body, [] + + def _convert_image_to_base64(self, image: Any) -> str: + """Convert image file to base64 string""" + # Handle list of images (take first one) + if isinstance(image, list): + if len(image) == 0: + raise ValueError("Empty image list provided") + image = image[0] + + if isinstance(image, BufferedReader): + image_bytes = image.read() + image.seek(0) # Reset file pointer for potential reuse + elif isinstance(image, bytes): + image_bytes = image + elif hasattr(image, "read"): + image_bytes = image.read() # type: ignore + else: + raise ValueError(f"Unsupported image type: {type(image)}") + + return base64.b64encode(image_bytes).decode("utf-8") + + def get_complete_url( + self, + model: str, + api_base: Optional[str], + litellm_params: dict, + ) -> str: + """ + Constructs a complete URL for Azure AI Foundry FLUX 2 image edits. + + Uses the same /providers/blackforestlabs/v1/flux-2-pro endpoint as image generation. + """ + api_base = AzureFoundryModelInfo.get_api_base(api_base) + + if api_base is None: + raise ValueError( + "Azure AI API base is required. Set AZURE_AI_API_BASE environment variable or pass api_base parameter." + ) + + api_version = ( + litellm_params.get("api_version") + or litellm.api_version + or get_secret_str("AZURE_AI_API_VERSION") + or "preview" + ) + + return AzureFoundryFluxImageGenerationConfig.get_flux2_image_generation_url( + api_base=api_base, + model=model, + api_version=api_version, + ) + diff --git a/litellm/llms/azure_ai/image_edit/transformation.py b/litellm/llms/azure_ai/image_edit/transformation.py index 47f612912ce..930b6d4db90 100644 --- a/litellm/llms/azure_ai/image_edit/transformation.py +++ b/litellm/llms/azure_ai/image_edit/transformation.py @@ -71,9 +71,11 @@ class AzureFoundryFluxImageEditConfig(OpenAIImageEditConfig): "Azure AI API base is required. Set AZURE_AI_API_BASE environment variable or pass api_base parameter." ) - api_version = (litellm_params.get("api_version") or litellm.api_version - or get_secret_str("AZURE_AI_API_VERSION") - ) + api_version = ( + litellm_params.get("api_version") + or litellm.api_version + or get_secret_str("AZURE_AI_API_VERSION") + ) if api_version is None: # API version is mandatory for Azure AI Foundry raise ValueError( diff --git a/litellm/llms/azure_ai/image_generation/flux_transformation.py b/litellm/llms/azure_ai/image_generation/flux_transformation.py index 5325f32ef63..6a1868d94cc 100644 --- a/litellm/llms/azure_ai/image_generation/flux_transformation.py +++ b/litellm/llms/azure_ai/image_generation/flux_transformation.py @@ -1,3 +1,5 @@ +from typing import Optional + from litellm.llms.openai.image_generation import GPTImageGenerationConfig @@ -11,4 +13,56 @@ class AzureFoundryFluxImageGenerationConfig(GPTImageGenerationConfig): From our test suite - following GPTImageGenerationConfig is working for this model """ - pass + + @staticmethod + def get_flux2_image_generation_url( + api_base: Optional[str], + model: str, + api_version: Optional[str], + ) -> str: + """ + Constructs the complete URL for Azure AI FLUX 2 image generation. + + FLUX 2 models on Azure AI use a different URL pattern than standard Azure OpenAI: + - Standard: /openai/deployments/{model}/images/generations + - FLUX 2: /providers/blackforestlabs/v1/flux-2-pro + + Args: + api_base: Base URL (e.g., https://litellm-ci-cd-prod.services.ai.azure.com) + model: Model name (e.g., flux.2-pro) + api_version: API version (e.g., preview) + + Returns: + Complete URL for the FLUX 2 image generation endpoint + """ + if api_base is None: + raise ValueError( + "api_base is required for Azure AI FLUX 2 image generation" + ) + + api_base = api_base.rstrip("/") + api_version = api_version or "preview" + + # If the api_base already contains /providers/, it's already a complete path + if "/providers/" in api_base: + if "?" in api_base: + return api_base + return f"{api_base}?api-version={api_version}" + + # Construct the FLUX 2 provider path + # Model name flux.2-pro maps to endpoint flux-2-pro + return f"{api_base}/providers/blackforestlabs/v1/flux-2-pro?api-version={api_version}" + + @staticmethod + def is_flux2_model(model: str) -> bool: + """ + Check if the model is an Azure AI FLUX 2 model. + + Args: + model: Model name (e.g., flux.2-pro, azure_ai/flux.2-pro) + + Returns: + True if the model is a FLUX 2 model + """ + model_lower = model.lower().replace(".", "-").replace("_", "-") + return "flux-2" in model_lower or "flux2" in model_lower diff --git a/litellm/llms/custom_httpx/container_handler.py b/litellm/llms/custom_httpx/container_handler.py index ed112e4dd58..73017eaaf30 100644 --- a/litellm/llms/custom_httpx/container_handler.py +++ b/litellm/llms/custom_httpx/container_handler.py @@ -88,6 +88,34 @@ def _build_query_params( return params +def _prepare_multipart_file_upload( + file: Any, + headers: Dict[str, Any], +) -> tuple: + """ + Prepare file and headers for multipart upload. + + Returns: + Tuple of (files_dict, headers_without_content_type) + """ + from litellm.litellm_core_utils.prompt_templates.common_utils import ( + extract_file_data, + ) + + extracted = extract_file_data(file) + filename = extracted.get("filename") or "file" + content = extracted.get("content") or b"" + content_type = extracted.get("content_type") or "application/octet-stream" + files = {"file": (filename, content, content_type)} + + # Remove content-type header - httpx will set it automatically for multipart + headers_copy = headers.copy() + headers_copy.pop("content-type", None) + headers_copy.pop("Content-Type", None) + + return files, headers_copy + + class GenericContainerHandler: """ Generic handler for container file API endpoints. @@ -210,6 +238,7 @@ class GenericContainerHandler: # Make request method = endpoint_config["method"].upper() returns_binary = endpoint_config.get("returns_binary", False) + is_multipart = endpoint_config.get("is_multipart", False) try: if method == "GET": @@ -217,7 +246,11 @@ class GenericContainerHandler: elif method == "DELETE": response = http_client.delete(url=url, headers=headers, params=query_params) elif method == "POST": - response = http_client.post(url=url, headers=headers, params=query_params) + if is_multipart and "file" in kwargs: + files, headers = _prepare_multipart_file_upload(kwargs["file"], headers) + response = http_client.post(url=url, headers=headers, params=query_params, files=files) + else: + response = http_client.post(url=url, headers=headers, params=query_params) else: raise ValueError(f"Unsupported HTTP method: {method}") @@ -307,6 +340,7 @@ class GenericContainerHandler: # Make request method = endpoint_config["method"].upper() returns_binary = endpoint_config.get("returns_binary", False) + is_multipart = endpoint_config.get("is_multipart", False) try: if method == "GET": @@ -314,7 +348,11 @@ class GenericContainerHandler: elif method == "DELETE": response = await http_client.delete(url=url, headers=headers, params=query_params) elif method == "POST": - response = await http_client.post(url=url, headers=headers, params=query_params) + if is_multipart and "file" in kwargs: + files, headers = _prepare_multipart_file_upload(kwargs["file"], headers) + response = await http_client.post(url=url, headers=headers, params=query_params, files=files) + else: + response = await http_client.post(url=url, headers=headers, params=query_params) else: raise ValueError(f"Unsupported HTTP method: {method}") diff --git a/litellm/llms/openai/chat/gpt_transformation.py b/litellm/llms/openai/chat/gpt_transformation.py index 034ccae94ad..04a10bd7fbe 100644 --- a/litellm/llms/openai/chat/gpt_transformation.py +++ b/litellm/llms/openai/chat/gpt_transformation.py @@ -771,9 +771,9 @@ class OpenAIChatCompletionStreamingHandler(BaseModelResponseIterator): return ModelResponseStream( id=chunk["id"], object="chat.completion.chunk", - created=chunk["created"], - model=chunk["model"], - choices=chunk["choices"], + created=chunk.get("created"), + model=chunk.get("model"), + choices=chunk.get("choices", []), ) except Exception as e: raise e diff --git a/litellm/llms/sap/chat/transformation.py b/litellm/llms/sap/chat/transformation.py index d9307ed4b92..2b1573bf4ed 100755 --- a/litellm/llms/sap/chat/transformation.py +++ b/litellm/llms/sap/chat/transformation.py @@ -203,12 +203,8 @@ class GenAIHubOrchestrationConfig(OpenAIGPTConfig): litellm_params: dict, headers: dict, ) -> dict: - supported_params = self.get_supported_openai_params(model) - # Include extra params that passed validation (e.g., thinking_config for Gemini models via allowed_openai_params) - extra_params = [k for k in optional_params if k not in supported_params and k not in {"tools", "model_version"}] - supported_params = supported_params + extra_params model_params = { - k: v for k, v in optional_params.items() if k in supported_params + k: v for k, v in optional_params.items() if k not in {"tools", "model_version", "deployment_url"} } model_version = optional_params.pop("model_version", "latest") diff --git a/litellm/llms/vertex_ai/vertex_ai_partner_models/main.py b/litellm/llms/vertex_ai/vertex_ai_partner_models/main.py index 712a06dece1..123d925f7c1 100644 --- a/litellm/llms/vertex_ai/vertex_ai_partner_models/main.py +++ b/litellm/llms/vertex_ai/vertex_ai_partner_models/main.py @@ -40,6 +40,7 @@ class PartnerModelPrefixes(str, Enum): GPT_OSS_PREFIX = "openai/gpt-oss-" MINIMAX_PREFIX = "minimaxai/" MOONSHOT_PREFIX = "moonshotai/" + ZAI_PREFIX = "zai-org/" class VertexAIPartnerModels(VertexBase): @@ -66,6 +67,7 @@ class VertexAIPartnerModels(VertexBase): or model.startswith(PartnerModelPrefixes.GPT_OSS_PREFIX) or model.startswith(PartnerModelPrefixes.MINIMAX_PREFIX) or model.startswith(PartnerModelPrefixes.MOONSHOT_PREFIX) + or model.startswith(PartnerModelPrefixes.ZAI_PREFIX) ): return True return False @@ -79,6 +81,7 @@ class VertexAIPartnerModels(VertexBase): PartnerModelPrefixes.GPT_OSS_PREFIX, PartnerModelPrefixes.MINIMAX_PREFIX, PartnerModelPrefixes.MOONSHOT_PREFIX, + PartnerModelPrefixes.ZAI_PREFIX, ] if any(provider in model for provider in OPENAI_LIKE_VERTEX_PROVIDERS): return True diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index c7a2f60856d..73579db75cd 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -405,7 +405,23 @@ "supports_video_input": true, "supports_vision": true }, - + "amazon.nova-2-multimodal-embeddings-v1:0": { + "litellm_provider": "bedrock", + "max_input_tokens": 8172, + "max_tokens": 8172, + "mode": "embedding", + "input_cost_per_token": 1.35e-7, + "input_cost_per_image": 6e-5, + "input_cost_per_video_per_second": 0.0007, + "input_cost_per_audio_per_second": 0.00014, + "output_cost_per_token": 0.0, + "output_vector_size": 3072, + "source": "https://us-east-1.console.aws.amazon.com/bedrock/home?region=us-east-1#/model-catalog/serverless/amazon.nova-2-multimodal-embeddings-v1:0", + "supports_embedding_image_input": true, + "supports_image_input": true, + "supports_video_input": true, + "supports_audio_input": true + }, "amazon.nova-micro-v1:0": { "input_cost_per_token": 3.5e-08, "litellm_provider": "bedrock_converse", @@ -4893,6 +4909,15 @@ "/v1/images/generations" ] }, + "azure_ai/flux.2-pro": { + "litellm_provider": "azure_ai", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://ai.azure.com/explore/models/flux.2-pro/version/1/registry/azureml-blackforestlabs", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, "azure_ai/Llama-3.2-11B-Vision-Instruct": { "input_cost_per_token": 3.7e-07, "litellm_provider": "azure_ai", @@ -28320,6 +28345,19 @@ "supports_tool_choice": true, "supports_web_search": true }, + "vertex_ai/zai-org/glm-4.7-maas": { + "input_cost_per_token": 3e-07, + "litellm_provider": "vertex_ai-zai_models", + "max_input_tokens": 200000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 1.2e-06, + "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing#partner-models", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_tool_choice": true + }, "vertex_ai/mistral-medium-3": { "input_cost_per_token": 4e-07, "litellm_provider": "vertex_ai-mistral_models", @@ -32152,6 +32190,181 @@ "output_cost_per_token": 2e-07, "litellm_provider": "fireworks_ai", "mode": "chat" + }, + "llamagate/llama-3.1-8b": { + "max_tokens": 8192, + "max_input_tokens": 131072, + "max_output_tokens": 8192, + "input_cost_per_token": 3e-08, + "output_cost_per_token": 5e-08, + "litellm_provider": "llamagate", + "mode": "chat", + "supports_function_calling": true, + "supports_response_schema": true + }, + "llamagate/llama-3.2-3b": { + "max_tokens": 8192, + "max_input_tokens": 131072, + "max_output_tokens": 8192, + "input_cost_per_token": 4e-08, + "output_cost_per_token": 8e-08, + "litellm_provider": "llamagate", + "mode": "chat", + "supports_function_calling": true, + "supports_response_schema": true + }, + "llamagate/mistral-7b-v0.3": { + "max_tokens": 8192, + "max_input_tokens": 32768, + "max_output_tokens": 8192, + "input_cost_per_token": 1e-07, + "output_cost_per_token": 1.5e-07, + "litellm_provider": "llamagate", + "mode": "chat", + "supports_function_calling": true, + "supports_response_schema": true + }, + "llamagate/qwen3-8b": { + "max_tokens": 8192, + "max_input_tokens": 32768, + "max_output_tokens": 8192, + "input_cost_per_token": 4e-08, + "output_cost_per_token": 1.4e-07, + "litellm_provider": "llamagate", + "mode": "chat", + "supports_function_calling": true, + "supports_response_schema": true + }, + "llamagate/dolphin3-8b": { + "max_tokens": 8192, + "max_input_tokens": 128000, + "max_output_tokens": 8192, + "input_cost_per_token": 8e-08, + "output_cost_per_token": 1.5e-07, + "litellm_provider": "llamagate", + "mode": "chat", + "supports_function_calling": true, + "supports_response_schema": true + }, + "llamagate/deepseek-r1-8b": { + "max_tokens": 16384, + "max_input_tokens": 65536, + "max_output_tokens": 16384, + "input_cost_per_token": 1e-07, + "output_cost_per_token": 2e-07, + "litellm_provider": "llamagate", + "mode": "chat", + "supports_function_calling": true, + "supports_response_schema": true, + "supports_reasoning": true + }, + "llamagate/deepseek-r1-7b-qwen": { + "max_tokens": 16384, + "max_input_tokens": 131072, + "max_output_tokens": 16384, + "input_cost_per_token": 8e-08, + "output_cost_per_token": 1.5e-07, + "litellm_provider": "llamagate", + "mode": "chat", + "supports_function_calling": true, + "supports_response_schema": true, + "supports_reasoning": true + }, + "llamagate/openthinker-7b": { + "max_tokens": 8192, + "max_input_tokens": 32768, + "max_output_tokens": 8192, + "input_cost_per_token": 8e-08, + "output_cost_per_token": 1.5e-07, + "litellm_provider": "llamagate", + "mode": "chat", + "supports_function_calling": true, + "supports_response_schema": true, + "supports_reasoning": true + }, + "llamagate/qwen2.5-coder-7b": { + "max_tokens": 8192, + "max_input_tokens": 32768, + "max_output_tokens": 8192, + "input_cost_per_token": 6e-08, + "output_cost_per_token": 1.2e-07, + "litellm_provider": "llamagate", + "mode": "chat", + "supports_function_calling": true, + "supports_response_schema": true + }, + "llamagate/deepseek-coder-6.7b": { + "max_tokens": 4096, + "max_input_tokens": 16384, + "max_output_tokens": 4096, + "input_cost_per_token": 6e-08, + "output_cost_per_token": 1.2e-07, + "litellm_provider": "llamagate", + "mode": "chat", + "supports_function_calling": true, + "supports_response_schema": true + }, + "llamagate/codellama-7b": { + "max_tokens": 4096, + "max_input_tokens": 16384, + "max_output_tokens": 4096, + "input_cost_per_token": 6e-08, + "output_cost_per_token": 1.2e-07, + "litellm_provider": "llamagate", + "mode": "chat", + "supports_function_calling": true, + "supports_response_schema": true + }, + "llamagate/qwen3-vl-8b": { + "max_tokens": 8192, + "max_input_tokens": 32768, + "max_output_tokens": 8192, + "input_cost_per_token": 1.5e-07, + "output_cost_per_token": 5.5e-07, + "litellm_provider": "llamagate", + "mode": "chat", + "supports_function_calling": true, + "supports_response_schema": true, + "supports_vision": true + }, + "llamagate/llava-7b": { + "max_tokens": 2048, + "max_input_tokens": 4096, + "max_output_tokens": 2048, + "input_cost_per_token": 1e-07, + "output_cost_per_token": 2e-07, + "litellm_provider": "llamagate", + "mode": "chat", + "supports_response_schema": true, + "supports_vision": true + }, + "llamagate/gemma3-4b": { + "max_tokens": 8192, + "max_input_tokens": 128000, + "max_output_tokens": 8192, + "input_cost_per_token": 3e-08, + "output_cost_per_token": 8e-08, + "litellm_provider": "llamagate", + "mode": "chat", + "supports_function_calling": true, + "supports_response_schema": true, + "supports_vision": true + }, + "llamagate/nomic-embed-text": { + "max_tokens": 8192, + "max_input_tokens": 8192, + "input_cost_per_token": 2e-08, + "output_cost_per_token": 0, + "litellm_provider": "llamagate", + "mode": "embedding" + }, + "llamagate/qwen3-embedding-8b": { + "max_tokens": 40960, + "max_input_tokens": 40960, + "input_cost_per_token": 2e-08, + "output_cost_per_token": 0, + "litellm_provider": "llamagate", + "mode": "embedding" } } diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index cff3bee1ca4..4140273ea25 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -863,6 +863,7 @@ class KeyRequestBase(GenerateRequestBase): tpm_limit_type: Optional[ Literal["guaranteed_throughput", "best_effort_throughput", "dynamic"] ] = None # raise an error if 'guaranteed_throughput' is set and we're overallocating tpm + router_settings: Optional[UpdateRouterConfig] = None class LiteLLMKeyType(str, enum.Enum): @@ -918,6 +919,7 @@ class GenerateKeyResponse(KeyRequestBase): "config", "permissions", "model_max_budget", + "router_settings", ] for field in dict_fields: value = values.get(field) @@ -1460,6 +1462,7 @@ class TeamBase(LiteLLMPydanticObjectBase): models: list = [] blocked: bool = False + router_settings: Optional[dict] = None class NewTeamRequest(TeamBase): @@ -1542,6 +1545,7 @@ class UpdateTeamRequest(LiteLLMPydanticObjectBase): model_rpm_limit: Optional[Dict[str, int]] = None model_tpm_limit: Optional[Dict[str, int]] = None allowed_vector_store_indexes: Optional[List[AllowedVectorStoreIndexItem]] = None + router_settings: Optional[dict] = None class ResetTeamBudgetRequest(LiteLLMPydanticObjectBase): @@ -1684,6 +1688,7 @@ class LiteLLM_TeamTable(TeamBase): "permissions", "model_max_budget", "model_aliases", + "router_settings", ] if isinstance(values, BaseModel): @@ -3735,6 +3740,7 @@ class BaseDailySpendTransaction(TypedDict): model_group: Optional[str] mcp_namespaced_tool_name: Optional[str] custom_llm_provider: Optional[str] + endpoint: Optional[str] # token count metrics prompt_tokens: int diff --git a/litellm/proxy/common_utils/performance_utils.md b/litellm/proxy/common_utils/performance_utils.md new file mode 100644 index 00000000000..331955fe4bf --- /dev/null +++ b/litellm/proxy/common_utils/performance_utils.md @@ -0,0 +1,214 @@ +# Performance Utilities Documentation + +This module provides performance monitoring and profiling functionality for LiteLLM proxy server using `cProfile` and `line_profiler`. + +## Table of Contents + +- [Line Profiler Usage](#line-profiler-usage) + - [Example 1: Wrapping a function directly](#example-1-wrapping-a-function-directly) + - [Example 2: Wrapping a module function dynamically](#example-2-wrapping-a-module-function-dynamically) + - [Example 3: Manual stats collection](#example-3-manual-stats-collection) + - [Example 4: Analyzing the profile output](#example-4-analyzing-the-profile-output) + - [Example 5: Using in a decorator pattern](#example-5-using-in-a-decorator-pattern) +- [cProfile Usage](#cprofile-usage) +- [Installation](#installation) +- [Notes](#notes) + +## Line Profiler Usage + +### Example 1: Wrapping a function directly + +This is how it's used in `litellm/utils.py` to profile `wrapper_async`: + +```python +from litellm.proxy.common_utils.performance_utils import ( + register_shutdown_handler, + wrap_function_directly, +) + +def client(original_function): + @wraps(original_function) + async def wrapper_async(*args, **kwargs): + # ... function implementation ... + pass + + # Wrap the function with line_profiler + wrapper_async = wrap_function_directly(wrapper_async) + + # Register shutdown handler to collect stats on server shutdown + register_shutdown_handler(output_file="wrapper_async_line_profile.lprof") + + return wrapper_async +``` + +### Example 2: Wrapping a module function dynamically + +```python +import my_module +from litellm.proxy.common_utils.performance_utils import ( + wrap_function_with_line_profiler, + register_shutdown_handler, +) + +# Wrap a function in a module +wrap_function_with_line_profiler(my_module, "expensive_function") + +# Register shutdown handler +register_shutdown_handler(output_file="my_profile.lprof") + +# Now all calls to my_module.expensive_function will be profiled +my_module.expensive_function() +``` + +### Example 3: Manual stats collection + +```python +from litellm.proxy.common_utils.performance_utils import ( + wrap_function_directly, + collect_line_profiler_stats, +) + +def my_function(): + # ... implementation ... + pass + +# Wrap the function +my_function = wrap_function_directly(my_function) + +# Run your code +my_function() + +# Collect stats manually (instead of waiting for shutdown) +collect_line_profiler_stats(output_file="manual_profile.lprof") +``` + +### Example 4: Analyzing the profile output + +After running your code, analyze the `.lprof` file: + +```bash +# View the profile +python -m line_profiler wrapper_async_line_profile.lprof + +# Save to text file +python -m line_profiler wrapper_async_line_profile.lprof > profile_report.txt +``` + +The output shows: +- **Line #**: Line number in the source file +- **Hits**: Number of times the line was executed +- **Time**: Total time spent on that line (in microseconds) +- **Per Hit**: Average time per execution +- **% Time**: Percentage of total function time +- **Line Contents**: The actual source code + +Example output: +``` +Timer unit: 1e-06 s + +Total time: 3.73697 s +File: litellm/utils.py +Function: client..wrapper_async at line 1657 + +Line # Hits Time Per Hit % Time Line Contents +============================================================== + 1657 @wraps(original_function) + 1658 async def wrapper_async(*args, **kwargs): + 1659 2005 7577.1 3.8 0.2 print_args_passed_to_litellm(...) + 1763 2005 1351909.0 674.3 36.2 result = await original_function(*args, **kwargs) + 1846 4010 1543688.1 385.0 41.3 update_response_metadata(...) +``` + +### Example 5: Using in a decorator pattern + +```python +from litellm.proxy.common_utils.performance_utils import ( + wrap_function_directly, + register_shutdown_handler, +) + +def profile_decorator(func): + # Wrap the function + profiled_func = wrap_function_directly(func) + + # Register shutdown handler (only once) + if not hasattr(profile_decorator, '_registered'): + register_shutdown_handler(output_file="decorated_functions.lprof") + profile_decorator._registered = True + + return profiled_func + +@profile_decorator +async def my_async_function(): + # This function will be profiled + pass +``` + +## cProfile Usage + +### Example: Using the profile_endpoint decorator + +```python +from litellm.proxy.common_utils.performance_utils import profile_endpoint + +@profile_endpoint(sampling_rate=0.1) # Profile 10% of requests +async def my_endpoint(): + # ... implementation ... + pass +``` + +The `sampling_rate` parameter controls what percentage of requests are profiled: +- `1.0`: Profile all requests (100%) +- `0.1`: Profile 1 in 10 requests (10%) +- `0.0`: Profile no requests (0%) + +## Installation + +`line_profiler` must be installed to use the line profiling functionality: + +```bash +pip install line_profiler +``` + +On Windows with Python 3.14+, you may need to install Microsoft Visual C++ Build Tools to compile `line_profiler` from source. + +## Notes + +- The profiler aggregates stats by source code location, so multiple instances of the same function (e.g., closures) will be profiled together +- Stats are automatically collected on server shutdown via `atexit` handler when using `register_shutdown_handler()` +- You can also manually collect stats using `collect_line_profiler_stats()` +- The line profiler will fail with an `ImportError` if `line_profiler` is not installed (as configured in `litellm/utils.py`) + +## API Reference + +### `wrap_function_directly(func: Callable) -> Callable` + +Wrap a function directly with line_profiler. This is the recommended way to profile functions, especially closures or functions created dynamically. + +**Raises:** +- `ImportError`: If line_profiler is not available +- `RuntimeError`: If line_profiler cannot be enabled or function cannot be wrapped + +### `wrap_function_with_line_profiler(module: Any, function_name: str) -> bool` + +Dynamically wrap a function in a module with line_profiler. + +**Returns:** `True` if wrapping was successful, `False` otherwise + +### `collect_line_profiler_stats(output_file: Optional[str] = None) -> None` + +Collect and save line_profiler statistics. If `output_file` is provided, saves to file. Otherwise, prints to stdout. + +### `register_shutdown_handler(output_file: Optional[str] = None) -> None` + +Register an `atexit` handler that will automatically save profiling statistics when the Python process exits. Safe to call multiple times (only registers once). + +**Default output file:** `line_profile_stats.lprof` if not specified + +### `profile_endpoint(sampling_rate: float = 1.0)` + +Decorator to sample endpoint hits and save to a profile file using cProfile. + +**Args:** +- `sampling_rate`: Rate of requests to profile (0.0 to 1.0) + diff --git a/litellm/proxy/common_utils/performance_utils.py b/litellm/proxy/common_utils/performance_utils.py index fe238f2e331..f9537f85e2b 100644 --- a/litellm/proxy/common_utils/performance_utils.py +++ b/litellm/proxy/common_utils/performance_utils.py @@ -2,14 +2,19 @@ Performance utilities for LiteLLM proxy server. This module provides performance monitoring and profiling functionality for endpoint -performance analysis using cProfile with configurable sampling rates. +performance analysis using cProfile with configurable sampling rates, and line_profiler +for line-by-line profiling. + +See performance_utils.md for detailed usage examples and documentation. """ import asyncio +import atexit import cProfile import functools import threading from pathlib import Path as PathLib +from typing import Any, Callable, Optional from litellm._logging import verbose_proxy_logger @@ -20,6 +25,11 @@ _last_profile_file_path = None _sample_counter = 0 _sample_counter_lock = threading.Lock() +# Global line_profiler state +_line_profiler: Optional[Any] = None +_line_profiler_lock = threading.Lock() +_wrapped_functions: dict[str, Callable] = {} # Store original functions + def _should_sample(profile_sampling_rate: float) -> bool: """Determine if current request should be sampled based on sampling rate.""" @@ -123,3 +133,156 @@ def profile_endpoint(sampling_rate: float = 1.0): raise return sync_wrapper return decorator + + +def enable_line_profiler() -> None: + """Enable line_profiler for dynamic function wrapping. + + Raises: + ImportError: If line_profiler is not available + """ + global _line_profiler + from line_profiler import LineProfiler # Will raise ImportError if not available + + with _line_profiler_lock: + if _line_profiler is None: + _line_profiler = LineProfiler() + verbose_proxy_logger.info("Line profiler enabled") + + +def wrap_function_with_line_profiler(module: Any, function_name: str) -> bool: + """Dynamically wrap a function with line_profiler. + + Args: + module: The module containing the function + function_name: Name of the function to wrap + + Returns: + True if wrapping was successful, False otherwise + """ + try: + enable_line_profiler() # May raise ImportError if not available + except ImportError: + return False + + if _line_profiler is None: + return False + + try: + original_function = getattr(module, function_name, None) + if original_function is None: + verbose_proxy_logger.warning( + f"Function {function_name} not found in module {module.__name__}" + ) + return False + + # Store original function if not already wrapped + if function_name not in _wrapped_functions: + _wrapped_functions[function_name] = original_function + + # Wrap with line_profiler + profiled_function = _line_profiler(original_function) + setattr(module, function_name, profiled_function) + + verbose_proxy_logger.info( + f"Wrapped {module.__name__}.{function_name} with line_profiler" + ) + return True + except Exception as e: + verbose_proxy_logger.error( + f"Error wrapping {function_name} with line_profiler: {e}" + ) + return False + + +def wrap_function_directly(func: Callable) -> Callable: + """Wrap a function directly with line_profiler. + + This is the recommended way to profile functions, especially closures or + functions created dynamically (like wrapper_async in litellm/utils.py). + + Args: + func: The function to wrap + + Returns: + The wrapped function that will be profiled when called + + Raises: + ImportError: If line_profiler is not available + RuntimeError: If line_profiler cannot be enabled or function cannot be wrapped + """ + import warnings + + enable_line_profiler() # Will raise ImportError if not available + + if _line_profiler is None: + raise RuntimeError("Line profiler was not initialized") + + # Suppress warnings about __wrapped__ - we intentionally want to profile the wrapper + with warnings.catch_warnings(): + warnings.filterwarnings('ignore', message='.*__wrapped__.*', category=UserWarning) + # Add function to line_profiler and wrap it + _line_profiler.add_function(func) + profiled_function = _line_profiler(func) + + verbose_proxy_logger.info( + f"Wrapped function {func.__name__} with line_profiler" + ) + return profiled_function + + +def collect_line_profiler_stats(output_file: Optional[str] = None) -> None: + """Collect and save line_profiler statistics. + + This can be called manually to collect stats at any time, or it's + automatically called on shutdown if register_shutdown_handler() was used. + + Args: + output_file: Optional path to save stats. If None, prints to stdout. + """ + global _line_profiler + + with _line_profiler_lock: + if _line_profiler is None: + verbose_proxy_logger.debug("Line profiler not enabled, nothing to collect") + return + + try: + if output_file: + # Save to file + output_path = PathLib(output_file) + _line_profiler.dump_stats(str(output_path)) + verbose_proxy_logger.info( + f"Line profiler stats saved to {output_path}" + ) + else: + # Print to stdout + from io import StringIO + + stream = StringIO() + _line_profiler.print_stats(stream=stream) + stats_output = stream.getvalue() + verbose_proxy_logger.info("Line profiler stats:\n" + stats_output) + except Exception as e: + verbose_proxy_logger.error(f"Error collecting line profiler stats: {e}") + + +def register_shutdown_handler(output_file: Optional[str] = None) -> None: + """Register a shutdown handler to collect line_profiler stats. + + This registers an atexit handler that will automatically save profiling + statistics when the Python process exits. Safe to call multiple times + (only registers once). + + Args: + output_file: Optional path to save stats on shutdown. + Defaults to 'line_profile_stats.lprof' + """ + if output_file is None: + output_file = "line_profile_stats.lprof" + + def shutdown_handler(): + collect_line_profiler_stats(output_file=output_file) + + atexit.register(shutdown_handler) + verbose_proxy_logger.debug(f"Registered line_profiler shutdown handler for {output_file}") diff --git a/litellm/proxy/container_endpoints/handler_factory.py b/litellm/proxy/container_endpoints/handler_factory.py index 7eee44afb4b..dc10e39bc91 100644 --- a/litellm/proxy/container_endpoints/handler_factory.py +++ b/litellm/proxy/container_endpoints/handler_factory.py @@ -43,7 +43,7 @@ def _get_container_provider_config(custom_llm_provider: str): raise ValueError(f"Container API not supported for provider: {custom_llm_provider}") -def _create_handler_for_path_params(path_params: List[str], route_type: str, returns_binary: bool = False): +def _create_handler_for_path_params(path_params: List[str], route_type: str, returns_binary: bool = False, is_multipart: bool = False): """ Dynamically create a handler with the correct path parameter signature. """ @@ -63,6 +63,23 @@ def _create_handler_for_path_params(path_params: List[str], route_type: str, ret ) return handler_binary_content + # For multipart file upload endpoints + if is_multipart: + async def handler_multipart_upload( + request: Request, + container_id: str, + fastapi_response: Response, + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), + ): + return await _process_multipart_upload_request( + request=request, + fastapi_response=fastapi_response, + user_api_key_dict=user_api_key_dict, + route_type=route_type, + container_id=container_id, + ) + return handler_multipart_upload + # Create handlers for different path parameter combinations if path_params == ["container_id"]: async def handler_container_id( @@ -193,6 +210,83 @@ async def _process_binary_request( raise e +async def _process_multipart_upload_request( + request: Request, + fastapi_response: Response, + user_api_key_dict: UserAPIKeyAuth, + route_type: str, + container_id: str, +): + """Process multipart file upload requests.""" + from litellm.proxy.common_utils.http_parsing_utils import ( + convert_upload_files_to_file_data, + get_form_data, + ) + from litellm.proxy.proxy_server import ( + general_settings, + llm_router, + proxy_config, + proxy_logging_obj, + select_data_generator, + user_api_base, + user_max_tokens, + user_model, + user_request_timeout, + user_temperature, + version, + ) + + # Parse multipart form data and convert files + form_data = await get_form_data(request) + data = await convert_upload_files_to_file_data(form_data) + + if "file" not in data: + from fastapi import HTTPException + raise HTTPException(status_code=400, detail="Missing required 'file' field") + + # convert_upload_files_to_file_data returns list of tuples, extract single file + file_list = data["file"] + if isinstance(file_list, list) and len(file_list) > 0: + data["file"] = file_list[0] + + data["container_id"] = container_id + + custom_llm_provider = ( + get_custom_llm_provider_from_request_headers(request=request) + or get_custom_llm_provider_from_request_query(request=request) + or "openai" + ) + data["custom_llm_provider"] = custom_llm_provider + + processor = ProxyBaseLLMRequestProcessing(data=data) + try: + return await processor.base_process_llm_request( + request=request, + fastapi_response=fastapi_response, + user_api_key_dict=user_api_key_dict, + route_type=route_type, # type: ignore[arg-type] + proxy_logging_obj=proxy_logging_obj, + llm_router=llm_router, + general_settings=general_settings, + proxy_config=proxy_config, + select_data_generator=select_data_generator, + model=None, + user_model=user_model, + user_temperature=user_temperature, + user_request_timeout=user_request_timeout, + user_max_tokens=user_max_tokens, + user_api_base=user_api_base, + version=version, + ) + except Exception as e: + raise await processor._handle_llm_api_exception( + e=e, + user_api_key_dict=user_api_key_dict, + proxy_logging_obj=proxy_logging_obj, + version=version, + ) + + async def _process_request( request: Request, fastapi_response: Response, @@ -272,9 +366,10 @@ def register_container_file_endpoints(router: APIRouter) -> None: path_params = endpoint_config.get("path_params", []) route_type = endpoint_config["async_name"] returns_binary = endpoint_config.get("returns_binary", False) + is_multipart = endpoint_config.get("is_multipart", False) # Create handler with correct signature for path params - handler = _create_handler_for_path_params(path_params, route_type, returns_binary) + handler = _create_handler_for_path_params(path_params, route_type, returns_binary, is_multipart) # Register routes route_method = getattr(router, method) diff --git a/litellm/proxy/db/db_spend_update_writer.py b/litellm/proxy/db/db_spend_update_writer.py index 5c5cd7c19f7..429e56c805b 100644 --- a/litellm/proxy/db/db_spend_update_writer.py +++ b/litellm/proxy/db/db_spend_update_writer.py @@ -42,6 +42,7 @@ from litellm.proxy.db.db_transaction_queue.daily_spend_update_queue import ( from litellm.proxy.db.db_transaction_queue.pod_lock_manager import PodLockManager from litellm.proxy.db.db_transaction_queue.redis_update_buffer import RedisUpdateBuffer from litellm.proxy.db.db_transaction_queue.spend_update_queue import SpendUpdateQueue +from litellm.proxy.route_llm_request import ROUTE_ENDPOINT_MAPPING if TYPE_CHECKING: from litellm.proxy.utils import PrismaClient, ProxyLogging @@ -1205,6 +1206,7 @@ class DBSpendUpdateWriter: "mcp_namespaced_tool_name" ) or "", + "endpoint": transaction.get("endpoint") or "", } } @@ -1225,6 +1227,7 @@ class DBSpendUpdateWriter: "custom_llm_provider": transaction.get( "custom_llm_provider" ), + "endpoint": transaction.get("endpoint"), "prompt_tokens": transaction["prompt_tokens"], "completion_tokens": transaction["completion_tokens"], "spend": transaction["spend"], @@ -1287,6 +1290,9 @@ class DBSpendUpdateWriter: if entity_type == "tag" and "request_id" in transaction: update_data["request_id"] = transaction.get("request_id") + # Add endpoint to update_data so existing rows get their endpoint field updated + update_data["endpoint"] = transaction.get("endpoint") or "" + table.upsert( where=where_clause, data={ @@ -1347,7 +1353,7 @@ class DBSpendUpdateWriter: entity_type="user", entity_id_field="user_id", table_name="litellm_dailyuserspend", - unique_constraint_name="user_id_date_api_key_model_custom_llm_provider_mcp_namespaced_tool_name", + unique_constraint_name="user_id_date_api_key_model_custom_llm_provider_mcp_namespaced_tool_name_endpoint", ) @staticmethod @@ -1368,7 +1374,7 @@ class DBSpendUpdateWriter: entity_type="team", entity_id_field="team_id", table_name="litellm_dailyteamspend", - unique_constraint_name="team_id_date_api_key_model_custom_llm_provider_mcp_namespaced_tool_name", + unique_constraint_name="team_id_date_api_key_model_custom_llm_provider_mcp_namespaced_tool_name_endpoint", ) @staticmethod @@ -1389,7 +1395,7 @@ class DBSpendUpdateWriter: entity_type="org", entity_id_field="organization_id", table_name="litellm_dailyorganizationspend", - unique_constraint_name="organization_id_date_api_key_model_custom_llm_provider_mcp_namespaced_tool_name", + unique_constraint_name="organization_id_date_api_key_model_custom_llm_provider_mcp_namespaced_tool_name_endpoint", ) @staticmethod @@ -1410,7 +1416,7 @@ class DBSpendUpdateWriter: entity_type="end_user", entity_id_field="end_user_id", table_name="litellm_dailyenduserspend", - unique_constraint_name="end_user_id_date_api_key_model_custom_llm_provider_mcp_namespaced_tool_name", + unique_constraint_name="end_user_id_date_api_key_model_custom_llm_provider_mcp_namespaced_tool_name_endpoint", ) @staticmethod @@ -1431,7 +1437,7 @@ class DBSpendUpdateWriter: entity_type="agent", entity_id_field="agent_id", table_name="litellm_dailyagentspend", - unique_constraint_name="agent_id_date_api_key_model_custom_llm_provider_mcp_namespaced_tool_name", + unique_constraint_name="agent_id_date_api_key_model_custom_llm_provider_mcp_namespaced_tool_name_endpoint", ) @staticmethod @@ -1452,7 +1458,7 @@ class DBSpendUpdateWriter: entity_type="tag", entity_id_field="tag", table_name="litellm_dailytagspend", - unique_constraint_name="tag_date_api_key_model_custom_llm_provider_mcp_namespaced_tool_name", + unique_constraint_name="tag_date_api_key_model_custom_llm_provider_mcp_namespaced_tool_name_endpoint", ) async def _common_add_spend_log_transaction_to_daily_transaction( @@ -1513,6 +1519,12 @@ class DBSpendUpdateWriter: ) return None try: + # Map call_type to endpoint using ROUTE_ENDPOINT_MAPPING + call_type = payload.get("call_type", None) + endpoint = None + if call_type: + endpoint = ROUTE_ENDPOINT_MAPPING.get(call_type, None) + daily_transaction = BaseDailySpendTransaction( date=date, api_key=payload["api_key"], @@ -1520,6 +1532,7 @@ class DBSpendUpdateWriter: model_group=payload.get("model_group", None), mcp_namespaced_tool_name=payload.get("mcp_namespaced_tool_name", None), custom_llm_provider=payload.get("custom_llm_provider", None), + endpoint=endpoint, prompt_tokens=payload["prompt_tokens"], completion_tokens=payload["completion_tokens"], spend=payload["spend"], @@ -1563,7 +1576,8 @@ class DBSpendUpdateWriter: if base_daily_transaction is None: return - daily_transaction_key = f"{payload['user']}_{base_daily_transaction['date']}_{payload['api_key']}_{payload['model']}_{payload['custom_llm_provider']}" + endpoint_str = base_daily_transaction.get("endpoint") or "" + daily_transaction_key = f"{payload['user']}_{base_daily_transaction['date']}_{payload['api_key']}_{payload['model']}_{payload['custom_llm_provider']}_{endpoint_str}" daily_transaction = DailyUserSpendTransaction( user_id=payload["user"], **base_daily_transaction ) @@ -1595,7 +1609,8 @@ class DBSpendUpdateWriter: ) return - daily_transaction_key = f"{payload['team_id']}_{base_daily_transaction['date']}_{payload['api_key']}_{payload['model']}_{payload['custom_llm_provider']}" + endpoint_str = base_daily_transaction.get("endpoint") or "" + daily_transaction_key = f"{payload['team_id']}_{base_daily_transaction['date']}_{payload['api_key']}_{payload['model']}_{payload['custom_llm_provider']}_{endpoint_str}" daily_transaction = DailyTeamSpendTransaction( team_id=payload["team_id"], **base_daily_transaction ) @@ -1637,7 +1652,8 @@ class DBSpendUpdateWriter: if base_daily_transaction is None: return - daily_transaction_key = f"{org_id}_{base_daily_transaction['date']}_{payload_with_org['api_key']}_{payload_with_org['model']}_{payload_with_org['custom_llm_provider']}" + endpoint_str = base_daily_transaction.get("endpoint") or "" + daily_transaction_key = f"{org_id}_{base_daily_transaction['date']}_{payload_with_org['api_key']}_{payload_with_org['model']}_{payload_with_org['custom_llm_provider']}_{endpoint_str}" daily_transaction = DailyOrganizationSpendTransaction( organization_id=org_id, **base_daily_transaction ) @@ -1679,7 +1695,8 @@ class DBSpendUpdateWriter: if base_daily_transaction is None: return - daily_transaction_key = f"{end_user_id}_{base_daily_transaction['date']}_{payload_with_end_user_id['api_key']}_{payload_with_end_user_id['model']}_{payload_with_end_user_id['custom_llm_provider']}" + endpoint_str = base_daily_transaction.get("endpoint") or "" + daily_transaction_key = f"{end_user_id}_{base_daily_transaction['date']}_{payload_with_end_user_id['api_key']}_{payload_with_end_user_id['model']}_{payload_with_end_user_id['custom_llm_provider']}_{endpoint_str}" daily_transaction = DailyEndUserSpendTransaction( end_user_id=end_user_id, **base_daily_transaction ) @@ -1723,7 +1740,8 @@ class DBSpendUpdateWriter: ) if base_daily_transaction is None: return - daily_transaction_key = f"{payload['agent_id']}_{base_daily_transaction['date']}_{payload_with_agent_id['api_key']}_{payload_with_agent_id['model']}_{payload_with_agent_id['custom_llm_provider']}" + endpoint_str = base_daily_transaction.get("endpoint") or "" + daily_transaction_key = f"{payload['agent_id']}_{base_daily_transaction['date']}_{payload_with_agent_id['api_key']}_{payload_with_agent_id['model']}_{payload_with_agent_id['custom_llm_provider']}_{endpoint_str}" daily_transaction = DailyAgentSpendTransaction( agent_id=payload['agent_id'], **base_daily_transaction ) @@ -1763,7 +1781,8 @@ class DBSpendUpdateWriter: else: raise ValueError(f"Invalid request_tags: {payload['request_tags']}") for tag in request_tags: - daily_transaction_key = f"{tag}_{base_daily_transaction['date']}_{payload['api_key']}_{payload['model']}_{payload['custom_llm_provider']}" + endpoint_str = base_daily_transaction.get("endpoint") or "" + daily_transaction_key = f"{tag}_{base_daily_transaction['date']}_{payload['api_key']}_{payload['model']}_{payload['custom_llm_provider']}_{endpoint_str}" daily_transaction = DailyTagSpendTransaction( tag=tag, **base_daily_transaction, request_id=payload["request_id"] ) diff --git a/litellm/proxy/management_endpoints/common_daily_activity.py b/litellm/proxy/management_endpoints/common_daily_activity.py index cd28cbb7145..f52abf86b97 100644 --- a/litellm/proxy/management_endpoints/common_daily_activity.py +++ b/litellm/proxy/management_endpoints/common_daily_activity.py @@ -227,6 +227,41 @@ def update_breakdown_metrics( ) ) + # Update endpoint breakdown + if record.endpoint: + if record.endpoint not in breakdown.endpoints: + breakdown.endpoints[record.endpoint] = MetricWithMetadata( + metrics=SpendMetrics(), + metadata={}, + ) + breakdown.endpoints[record.endpoint].metrics = update_metrics( + breakdown.endpoints[record.endpoint].metrics, record + ) + + # Update API key breakdown for this endpoint + if record.api_key not in breakdown.endpoints[record.endpoint].api_key_breakdown: + breakdown.endpoints[record.endpoint].api_key_breakdown[record.api_key] = ( + KeyMetricWithMetadata( + metrics=SpendMetrics(), + metadata=KeyMetadata( + key_alias=api_key_metadata.get(record.api_key, {}).get( + "key_alias", None + ), + team_id=api_key_metadata.get(record.api_key, {}).get( + "team_id", None + ), + ), + ) + ) + breakdown.endpoints[record.endpoint].api_key_breakdown[record.api_key].metrics = ( + update_metrics( + breakdown.endpoints[record.endpoint] + .api_key_breakdown[record.api_key] + .metrics, + record, + ) + ) + # Update api key breakdown if record.api_key not in breakdown.api_keys: breakdown.api_keys[record.api_key] = KeyMetricWithMetadata( diff --git a/litellm/proxy/management_endpoints/key_management_endpoints.py b/litellm/proxy/management_endpoints/key_management_endpoints.py index 8d45493bd95..39b6774a61c 100644 --- a/litellm/proxy/management_endpoints/key_management_endpoints.py +++ b/litellm/proxy/management_endpoints/key_management_endpoints.py @@ -14,9 +14,10 @@ import copy import json import secrets import traceback +import yaml from datetime import datetime, timedelta, timezone from typing import List, Literal, Optional, Tuple, cast - +from litellm.litellm_core_utils.safe_json_dumps import safe_dumps import fastapi from fastapi import APIRouter, Depends, Header, HTTPException, Query, Request, status @@ -1033,7 +1034,7 @@ async def generate_key_fn( - auto_rotate: Optional[bool] - Whether this key should be automatically rotated (regenerated) - rotation_interval: Optional[str] - How often to auto-rotate this key (e.g., '30s', '30m', '30h', '30d'). Required if auto_rotate=True. - allowed_vector_store_indexes: Optional[List[dict]] - List of allowed vector store indexes for the key. Example - [{"index_name": "my-index", "index_permissions": ["write", "read"]}]. If specified, the key will only be able to use these specific vector store indexes. Create index, using `/v1/indexes` endpoint. - + - router_settings: Optional[UpdateRouterConfig] - key-specific router settings. Example - {"model_group_retry_policy": {"max_retries": 5}}. IF null or {} then no router settings. Examples: @@ -1388,6 +1389,10 @@ async def prepare_key_update_data( if "model_max_budget" in non_default_values: validate_model_max_budget(non_default_values["model_max_budget"]) + # Serialize router_settings to JSON if present + if "router_settings" in non_default_values and non_default_values["router_settings"] is not None: + non_default_values["router_settings"] = safe_dumps(non_default_values["router_settings"]) + non_default_values = prepare_metadata_fields( data=data, non_default_values=non_default_values, existing_metadata=_metadata ) @@ -1489,7 +1494,8 @@ async def update_key_fn( - auto_rotate: Optional[bool] - Whether this key should be automatically rotated - rotation_interval: Optional[str] - How often to rotate this key (e.g., '30d', '90d'). Required if auto_rotate=True - allowed_vector_store_indexes: Optional[List[dict]] - List of allowed vector store indexes for the key. Example - [{"index_name": "my-index", "index_permissions": ["write", "read"]}]. If specified, the key will only be able to use these specific vector store indexes. Create index, using `/v1/indexes` endpoint. - + - router_settings: Optional[UpdateRouterConfig] - key-specific router settings. Example - {"model_group_retry_policy": {"max_retries": 5}}. IF null or {} then no router settings. + Example: ```bash curl --location 'http://0.0.0.0:4000/key/update' \ @@ -2080,6 +2086,7 @@ async def generate_key_helper_fn( # noqa: PLR0915 object_permission: Optional[LiteLLM_ObjectPermissionBase] = None, auto_rotate: Optional[bool] = None, rotation_interval: Optional[str] = None, + router_settings: Optional[dict] = None, ): from litellm.proxy.proxy_server import premium_user, prisma_client @@ -2114,6 +2121,7 @@ async def generate_key_helper_fn( # noqa: PLR0915 aliases_json = json.dumps(aliases) config_json = json.dumps(config) permissions_json = json.dumps(permissions) + router_settings_json = safe_dumps(router_settings) if router_settings is not None else safe_dumps({}) # Add model_rpm_limit and model_tpm_limit to metadata if model_rpm_limit is not None: @@ -2189,6 +2197,7 @@ async def generate_key_helper_fn( # noqa: PLR0915 "updated_by": updated_by, "allowed_routes": allowed_routes or [], "object_permission_id": object_permission_id, + "router_settings": router_settings_json, } # Add rotation fields if auto_rotate is enabled @@ -2225,6 +2234,13 @@ async def generate_key_helper_fn( # noqa: PLR0915 saved_token["model_max_budget"] = json.loads( saved_token["model_max_budget"] ) + router_settings = cast(Optional[dict], saved_token.get("router_settings")) + if router_settings is not None and isinstance(router_settings, str): + try: + saved_token["router_settings"] = yaml.safe_load(router_settings) + except yaml.YAMLError: + # If it's not valid JSON/YAML, keep as is or set to empty dict + saved_token["router_settings"] = {} if saved_token.get("expires", None) is not None and isinstance( saved_token["expires"], datetime @@ -2269,6 +2285,15 @@ async def generate_key_helper_fn( # noqa: PLR0915 ) key_data["created_at"] = getattr(create_key_response, "created_at", None) key_data["updated_at"] = getattr(create_key_response, "updated_at", None) + + # Deserialize router_settings from JSON string to dict for response + router_settings_value = key_data.get("router_settings") + if router_settings_value is not None and isinstance(router_settings_value, str): + try: + key_data["router_settings"] = yaml.safe_load(router_settings_value) + except yaml.YAMLError: + # If it's not valid JSON/YAML, keep as is or set to empty dict + key_data["router_settings"] = {} except Exception as e: verbose_proxy_logger.error( "litellm.proxy.proxy_server.generate_key_helper_fn(): Exception occured - {}".format( diff --git a/litellm/proxy/management_endpoints/team_endpoints.py b/litellm/proxy/management_endpoints/team_endpoints.py index 920105edc16..78caa86db7b 100644 --- a/litellm/proxy/management_endpoints/team_endpoints.py +++ b/litellm/proxy/management_endpoints/team_endpoints.py @@ -100,7 +100,7 @@ from litellm.types.proxy.management_endpoints.team_endpoints import ( TeamMemberAddResult, UpdateTeamMemberPermissionsRequest, ) - +from litellm.litellm_core_utils.safe_json_dumps import safe_dumps router = APIRouter() @@ -696,8 +696,7 @@ async def new_team( # noqa: PLR0915 - allowed_passthrough_routes: Optional[List[str]] - List of allowed pass through routes for the team. - allowed_vector_store_indexes: Optional[List[dict]] - List of allowed vector store indexes for the key. Example - [{"index_name": "my-index", "index_permissions": ["write", "read"]}]. If specified, the key will only be able to use these specific vector store indexes. Create index, using `/v1/indexes` endpoint. - secret_manager_settings: Optional[dict] - Secret manager settings for the team. [Docs](https://docs.litellm.ai/docs/secret_managers/overview) - - + - router_settings: Optional[UpdateRouterConfig] - team-specific router settings. Example - {"model_group_retry_policy": {"max_retries": 5}}. IF null or {} then no router settings. Returns: - team_id: (str) Unique team id - used for tracking spend across multiple keys for same team id. @@ -911,6 +910,12 @@ async def new_team( # noqa: PLR0915 complete_team_data.members_with_roles = [] complete_team_data_dict = complete_team_data.model_dump(exclude_none=True) + + # Serialize router_settings to JSON (matching key creation pattern) + router_settings_value = getattr(data, "router_settings", None) + router_settings_json = safe_dumps(router_settings_value) if router_settings_value is not None else safe_dumps({}) + complete_team_data_dict["router_settings"] = router_settings_json + complete_team_data_dict = prisma_client.jsonify_team_object( db_data=complete_team_data_dict ) @@ -1234,7 +1239,7 @@ async def update_team( # noqa: PLR0915 Example - update team TPM Limit - allowed_vector_store_indexes: Optional[List[dict]] - List of allowed vector store indexes for the key. Example - [{"index_name": "my-index", "index_permissions": ["write", "read"]}]. If specified, the key will only be able to use these specific vector store indexes. Create index, using `/v1/indexes` endpoint. - secret_manager_settings: Optional[dict] - Secret manager settings for the team. [Docs](https://docs.litellm.ai/docs/secret_managers/overview) - + - router_settings: Optional[UpdateRouterConfig] - team-specific router settings. Example - {"model_group_retry_policy": {"max_retries": 5}}. IF null or {} then no router settings. ``` curl --location 'http://0.0.0.0:4000/team/update' \ @@ -1396,6 +1401,10 @@ async def update_team( # noqa: PLR0915 if _model_id is not None: updated_kv["model_id"] = _model_id + # Serialize router_settings to JSON if present (matching key update pattern) + if "router_settings" in updated_kv and updated_kv["router_settings"] is not None: + updated_kv["router_settings"] = safe_dumps(updated_kv["router_settings"]) + updated_kv = prisma_client.jsonify_team_object(db_data=updated_kv) team_row: Optional[LiteLLM_TeamTable] = ( await prisma_client.db.litellm_teamtable.update( diff --git a/litellm/proxy/proxy_config.yaml b/litellm/proxy/proxy_config.yaml index 2191968e86c..8a8fd6794e7 100644 --- a/litellm/proxy/proxy_config.yaml +++ b/litellm/proxy/proxy_config.yaml @@ -2,4 +2,7 @@ model_list: - model_name: anthropic/* litellm_params: model: anthropic/* + - model_name: openai/* + litellm_params: + model: openai/* diff --git a/litellm/proxy/route_llm_request.py b/litellm/proxy/route_llm_request.py index a321e25a9a5..5d2e13a78b9 100644 --- a/litellm/proxy/route_llm_request.py +++ b/litellm/proxy/route_llm_request.py @@ -38,6 +38,7 @@ ROUTE_ENDPOINT_MAPPING = { "aretrieve_container": "/containers/{container_id}", "adelete_container": "/containers/{container_id}", # Auto-generated container file routes + "aupload_container_file": "/containers/{container_id}/files", "alist_container_files": "/containers/{container_id}/files", "aretrieve_container_file": "/containers/{container_id}/files/{file_id}", "adelete_container_file": "/containers/{container_id}/files/{file_id}", @@ -144,6 +145,7 @@ async def route_request( "alist_containers", "aretrieve_container", "adelete_container", + "aupload_container_file", "alist_container_files", "aretrieve_container_file", "adelete_container_file", @@ -204,6 +206,7 @@ async def route_request( "alist_containers", "aretrieve_container", "adelete_container", + "aupload_container_file", "alist_container_files", "aretrieve_container_file", "adelete_container_file", @@ -287,6 +290,7 @@ async def route_request( "alist_containers", "aretrieve_container", "adelete_container", + "aupload_container_file", "alist_container_files", "aretrieve_container_file", "adelete_container_file", diff --git a/litellm/proxy/schema.prisma b/litellm/proxy/schema.prisma index e565135bbc4..56fe093a8bc 100644 --- a/litellm/proxy/schema.prisma +++ b/litellm/proxy/schema.prisma @@ -124,6 +124,7 @@ model LiteLLM_TeamTable { updated_at DateTime @default(now()) @updatedAt @map("updated_at") model_spend Json @default("{}") model_max_budget Json @default("{}") + router_settings Json? @default("{}") team_member_permissions String[] @default([]) model_id Int? @unique // id for LiteLLM_ModelTable -> stores team-level model aliases litellm_organization_table LiteLLM_OrganizationTable? @relation(fields: [organization_id], references: [organization_id]) @@ -225,6 +226,7 @@ model LiteLLM_VerificationToken { models String[] aliases Json @default("{}") config Json @default("{}") + router_settings Json? @default("{}") user_id String? team_id String? permissions Json @default("{}") @@ -422,6 +424,7 @@ model LiteLLM_DailyUserSpend { model_group String? custom_llm_provider String? mcp_namespaced_tool_name String? + endpoint String? prompt_tokens BigInt @default(0) completion_tokens BigInt @default(0) cache_read_input_tokens BigInt @default(0) @@ -433,12 +436,13 @@ model LiteLLM_DailyUserSpend { created_at DateTime @default(now()) updated_at DateTime @updatedAt - @@unique([user_id, date, api_key, model, custom_llm_provider, mcp_namespaced_tool_name]) + @@unique([user_id, date, api_key, model, custom_llm_provider, mcp_namespaced_tool_name, endpoint]) @@index([date]) @@index([user_id]) @@index([api_key]) @@index([model]) @@index([mcp_namespaced_tool_name]) + @@index([endpoint]) } // Track daily organization spend metrics per model and key @@ -451,6 +455,7 @@ model LiteLLM_DailyOrganizationSpend { model_group String? custom_llm_provider String? mcp_namespaced_tool_name String? + endpoint String? prompt_tokens BigInt @default(0) completion_tokens BigInt @default(0) cache_read_input_tokens BigInt @default(0) @@ -462,12 +467,13 @@ model LiteLLM_DailyOrganizationSpend { created_at DateTime @default(now()) updated_at DateTime @updatedAt - @@unique([organization_id, date, api_key, model, custom_llm_provider, mcp_namespaced_tool_name]) + @@unique([organization_id, date, api_key, model, custom_llm_provider, mcp_namespaced_tool_name, endpoint]) @@index([date]) @@index([organization_id]) @@index([api_key]) @@index([model]) @@index([mcp_namespaced_tool_name]) + @@index([endpoint]) } // Track daily end user (customer) spend metrics per model and key @@ -480,6 +486,7 @@ model LiteLLM_DailyEndUserSpend { model_group String? custom_llm_provider String? mcp_namespaced_tool_name String? + endpoint String? prompt_tokens BigInt @default(0) completion_tokens BigInt @default(0) cache_read_input_tokens BigInt @default(0) @@ -490,12 +497,13 @@ model LiteLLM_DailyEndUserSpend { failed_requests BigInt @default(0) created_at DateTime @default(now()) updated_at DateTime @updatedAt - @@unique([end_user_id, date, api_key, model, custom_llm_provider, mcp_namespaced_tool_name]) + @@unique([end_user_id, date, api_key, model, custom_llm_provider, mcp_namespaced_tool_name, endpoint]) @@index([date]) @@index([end_user_id]) @@index([api_key]) @@index([model]) @@index([mcp_namespaced_tool_name]) + @@index([endpoint]) } // Track daily agent spend metrics per model and key @@ -508,6 +516,7 @@ model LiteLLM_DailyAgentSpend { model_group String? custom_llm_provider String? mcp_namespaced_tool_name String? + endpoint String? prompt_tokens BigInt @default(0) completion_tokens BigInt @default(0) cache_read_input_tokens BigInt @default(0) @@ -518,12 +527,13 @@ model LiteLLM_DailyAgentSpend { failed_requests BigInt @default(0) created_at DateTime @default(now()) updated_at DateTime @updatedAt - @@unique([agent_id, date, api_key, model, custom_llm_provider, mcp_namespaced_tool_name]) + @@unique([agent_id, date, api_key, model, custom_llm_provider, mcp_namespaced_tool_name, endpoint]) @@index([date]) @@index([agent_id]) @@index([api_key]) @@index([model]) @@index([mcp_namespaced_tool_name]) + @@index([endpoint]) } // Track daily team spend metrics per model and key @@ -536,6 +546,7 @@ model LiteLLM_DailyTeamSpend { model_group String? custom_llm_provider String? mcp_namespaced_tool_name String? + endpoint String? prompt_tokens BigInt @default(0) completion_tokens BigInt @default(0) cache_read_input_tokens BigInt @default(0) @@ -547,12 +558,13 @@ model LiteLLM_DailyTeamSpend { created_at DateTime @default(now()) updated_at DateTime @updatedAt - @@unique([team_id, date, api_key, model, custom_llm_provider, mcp_namespaced_tool_name]) + @@unique([team_id, date, api_key, model, custom_llm_provider, mcp_namespaced_tool_name, endpoint]) @@index([date]) @@index([team_id]) @@index([api_key]) @@index([model]) @@index([mcp_namespaced_tool_name]) + @@index([endpoint]) } // Track daily team spend metrics per model and key @@ -566,6 +578,7 @@ model LiteLLM_DailyTagSpend { model_group String? custom_llm_provider String? mcp_namespaced_tool_name String? + endpoint String? prompt_tokens BigInt @default(0) completion_tokens BigInt @default(0) cache_read_input_tokens BigInt @default(0) @@ -577,12 +590,13 @@ model LiteLLM_DailyTagSpend { created_at DateTime @default(now()) updated_at DateTime @updatedAt - @@unique([tag, date, api_key, model, custom_llm_provider, mcp_namespaced_tool_name]) + @@unique([tag, date, api_key, model, custom_llm_provider, mcp_namespaced_tool_name, endpoint]) @@index([date]) @@index([tag]) @@index([api_key]) @@index([model]) @@index([mcp_namespaced_tool_name]) + @@index([endpoint]) } diff --git a/litellm/router.py b/litellm/router.py index fa58340b9b1..98ccf41c96d 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -4003,6 +4003,8 @@ class Router: "retrieve_container", "adelete_container", "delete_container", + "aupload_container_file", + "upload_container_file", "alist_container_files", "list_container_files", "aretrieve_container_file", @@ -4154,6 +4156,7 @@ class Router: "alist_containers", "aretrieve_container", "adelete_container", + "aupload_container_file", "alist_container_files", "aretrieve_container_file", "adelete_container_file", diff --git a/litellm/types/integrations/arize.py b/litellm/types/integrations/arize.py index be4df30e794..248fdac3b3a 100644 --- a/litellm/types/integrations/arize.py +++ b/litellm/types/integrations/arize.py @@ -14,3 +14,4 @@ class ArizeConfig(BaseModel): api_key: Optional[str] = None protocol: Protocol endpoint: str + project_name: Optional[str] = None diff --git a/litellm/types/proxy/management_endpoints/common_daily_activity.py b/litellm/types/proxy/management_endpoints/common_daily_activity.py index 08ffc5d097a..948401fb8bf 100644 --- a/litellm/types/proxy/management_endpoints/common_daily_activity.py +++ b/litellm/types/proxy/management_endpoints/common_daily_activity.py @@ -68,6 +68,9 @@ class BreakdownMetrics(BaseModel): providers: Dict[str, MetricWithMetadata] = Field( default_factory=dict ) # provider -> {metrics, metadata} + endpoints: Dict[str, MetricWithMetadata] = Field( + default_factory=dict + ) # endpoint -> {metrics, metadata} api_keys: Dict[str, KeyMetricWithMetadata] = Field( default_factory=dict ) # api_key -> {metrics, metadata} diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 784c8403c3f..3817f46c3e2 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -324,6 +324,8 @@ class CallTypes(str, Enum): adelete_container = "adelete_container" list_container_files = "list_container_files" alist_container_files = "alist_container_files" + upload_container_file = "upload_container_file" + aupload_container_file = "aupload_container_file" acancel_fine_tuning_job = "acancel_fine_tuning_job" cancel_fine_tuning_job = "cancel_fine_tuning_job" diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 90b73e4709c..73579db75cd 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -4909,6 +4909,15 @@ "/v1/images/generations" ] }, + "azure_ai/flux.2-pro": { + "litellm_provider": "azure_ai", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://ai.azure.com/explore/models/flux.2-pro/version/1/registry/azureml-blackforestlabs", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, "azure_ai/Llama-3.2-11B-Vision-Instruct": { "input_cost_per_token": 3.7e-07, "litellm_provider": "azure_ai", @@ -28336,6 +28345,19 @@ "supports_tool_choice": true, "supports_web_search": true }, + "vertex_ai/zai-org/glm-4.7-maas": { + "input_cost_per_token": 3e-07, + "litellm_provider": "vertex_ai-zai_models", + "max_input_tokens": 200000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 1.2e-06, + "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing#partner-models", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_tool_choice": true + }, "vertex_ai/mistral-medium-3": { "input_cost_per_token": 4e-07, "litellm_provider": "vertex_ai-mistral_models", diff --git a/poetry.lock b/poetry.lock index a0a0f8540e5..0a4ef10d09f 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 2.2.1 and should not be changed by hand. +# This file is automatically @generated by Poetry 2.2.0 and should not be changed by hand. [[package]] name = "aiofiles" @@ -3081,15 +3081,15 @@ files = [ [[package]] name = "litellm-proxy-extras" -version = "0.4.16" +version = "0.4.18" description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package." optional = true python-versions = "!=2.7.*,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,!=3.7.*,>=3.8" groups = ["main"] markers = "extra == \"proxy\"" files = [ - {file = "litellm_proxy_extras-0.4.16-py3-none-any.whl", hash = "sha256:5651e777c7f4c0e87c6722971bca19b8f40f417b08f74001cab2d0a5b1c63a91"}, - {file = "litellm_proxy_extras-0.4.16.tar.gz", hash = "sha256:ff1ee4ea119318b471bb71a99d8bc941159d4d2c09bee797dd29768e9504befb"}, + {file = "litellm_proxy_extras-0.4.18-py3-none-any.whl", hash = "sha256:c3edee68bf8eb073c6158dcf7df05727dfc829e63c03a617fcb48853d11490df"}, + {file = "litellm_proxy_extras-0.4.18.tar.gz", hash = "sha256:898b28e3e74acdc29142906b84787ab05a90e30aa3c0c8aee849915e3a16adb3"}, ] [[package]] @@ -7981,4 +7981,4 @@ utils = ["numpydoc"] [metadata] lock-version = "2.1" python-versions = ">=3.9,<4.0" -content-hash = "7eed2b2c25173a275ac83c55fd901b9b84663b1d7daa54f0e78b30bf1c8f0e3e" +content-hash = "e9fd12b5ccc703ec156d98877452417083e3ac18b5970cb3a58c3bde09d267bb" diff --git a/provider_endpoints_support.json b/provider_endpoints_support.json index bc5dea7b97c..f671409175a 100644 --- a/provider_endpoints_support.json +++ b/provider_endpoints_support.json @@ -20,16 +20,14 @@ "skills": "Supports /skills endpoint", "interactions": "Supports /interactions endpoint (Google AI Interactions API)", "a2a_(Agent Gateway)": "Supports /a2a/{agent}/message/send endpoint (A2A Protocol)", - "create_container": "Supports POST /containers endpoint", - "list_containers": "Supports GET /containers endpoint", - "retrieve_container": "Supports GET /containers/{id} endpoint", - "delete_container": "Supports DELETE /containers/{id} endpoint", - "create_container_file": "Supports POST /containers/{id}/files endpoint", - "list_container_files": "Supports GET /containers/{id}/files endpoint", - "retrieve_container_file": "Supports GET /containers/{id}/files/{file_id} endpoint", - "retrieve_container_file_content": "Supports GET /containers/{id}/files/{file_id}/content endpoint", - "delete_container_file": "Supports DELETE /containers/{id}/files/{file_id} endpoint", - "compact": "Supports /responses/compact endpoint" + "container": "Supports OpenAI's /containers endpoint", + "container_file": "Supports OpenAI's /containers/{id}/files endpoint", + "compact": "Supports /responses/compact endpoint", + "files": "Supports /files endpoint for file operations", + "image_edits": "Supports /images/edits endpoint for image editing", + "vector_stores_create": "Supports creating a new vector store via /vector_stores endpoint", + "vector_stores_search": "Supports searching a vector store via /vector_stores/{id}/search endpoint", + "video_generations": "Supports /videos/generations endpoint for video generation" } } }, @@ -122,7 +120,8 @@ "rerank": false, "skills": true, "a2a": true, - "interactions": true + "interactions": true, + "count_tokens": true } }, "anthropic_text": { @@ -211,7 +210,13 @@ "batches": false, "rerank": true, "a2a": true, - "interactions": true + "interactions": true, + "bedrock_invoke": true, + "bedrock_converse": true, + "vector_stores_search": true, + "count_tokens": true, + "rag_ingest": true, + "rag_query": true } }, "sagemaker": { @@ -263,7 +268,11 @@ "batches": true, "rerank": false, "a2a": true, - "interactions": true + "interactions": true, + "vector_stores_search": true, + "assistants": true, + "fine_tuning": true, + "text_completion": true } }, "azure_ai": { @@ -275,6 +284,7 @@ "responses": true, "embeddings": true, "image_generations": true, + "image_edits": true, "audio_transcriptions": true, "audio_speech": true, "moderations": true, @@ -282,7 +292,9 @@ "rerank": false, "ocr": true, "a2a": true, - "interactions": true + "interactions": true, + "vector_stores_create": true, + "vector_stores_search": true } }, "azure_ai/doc-intelligence": { @@ -918,29 +930,19 @@ "embeddings": true, "image_generations": true, "audio_transcriptions": false, - "audio_speech": false, + "audio_speech": true, "moderations": false, "batches": false, "rerank": false, "ocr": true, "a2a": true, - "interactions": true - } - }, - "vertex_ai/chirp": { - "display_name": "Google - Vertex AI Chirp3 HD (`vertex_ai/chirp`)", - "url": "https://docs.litellm.ai/docs/providers/vertex_speech", - "endpoints": { - "chat_completions": false, - "messages": false, - "responses": false, - "embeddings": false, - "image_generations": false, - "audio_transcriptions": false, - "audio_speech": true, - "moderations": false, - "batches": false, - "rerank": false + "interactions": true, + "vector_stores_search": true, + "count_tokens": true, + "fine_tuning": true, + "rag_ingest": true, + "rag_query": true, + "generateContent": true } }, "gemini": { @@ -958,7 +960,12 @@ "batches": false, "rerank": false, "interactions": true, - "a2a": true + "a2a": true, + "vector_stores_search": true, + "count_tokens": true, + "rag_ingest": true, + "realtime": true, + "generateContent": true } }, "gradient_ai": { @@ -1511,18 +1518,21 @@ "moderations": true, "batches": true, "rerank": false, - "create_container": true, - "list_containers": true, - "retrieve_container": true, - "delete_container": true, - "create_container_file": false, - "list_container_files": true, - "retrieve_container_file": true, - "retrieve_container_file_content": true, - "delete_container_file": true, + "container": true, "compact": true, "a2a": true, - "interactions": true + "interactions": true, + "vector_store_files": true, + "vector_stores_create": true, + "vector_stores_search": true, + "assistants": true, + "container_files": true, + "fine_tuning": true, + "image_variations": true, + "rag_ingest": true, + "rag_query": true, + "realtime": true, + "text_completion": true } }, "openai_like": { @@ -1538,7 +1548,8 @@ "audio_speech": false, "moderations": false, "batches": false, - "rerank": false + "rerank": false, + "assistants": true } }, "openrouter": { @@ -1897,34 +1908,13 @@ "display_name": "Topaz (`topaz`)", "url": "https://docs.litellm.ai/docs/providers/topaz", "endpoints": { - "chat_completions": true, - "messages": true, - "responses": true, - "embeddings": false, - "image_generations": false, - "audio_transcriptions": false, - "audio_speech": false, - "moderations": false, - "batches": false, - "rerank": false, - "a2a": true, - "interactions": true + "image_variations": true } }, "tavily": { "display_name": "Tavily (`tavily`)", "url": "https://docs.litellm.ai/docs/search/tavily", "endpoints": { - "chat_completions": false, - "messages": false, - "responses": false, - "embeddings": false, - "image_generations": false, - "audio_transcriptions": false, - "audio_speech": false, - "moderations": false, - "batches": false, - "rerank": false, "search": true } }, @@ -2137,7 +2127,7 @@ "moderations": false, "batches": false, "rerank": false, - "vector_stores": true, + "vector_stores_create": true, "a2a": true, "interactions": true } @@ -2247,6 +2237,340 @@ "a2a": true, "interactions": true } + }, + "gigachat": { + "display_name": "GigaChat (`gigachat`)", + "url": "https://docs.litellm.ai/docs/providers/gigachat", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": true + } + }, + "google_pse": { + "display_name": "Google PSE (`google_pse`)", + "url": "https://docs.litellm.ai/docs/search/google_pse", + "endpoints": { + "search": true + } + }, + "milvus": { + "display_name": "Milvus (`milvus`)", + "url": "https://docs.litellm.ai/docs/providers/milvus_vector_stores", + "endpoints": { + "vector_stores_search": true + } + }, + "minimax": { + "display_name": "Minimax (`minimax`)", + "url": "https://docs.litellm.ai/docs/providers/minimax", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true + } + }, + "pg_vector": { + "display_name": "PG Vector (`pg_vector`)", + "url": "https://docs.litellm.ai/docs/providers/pg_vector", + "endpoints": { + "vector_stores_search": true + } + }, + "helicone": { + "display_name": "Helicone (`helicone`)", + "url": "https://docs.litellm.ai/docs/providers/helicone", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true + } + }, + "llamagate": { + "display_name": "LlamaGate (`llamagate`)", + "url": "https://docs.litellm.ai/docs/providers/llamagate", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true + } + }, + "xiaomi_mimo": { + "display_name": "Xiaomi Mimo (`xiaomi_mimo`)", + "url": "https://docs.litellm.ai/docs/providers/xiaomi_mimo", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true + } + } + }, + "endpoints": { + "a2a": { + "docs_label": "a2a", + "display_name": "A2A (Agent-to-Agent) protocol for agent communication", + "leftnav_label": "/a2a", + "provider_json_field": "a2a", + "url": "https://docs.litellm.ai/docs/a2a", + "bridges_to_chat_completion": true + }, + "messages": { + "docs_label": "anthropic_unified", + "display_name": "Anthropic /v1/messages API", + "leftnav_label": "/messages", + "provider_json_field": "messages", + "url": "https://docs.litellm.ai/docs/anthropic_unified", + "bridges_to_chat_completion": true + }, + "anthropic_count_tokens": { + "docs_label": "anthropic_count_tokens", + "display_name": "Anthropic /v1/messages/count_tokens API", + "leftnav_label": "/count_tokens", + "provider_json_field": "count_tokens", + "url": "https://docs.litellm.ai/docs/anthropic_count_tokens" + }, + "apply_guardrail": { + "docs_label": "apply_guardrail", + "display_name": "Unified Apply Guardrail API", + "leftnav_label": "/guardrails/apply_guardrail", + "provider_json_field": "apply_guardrail", + "url": "https://docs.litellm.ai/docs/apply_guardrail" + }, + "assistants": { + "docs_label": "assistants", + "display_name": "OpenAI Assistants API", + "leftnav_label": "/assistants", + "provider_json_field": "assistants", + "url": "https://docs.litellm.ai/docs/assistants" + }, + "audio_transcription": { + "docs_label": "audio_transcription", + "display_name": "Audio Transcription API", + "leftnav_label": "/audio/transcriptions", + "provider_json_field": "audio_transcriptions", + "url": "https://docs.litellm.ai/docs/audio_transcription" + }, + "batches": { + "docs_label": "batches", + "display_name": "Batches API", + "leftnav_label": "/batches", + "provider_json_field": "batches", + "url": "https://docs.litellm.ai/docs/batches" + }, + "bedrock_invoke": { + "docs_label": "bedrock_invoke", + "display_name": "Bedrock Invoke API", + "leftnav_label": "/invoke", + "provider_json_field": "bedrock_invoke", + "url": "https://docs.litellm.ai/docs/bedrock_invoke" + }, + "bedrock_converse": { + "docs_label": "bedrock_converse", + "display_name": "Bedrock Converse API", + "leftnav_label": "/converse", + "provider_json_field": "bedrock_converse", + "url": "https://docs.litellm.ai/docs/bedrock_converse" + }, + "chat_completions": { + "docs_label": "chat_completions", + "display_name": "Chat Completions API", + "leftnav_label": "/chat/completions", + "provider_json_field": "chat_completions", + "url": "https://docs.litellm.ai/docs/chat_completions" + }, + "container_files": { + "docs_label": "container_files", + "display_name": "OpenAI Container Files API", + "leftnav_label": "/create/container/files", + "provider_json_field": "container_files", + "url": "https://docs.litellm.ai/docs/container_files" + }, + "container": { + "docs_label": "containers", + "display_name": "OpenAI Containers API", + "leftnav_label": "/container", + "provider_json_field": "container", + "url": "https://docs.litellm.ai/docs/containers" + }, + "embeddings": { + "docs_label": "embedding/supported_embedding", + "display_name": "Embedding API (OpenAI Format)", + "leftnav_label": "/embeddings", + "provider_json_field": "embeddings", + "url": "https://docs.litellm.ai/docs/embedding/supported_embedding" + }, + "files": { + "docs_label": "files", + "display_name": "OpenAI Files API", + "leftnav_label": "/files", + "provider_json_field": "files", + "url": "https://docs.litellm.ai/docs/proxy/litellm_managed_files" + }, + "fine_tuning": { + "docs_label": "fine_tuning", + "display_name": "OpenAI Fine-Tuning API", + "leftnav_label": "/fine_tuning", + "provider_json_field": "fine_tuning", + "url": "https://docs.litellm.ai/docs/proxy/managed_finetuning" + }, + "generateContent": { + "docs_label": "generateContent", + "display_name": "Google's GenerateContent API", + "leftnav_label": "/generateContent", + "provider_json_field": "generateContent", + "url": "https://docs.litellm.ai/docs/generateContent", + "bridges_to_chat_completion": true + }, + "image_edits": { + "docs_label": "image_edits", + "display_name": "OpenAI Images Edits API", + "leftnav_label": "/images/edits", + "provider_json_field": "image_edits", + "url": "https://docs.litellm.ai/docs/image_edits" + }, + "image_generations": { + "docs_label": "image_generation", + "display_name": "OpenAI Images Generations API", + "leftnav_label": "/images/generations", + "provider_json_field": "image_generations", + "url": "https://docs.litellm.ai/docs/image_generation" + }, + "image_variations": { + "docs_label": "image_variations", + "display_name": "OpenAI Images Variations API", + "leftnav_label": "/images/variations", + "provider_json_field": "image_variations", + "url": "https://docs.litellm.ai/docs/image_variations" + }, + "interactions": { + "docs_label": "interactions", + "display_name": "Google Interactions API", + "leftnav_label": "/interactions", + "provider_json_field": "interactions", + "url": "https://docs.litellm.ai/docs/interactions", + "bridges_to_chat_completion": true + }, + "mcp": { + "docs_label": "mcp", + "display_name": "Model Context Protocol (MCP)", + "leftnav_label": "/mcp", + "provider_json_field": "mcp", + "url": "https://docs.litellm.ai/docs/mcp" + }, + "moderation": { + "docs_label": "moderation", + "display_name": "OpenAI Moderation API", + "leftnav_label": "/moderations", + "provider_json_field": "moderations", + "url": "https://docs.litellm.ai/docs/moderation" + }, + "ocr": { + "docs_label": "ocr", + "display_name": "OCR API (Mistral Format)", + "leftnav_label": "/ocr", + "provider_json_field": "ocr", + "url": "https://docs.litellm.ai/docs/ocr" + }, + "rag_ingest": { + "docs_label": "rag_ingest", + "display_name": "RAG Ingest API", + "leftnav_label": "/rag/ingest", + "provider_json_field": "rag_ingest", + "url": "https://docs.litellm.ai/docs/rag_ingest" + }, + "rag_query": { + "docs_label": "rag_query", + "display_name": "RAG Query API", + "leftnav_label": "/rag/query", + "provider_json_field": "rag_query", + "url": "https://docs.litellm.ai/docs/rag_query" + }, + "realtime": { + "docs_label": "realtime", + "display_name": "OpenAI Realtime API", + "leftnav_label": "/realtime", + "provider_json_field": "realtime", + "url": "https://docs.litellm.ai/docs/realtime" + }, + "rerank": { + "docs_label": "rerank", + "display_name": "Rerank API (Cohere Format)", + "leftnav_label": "/rerank", + "provider_json_field": "rerank", + "url": "https://docs.litellm.ai/docs/rerank" + }, + "responses": { + "docs_label": "response_api", + "display_name": "Responses API (OpenAI Format)", + "leftnav_label": "/responses", + "provider_json_field": "responses", + "url": "https://docs.litellm.ai/docs/response_api", + "bridges_to_chat_completion": true + }, + "response_api_compact": { + "docs_label": "response_api_compact", + "display_name": "Responses API (OpenAI Format)", + "leftnav_label": "/responses", + "provider_json_field": "compact", + "url": "https://docs.litellm.ai/docs/response_api" + }, + "search": { + "docs_label": "search", + "display_name": "Search API", + "leftnav_label": "/search", + "provider_json_field": "search", + "url": "https://docs.litellm.ai/docs/search" + }, + "skills": { + "docs_label": "skills", + "display_name": "Anthropic Skills API", + "leftnav_label": "/skills", + "provider_json_field": "skills", + "url": "https://docs.litellm.ai/docs/skills" + }, + "text_completion": { + "docs_label": "text_completion", + "display_name": "Completions API (OpenAI Format)", + "leftnav_label": "/completions", + "provider_json_field": "text_completion", + "url": "https://docs.litellm.ai/docs/text_completion", + "bridges_to_chat_completion": true + }, + "text_to_speech": { + "docs_label": "text_to_speech", + "display_name": "Text-to-Speech API (OpenAI Format)", + "leftnav_label": "/audio/speech", + "provider_json_field": "audio_speech", + "url": "https://docs.litellm.ai/docs/text_to_speech" + }, + "vector_store_files": { + "docs_label": "vector_store_files", + "display_name": "OpenAI Vector Store Files API", + "leftnav_label": "/vector_stores/files", + "provider_json_field": "vector_store_files", + "url": "https://docs.litellm.ai/docs/vector_store_files" + }, + "vector_stores_create": { + "docs_label": "vector_stores_create", + "display_name": "OpenAI Vector Stores Create API", + "leftnav_label": "/vector_stores/create", + "provider_json_field": "vector_stores_create", + "url": "https://docs.litellm.ai/docs/vector_stores/create" + }, + "vector_stores_search": { + "docs_label": "vector_stores_search", + "display_name": "OpenAI Vector Stores Search API", + "leftnav_label": "/vector_stores/search", + "provider_json_field": "vector_stores_search", + "url": "https://docs.litellm.ai/docs/vector_stores/search" + }, + "videos": { + "docs_label": "videos", + "display_name": "OpenAI Video Generation API", + "leftnav_label": "/videos", + "provider_json_field": "video_generations", + "url": "https://docs.litellm.ai/docs/videos" } } } diff --git a/pyproject.toml b/pyproject.toml index 3b09119a748..51ef8650d0c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "litellm" -version = "1.80.11" +version = "1.80.12" description = "Library to easily interface with LLM API providers" authors = ["BerriAI"] license = "MIT" @@ -59,7 +59,7 @@ websockets = {version = "^15.0.1", optional = true} boto3 = {version = "1.36.0", optional = true} redisvl = {version = "^0.4.1", optional = true, markers = "python_version >= '3.9' and python_version < '3.14'"} mcp = {version = "^1.21.2", optional = true, python = ">=3.10"} -litellm-proxy-extras = {version = "0.4.16", optional = true} +litellm-proxy-extras = {version = "0.4.20", optional = true} rich = {version = "13.7.1", optional = true} litellm-enterprise = {version = "0.1.27", optional = true} diskcache = {version = "^5.6.1", optional = true} @@ -167,7 +167,7 @@ requires = ["poetry-core", "wheel"] build-backend = "poetry.core.masonry.api" [tool.commitizen] -version = "1.80.11" +version = "1.80.12" version_files = [ "pyproject.toml:^version" ] diff --git a/requirements.txt b/requirements.txt index 249b899b86b..ceafa23a22f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -15,7 +15,7 @@ redis==5.2.1 # redis caching prisma==0.11.0 # for db nodejs-wheel-binaries==24.12.0 ## required by prisma for migrations, prevents runtime download (updated from nodejs-bin for security fixes) mangum==0.17.0 # for aws lambda functions -pynacl==1.5.0 # for encrypting keys +pynacl==1.6.2 # for encrypting keys google-cloud-aiplatform==1.47.0 # for vertex ai calls google-cloud-iam==2.19.1 # for GCP IAM Redis authentication google-genai==1.22.0 @@ -47,7 +47,7 @@ sentry_sdk==2.21.0 # for sentry error handling detect-secrets==1.5.0 # Enterprise - secret detection / masking in LLM requests cryptography==44.0.1 tzdata==2025.1 # IANA time zone database -litellm-proxy-extras==0.4.16 # for proxy extras - e.g. prisma migrations +litellm-proxy-extras==0.4.20 # for proxy extras - e.g. prisma migrations llm-sandbox==0.3.31 # for skill execution in sandbox ### LITELLM PACKAGE DEPENDENCIES python-dotenv==1.0.1 # for env @@ -57,7 +57,7 @@ tokenizers==0.20.2 # for calculating usage click==8.1.7 # for proxy cli rich==13.7.1 # for litellm proxy cli jinja2==3.1.6 # for prompt templates -aiohttp==3.12.14 # for network calls +aiohttp==3.13.3 # for network calls aioboto3==13.4.0 # for async sagemaker calls tenacity==8.5.0 # for retrying requests, when litellm.num_retries set pydantic>=2.11,<3 # proxy + openai req. + mcp diff --git a/schema.prisma b/schema.prisma index e565135bbc4..a16380fb5f3 100644 --- a/schema.prisma +++ b/schema.prisma @@ -124,6 +124,7 @@ model LiteLLM_TeamTable { updated_at DateTime @default(now()) @updatedAt @map("updated_at") model_spend Json @default("{}") model_max_budget Json @default("{}") + router_settings Json? @default("{}") team_member_permissions String[] @default([]) model_id Int? @unique // id for LiteLLM_ModelTable -> stores team-level model aliases litellm_organization_table LiteLLM_OrganizationTable? @relation(fields: [organization_id], references: [organization_id]) @@ -225,6 +226,7 @@ model LiteLLM_VerificationToken { models String[] aliases Json @default("{}") config Json @default("{}") + router_settings Json? @default("{}") user_id String? team_id String? permissions Json @default("{}") @@ -422,6 +424,7 @@ model LiteLLM_DailyUserSpend { model_group String? custom_llm_provider String? mcp_namespaced_tool_name String? + endpoint String? prompt_tokens BigInt @default(0) completion_tokens BigInt @default(0) cache_read_input_tokens BigInt @default(0) @@ -433,12 +436,13 @@ model LiteLLM_DailyUserSpend { created_at DateTime @default(now()) updated_at DateTime @updatedAt - @@unique([user_id, date, api_key, model, custom_llm_provider, mcp_namespaced_tool_name]) + @@unique([user_id, date, api_key, model, custom_llm_provider, mcp_namespaced_tool_name, endpoint]) @@index([date]) @@index([user_id]) @@index([api_key]) @@index([model]) @@index([mcp_namespaced_tool_name]) + @@index([endpoint]) } // Track daily organization spend metrics per model and key @@ -451,6 +455,7 @@ model LiteLLM_DailyOrganizationSpend { model_group String? custom_llm_provider String? mcp_namespaced_tool_name String? + endpoint String? prompt_tokens BigInt @default(0) completion_tokens BigInt @default(0) cache_read_input_tokens BigInt @default(0) @@ -462,12 +467,13 @@ model LiteLLM_DailyOrganizationSpend { created_at DateTime @default(now()) updated_at DateTime @updatedAt - @@unique([organization_id, date, api_key, model, custom_llm_provider, mcp_namespaced_tool_name]) + @@unique([organization_id, date, api_key, model, custom_llm_provider, mcp_namespaced_tool_name, endpoint]) @@index([date]) @@index([organization_id]) @@index([api_key]) @@index([model]) @@index([mcp_namespaced_tool_name]) + @@index([endpoint]) } // Track daily end user (customer) spend metrics per model and key @@ -480,6 +486,7 @@ model LiteLLM_DailyEndUserSpend { model_group String? custom_llm_provider String? mcp_namespaced_tool_name String? + endpoint String? prompt_tokens BigInt @default(0) completion_tokens BigInt @default(0) cache_read_input_tokens BigInt @default(0) @@ -490,12 +497,13 @@ model LiteLLM_DailyEndUserSpend { failed_requests BigInt @default(0) created_at DateTime @default(now()) updated_at DateTime @updatedAt - @@unique([end_user_id, date, api_key, model, custom_llm_provider, mcp_namespaced_tool_name]) + @@unique([end_user_id, date, api_key, model, custom_llm_provider, mcp_namespaced_tool_name, endpoint]) @@index([date]) @@index([end_user_id]) @@index([api_key]) @@index([model]) @@index([mcp_namespaced_tool_name]) + @@index([endpoint]) } // Track daily agent spend metrics per model and key @@ -508,6 +516,7 @@ model LiteLLM_DailyAgentSpend { model_group String? custom_llm_provider String? mcp_namespaced_tool_name String? + endpoint String? prompt_tokens BigInt @default(0) completion_tokens BigInt @default(0) cache_read_input_tokens BigInt @default(0) @@ -518,12 +527,13 @@ model LiteLLM_DailyAgentSpend { failed_requests BigInt @default(0) created_at DateTime @default(now()) updated_at DateTime @updatedAt - @@unique([agent_id, date, api_key, model, custom_llm_provider, mcp_namespaced_tool_name]) + @@unique([agent_id, date, api_key, model, custom_llm_provider, mcp_namespaced_tool_name, endpoint]) @@index([date]) @@index([agent_id]) @@index([api_key]) @@index([model]) @@index([mcp_namespaced_tool_name]) + @@index([endpoint]) } // Track daily team spend metrics per model and key @@ -536,6 +546,7 @@ model LiteLLM_DailyTeamSpend { model_group String? custom_llm_provider String? mcp_namespaced_tool_name String? + endpoint String? prompt_tokens BigInt @default(0) completion_tokens BigInt @default(0) cache_read_input_tokens BigInt @default(0) @@ -547,12 +558,13 @@ model LiteLLM_DailyTeamSpend { created_at DateTime @default(now()) updated_at DateTime @updatedAt - @@unique([team_id, date, api_key, model, custom_llm_provider, mcp_namespaced_tool_name]) + @@unique([team_id, date, api_key, model, custom_llm_provider, mcp_namespaced_tool_name, endpoint]) @@index([date]) @@index([team_id]) @@index([api_key]) @@index([model]) @@index([mcp_namespaced_tool_name]) + @@index([endpoint]) } // Track daily team spend metrics per model and key @@ -566,6 +578,7 @@ model LiteLLM_DailyTagSpend { model_group String? custom_llm_provider String? mcp_namespaced_tool_name String? + endpoint String? prompt_tokens BigInt @default(0) completion_tokens BigInt @default(0) cache_read_input_tokens BigInt @default(0) @@ -577,12 +590,13 @@ model LiteLLM_DailyTagSpend { created_at DateTime @default(now()) updated_at DateTime @updatedAt - @@unique([tag, date, api_key, model, custom_llm_provider, mcp_namespaced_tool_name]) + @@unique([tag, date, api_key, model, custom_llm_provider, mcp_namespaced_tool_name, endpoint]) @@index([date]) @@index([tag]) @@index([api_key]) @@index([model]) @@index([mcp_namespaced_tool_name]) + @@index([endpoint]) } diff --git a/test_image_edit.png b/test_image_edit.png new file mode 100644 index 00000000000..0f2de3749df Binary files /dev/null and b/test_image_edit.png differ diff --git a/tests/code_coverage_tests/check_endpoint_coverage.py b/tests/code_coverage_tests/check_endpoint_coverage.py new file mode 100644 index 00000000000..2d46d1ab469 --- /dev/null +++ b/tests/code_coverage_tests/check_endpoint_coverage.py @@ -0,0 +1,379 @@ +""" +Code coverage test to ensure all endpoints documented in sidebars.js are defined in provider_endpoints_support.json. + +This script: +1. Extracts all endpoint entries from the "Supported Endpoints" section of sidebars.js +2. Validates that each endpoint has a corresponding entry in the "endpoints" object of provider_endpoints_support.json +3. Checks that the "docs_label" field is present in each endpoint definition +""" + +import json +import re +import sys +from pathlib import Path +from typing import Dict, List, Set, Tuple + + +class MissingEndpointDefinitionError(Exception): + """Raised when endpoints are documented in sidebars.js but missing from provider_endpoints_support.json.""" + + pass + + +def get_repo_root() -> Path: + """Get the repository root directory.""" + # Check if litellm directory exists in current working directory + cwd = Path.cwd() + if (cwd / "litellm").exists() and (cwd / "litellm").is_dir(): + # We're already at the repo root + return cwd + + # Otherwise, navigate up from script location + current = Path(__file__).resolve() + # Navigate up from tests/code_coverage_tests/ + return current.parent.parent.parent + + +def extract_endpoints_from_sidebars() -> Dict[str, str]: + """ + Extract endpoint entries from sidebars.js. + + Returns a dict mapping endpoint_key -> label + Only extracts top-level endpoint entries from the "Supported Endpoints" section. + """ + repo_root = get_repo_root() + sidebars_path = repo_root / "docs" / "my-website" / "sidebars.js" + + if not sidebars_path.exists(): + print(f"❌ ERROR: Could not find sidebars.js at {sidebars_path}") + sys.exit(1) + + with open(sidebars_path, "r") as f: + content = f.read() + + # Find the Supported Endpoints section + supported_start = content.find('label: "Supported Endpoints"') + if supported_start == -1: + print("⚠️ WARNING: Could not find 'Supported Endpoints' section") + return {} + + # Find the items array within this section + items_start = content.find("items: [", supported_start) + if items_start == -1: + print("⚠️ WARNING: Could not find items array in Supported Endpoints") + return {} + + # Find the end of this items array + # Look for the closing ], at the same indentation level + items_end = content.find("\n ],\n },\n {", items_start) + if items_end == -1: + items_end = content.find("\n ],\n }", items_start) + + section = content[items_start:items_end] + + endpoints = {} + + # Pattern 1: Categories with labels at the top level (8 spaces indent) + # Example: " {type: "category", label: "/a2a - A2A Agent Gateway"" + category_pattern = ( + r'^\s{8}\{\s*\n\s{10}type:\s*"category",\s*\n\s{10}label:\s*"([^"]+)"' + ) + for match in re.finditer(category_pattern, section, re.MULTILINE): + label = match.group(1) + # Skip utility categories + if "Pass-through" in label or label == "Vertex AI": + continue + endpoint_key = label.split(" - ")[0].strip("/").replace("/", "_") + endpoints[endpoint_key] = label + + # Pattern 2: Standalone doc strings at top level (8 spaces indent) + # Example: " "assistants"," + standalone_pattern = r'^\s{8}"([a-zA-Z_][a-zA-Z0-9_]*)",?\s*$' + for match in re.finditer(standalone_pattern, section, re.MULTILINE): + doc_id = match.group(1) + endpoints[doc_id] = doc_id + + return endpoints + + +def load_provider_endpoints_file() -> Dict: + """Load the provider_endpoints_support.json file.""" + repo_root = get_repo_root() + file_path = repo_root / "provider_endpoints_support.json" + + if not file_path.exists(): + print( + f"❌ ERROR: Could not find provider_endpoints_support.json at {file_path}" + ) + sys.exit(1) + + with open(file_path, "r") as f: + return json.load(f) + + +def get_defined_endpoints(data: Dict) -> Dict[str, Dict]: + """Get all endpoint definitions from provider_endpoints_support.json.""" + return data.get("endpoints", {}) + + +def normalize_endpoint_key(key: str) -> Set[str]: + """ + Generate variations of an endpoint key for matching. + + Examples: + - "a2a" -> {"a2a"} + - "chat_completions" -> {"chat_completions", "chatcompletions"} + - "vector_stores" -> {"vector_stores", "vectorstores"} + """ + variations = {key, key.replace("_", "")} + return variations + + +def check_provider_endpoint_keys(data: Dict) -> List[str]: + """ + Check that all endpoint keys used in providers are defined in the root endpoints section. + + Returns a list of missing endpoint keys. + """ + # Collect all unique endpoint keys used across all providers + provider_endpoint_keys = set() + providers = data.get("providers", {}) + + for provider_name, provider_data in providers.items(): + if "endpoints" in provider_data and isinstance( + provider_data["endpoints"], dict + ): + provider_endpoint_keys.update(provider_data["endpoints"].keys()) + + # Get all endpoint definitions + defined_endpoints = data.get("endpoints", {}) + + # Collect all provider_json_field values from endpoint definitions + provider_json_fields = set() + for endpoint_key, endpoint_data in defined_endpoints.items(): + if isinstance(endpoint_data, dict) and "provider_json_field" in endpoint_data: + provider_json_fields.add(endpoint_data["provider_json_field"]) + + # Find missing endpoint keys + missing_keys = [] + for key in sorted(provider_endpoint_keys): + if key not in provider_json_fields: + missing_keys.append(key) + + return missing_keys + + +def check_unused_endpoints(data: Dict) -> List[Tuple[str, str]]: + """ + Check that all defined endpoints are used by at least one provider. + + Returns a list of tuples (endpoint_key, provider_json_field) for unused endpoints. + """ + # Special endpoints that don't need to be used by specific providers + # These are utility/framework endpoints available across the platform + SPECIAL_ENDPOINTS = { + "apply_guardrail", # Guardrail application - works across providers + "mcp", # Model Context Protocol - works across providers + } + + # Get all endpoint definitions + defined_endpoints = data.get("endpoints", {}) + providers = data.get("providers", {}) + + # Collect all endpoint keys used by providers + used_keys = set() + for provider_data in providers.values(): + if "endpoints" in provider_data and isinstance( + provider_data["endpoints"], dict + ): + used_keys.update(provider_data["endpoints"].keys()) + + # Find unused endpoints (excluding special ones) + unused = [] + for endpoint_key, endpoint_data in defined_endpoints.items(): + # Skip special endpoints + if endpoint_key in SPECIAL_ENDPOINTS: + continue + + if isinstance(endpoint_data, dict) and "provider_json_field" in endpoint_data: + provider_json_field = endpoint_data["provider_json_field"] + # Check if this provider_json_field is used by any provider + if provider_json_field not in used_keys: + unused.append((endpoint_key, provider_json_field)) + + return sorted(unused) + + +def main(): + """Main function to validate endpoint coverage.""" + print( + "🔍 Checking endpoint coverage between sidebars.js and provider_endpoints_support.json..." + ) + + has_errors = False + + # Load provider_endpoints_support.json + data = load_provider_endpoints_file() + defined_endpoints = get_defined_endpoints(data) + + # Test 1: Check that endpoints from sidebars.js have docs_label entries + print("\n📖 Test 1: Checking endpoints from sidebars.js...") + sidebar_endpoints = extract_endpoints_from_sidebars() + print(f"✓ Found {len(sidebar_endpoints)} endpoints in sidebars.js") + print( + f"✓ Found {len(defined_endpoints)} endpoint definitions in provider_endpoints_support.json" + ) + + # Check for missing endpoints + missing_endpoints = [] + + # Collect all docs_label values from defined endpoints + defined_docs_labels = set() + for endpoint_data in defined_endpoints.values(): + if isinstance(endpoint_data, dict) and "docs_label" in endpoint_data: + defined_docs_labels.add(endpoint_data["docs_label"]) + + for sidebar_key, sidebar_label in sorted(sidebar_endpoints.items()): + # Generate variations for matching against docs_label + variations = normalize_endpoint_key(sidebar_key) + + # Check if any variation exists in docs_label values + if not any(var in defined_docs_labels for var in variations): + missing_endpoints.append((sidebar_key, sidebar_label)) + + # Report missing endpoints from sidebars + if missing_endpoints: + has_errors = True + error_msg = "\n❌ ERROR: The following endpoints are in sidebars.js but missing from provider_endpoints_support.json:\n" + error_msg += "=" * 70 + "\n" + + for key, label in missing_endpoints: + error_msg += f" - {key}\n" + error_msg += f' Label in sidebars.js: "{label}"\n' + + error_msg += "\n" + "=" * 70 + "\n" + error_msg += f"\n💡 To fix: Add these {len(missing_endpoints)} endpoint(s) to the 'endpoints' object\n" + error_msg += " in provider_endpoints_support.json\n" + error_msg += "\nExample format:\n" + error_msg += ' "endpoints": {\n' + + for key, label in missing_endpoints[:5]: + error_msg += f' "{key}": {{\n' + error_msg += f' "docs_label": "{label}",\n' + error_msg += f' "provider_json_field": "{key}",\n' + error_msg += f' "description": "Description of the {label} endpoint"\n' + error_msg += " },\n" + + if len(missing_endpoints) > 5: + error_msg += " ...\n" + + error_msg += " }\n" + + print(error_msg) + else: + print( + f"✅ All {len(sidebar_endpoints)} endpoints from sidebars.js are defined!" + ) + + # Test 2: Check that all provider endpoint keys have provider_json_field entries + print("\n📋 Test 2: Checking provider endpoint keys...") + missing_provider_keys = check_provider_endpoint_keys(data) + + if missing_provider_keys: + has_errors = True + error_msg = "\n❌ ERROR: The following endpoint keys are used in providers but missing provider_json_field definitions:\n" + error_msg += "=" * 70 + "\n" + + for key in missing_provider_keys: + # Find which providers use this key + using_providers = [] + for provider_name, provider_data in data.get("providers", {}).items(): + if key in provider_data.get("endpoints", {}): + using_providers.append(provider_name) + + error_msg += f" - {key}\n" + error_msg += f" Used by {len(using_providers)} provider(s): {', '.join(using_providers[:3])}" + if len(using_providers) > 3: + error_msg += f" and {len(using_providers) - 3} more" + error_msg += "\n" + + error_msg += "\n" + "=" * 70 + "\n" + error_msg += f"\n💡 To fix: Add these {len(missing_provider_keys)} endpoint(s) to the 'endpoints' object\n" + error_msg += " in provider_endpoints_support.json with 'provider_json_field' matching the key\n" + error_msg += "\nExample format:\n" + error_msg += ' "endpoints": {\n' + + for key in missing_provider_keys[:3]: + error_msg += f' "{key}": {{\n' + error_msg += f' "docs_label": "{key}",\n' + error_msg += f' "provider_json_field": "{key}",\n' + error_msg += f' "description": "Description of the {key} endpoint"\n' + error_msg += " },\n" + + if len(missing_provider_keys) > 3: + error_msg += " ...\n" + + error_msg += " }\n" + + print(error_msg) + else: + print("✅ All provider endpoint keys have provider_json_field definitions!") + + # Test 3: Check that all defined endpoints are used by at least one provider + print("\n🔍 Test 3: Checking for unused endpoint definitions...") + unused_endpoints = check_unused_endpoints(data) + + if unused_endpoints: + has_errors = True + error_msg = "\n⚠️ WARNING: The following endpoint definitions are not used by any provider:\n" + error_msg += "=" * 70 + "\n" + + for endpoint_key, provider_json_field in unused_endpoints: + endpoint_data = defined_endpoints.get(endpoint_key, {}) + docs_label = endpoint_data.get("docs_label", "N/A") + error_msg += f" - {endpoint_key}\n" + error_msg += f" provider_json_field: '{provider_json_field}'\n" + error_msg += f" docs_label: '{docs_label}'\n" + + error_msg += "\n" + "=" * 70 + "\n" + error_msg += f"\n💡 These {len(unused_endpoints)} endpoint(s) are defined but not used by any provider.\n" + error_msg += " Either:\n" + error_msg += ( + " 1. Add the endpoint to relevant providers' 'endpoints' objects, OR\n" + ) + error_msg += " 2. Remove the endpoint definition if it's no longer needed\n" + + print(error_msg) + else: + print("✅ All endpoint definitions are used by at least one provider!") + + # Raise error if any tests failed + if has_errors: + error_summary = [] + if missing_endpoints: + error_summary.append(f"{len(missing_endpoints)} endpoints from sidebars.js") + if missing_provider_keys: + error_summary.append(f"{len(missing_provider_keys)} provider endpoint keys") + if unused_endpoints: + error_summary.append(f"{len(unused_endpoints)} unused endpoint definitions") + + raise MissingEndpointDefinitionError( + f"Endpoint validation failed: Missing definitions for {' and '.join(error_summary)}" + ) + + print("\n🎉 All endpoint coverage validations passed!") + return 0 + + +if __name__ == "__main__": + try: + sys.exit(main()) + except MissingEndpointDefinitionError as e: + print(f"\n🚨 CRITICAL ERROR: {e}\n") + sys.exit(1) + except Exception as e: + print(f"\n🚨 UNEXPECTED ERROR: {e}\n") + import traceback + + traceback.print_exc() + sys.exit(1) diff --git a/tests/code_coverage_tests/check_provider_folders_documented.py b/tests/code_coverage_tests/check_provider_folders_documented.py new file mode 100644 index 00000000000..60afc55331f --- /dev/null +++ b/tests/code_coverage_tests/check_provider_folders_documented.py @@ -0,0 +1,294 @@ +""" +Code coverage test to ensure all provider folders are documented. + +This script validates that: +1. Every provider folder in litellm/llms/ has a corresponding entry in provider_endpoints_support.json +2. Every provider in litellm/llms/openai_like/providers.json is documented in provider_endpoints_support.json +""" + +import json +import os +import sys +from pathlib import Path +from typing import Dict, List, Set, Tuple + + +class UndocumentedProviderError(Exception): + """Raised when providers are found without documentation.""" + + pass + + +# Special folders that should be excluded from validation +EXCLUDED_FOLDERS = { + "__pycache__", + "base_llm", + "deprecated_providers", + "custom_httpx", + "pass_through", + "openai_like", # This is a generic handler, not a specific provider + "aiohttp_openai", # Internal implementation detail for async HTTP +} + + +def get_repo_root() -> Path: + """Get the repository root directory.""" + # Check if litellm directory exists in current working directory + cwd = Path.cwd() + if (cwd / "litellm").exists() and (cwd / "litellm").is_dir(): + # We're already at the repo root + return cwd + + # Otherwise, navigate up from script location + current = Path(__file__).resolve() + # Navigate up from tests/code_coverage_tests/ + return current.parent.parent.parent + + +def get_llm_provider_folders() -> Set[str]: + """Get all provider folder names from litellm/llms directory.""" + repo_root = get_repo_root() + llms_dir = repo_root / "litellm" / "llms" + + if not llms_dir.exists(): + print(f"❌ ERROR: Could not find llms directory at {llms_dir}") + sys.exit(1) + + folders = set() + for item in llms_dir.iterdir(): + if item.is_dir() and item.name not in EXCLUDED_FOLDERS: + folders.add(item.name) + + return folders + + +def load_provider_endpoints_file() -> Dict: + """Load the provider_endpoints_support.json file.""" + repo_root = get_repo_root() + file_path = repo_root / "provider_endpoints_support.json" + + if not file_path.exists(): + print( + f"❌ ERROR: Could not find provider_endpoints_support.json at {file_path}" + ) + sys.exit(1) + + with open(file_path, "r") as f: + return json.load(f) + + +def get_openai_like_providers() -> Set[str]: + """Get all provider names from litellm/llms/openai_like/providers.json.""" + repo_root = get_repo_root() + providers_file = repo_root / "litellm" / "llms" / "openai_like" / "providers.json" + + if not providers_file.exists(): + print( + f"⚠️ WARNING: Could not find openai_like/providers.json at {providers_file}" + ) + return set() + + with open(providers_file, "r") as f: + data = json.load(f) + + # Return all provider keys from the JSON + return set(data.keys()) + + +def get_documented_providers(data: Dict) -> Set[str]: + """Get all provider slugs documented in provider_endpoints_support.json.""" + providers = data.get("providers", {}) + + # Get all provider keys, including those with slashes + documented = set() + for provider_key in providers.keys(): + # For providers like "azure_ai/doc-intelligence", extract base name + base_name = provider_key.split("/")[0] + documented.add(base_name) + # Also add the full key in case folder name matches exactly + documented.add(provider_key) + + return documented + + +def normalize_provider_name(folder_name: str) -> Set[str]: + """ + Generate possible provider names that might match a folder. + + Some folders might have variations in the JSON: + - github_copilot folder -> github_copilot provider + - azure folder -> azure, azure_text, azure_ai providers + """ + variations = {folder_name} + + # Add common variations + if "_" in folder_name: + # Try without underscores (though less common) + variations.add(folder_name.replace("_", "")) + + return variations + + +def main(): + """Main function to validate provider documentation.""" + print("🔍 Checking that all providers are documented...") + + has_errors = False + + # Check 1: Provider folders in litellm/llms + print("\n📁 Checking provider folders in litellm/llms/...") + provider_folders = get_llm_provider_folders() + print(f"✓ Found {len(provider_folders)} provider folders") + + # Check 2: OpenAI-like providers + print("\n📋 Checking openai_like providers...") + openai_like_providers = get_openai_like_providers() + print(f"✓ Found {len(openai_like_providers)} openai_like providers") + + # Load the JSON file + data = load_provider_endpoints_file() + documented_providers = get_documented_providers(data) + print( + f"\n✓ Found {len(data.get('providers', {}))} provider entries in provider_endpoints_support.json" + ) + + # Check for undocumented folders + undocumented_folders = [] + for folder in sorted(provider_folders): + # Check if folder name or any variation is documented + variations = normalize_provider_name(folder) + if not any(var in documented_providers for var in variations): + undocumented_folders.append(folder) + + # Check for undocumented openai_like providers + undocumented_openai_like = [] + for provider in sorted(openai_like_providers): + # Generate multiple possible variations of the provider name + variations = { + provider, # Original name (e.g., "nano-gpt") + provider.replace( + "-", "_" + ), # Replace hyphens with underscores (e.g., "nano_gpt") + provider.replace("-", ""), # Remove hyphens (e.g., "nanogpt") + provider.replace("_", ""), # Remove underscores + } + + # Special case mappings for known variations + special_mappings = { + "veniceai": "venice", + "nano-gpt": "nanogpt", + } + if provider in special_mappings: + variations.add(special_mappings[provider]) + + # Check if any variation is documented + if not any(var in documented_providers for var in variations): + undocumented_openai_like.append(provider) + + # Collect all error messages + error_messages: List[str] = [] + + # Report errors for undocumented folders + if undocumented_folders: + has_errors = True + error_msg = "\n❌ ERROR: The following provider folders are not documented:\n" + error_msg += "=" * 70 + "\n" + for folder in undocumented_folders: + error_msg += f" - litellm/llms/{folder}/\n" + + error_msg += "\n" + "=" * 70 + "\n" + error_msg += f"\n💡 To fix: Add entries for these {len(undocumented_folders)} provider(s)\n" + error_msg += ( + " in the 'providers' section of provider_endpoints_support.json\n" + ) + error_msg += "\nExample format:\n" + error_msg += ' "providers": {\n' + for folder in undocumented_folders[:3]: + error_msg += f' "{folder}": {{\n' + error_msg += f' "display_name": "{folder.replace("_", " ").title()} (`{folder}`)",\n' + error_msg += ( + f' "url": "https://docs.litellm.ai/docs/providers/{folder}",\n' + ) + error_msg += ' "endpoints": {\n' + error_msg += ' "chat_completions": true,\n' + error_msg += ' "messages": true,\n' + error_msg += ' "responses": true,\n' + error_msg += ' "embeddings": false,\n' + error_msg += " ...\n" + error_msg += " }\n" + error_msg += " },\n" + if len(undocumented_folders) > 3: + error_msg += " ...\n" + error_msg += " }\n" + + print(error_msg) + error_messages.append( + f"Found {len(undocumented_folders)} undocumented provider folders: {', '.join(undocumented_folders)}" + ) + + # Report errors for undocumented openai_like providers + if undocumented_openai_like: + has_errors = True + error_msg = ( + "\n❌ ERROR: The following openai_like providers are not documented:\n" + ) + error_msg += "=" * 70 + "\n" + for provider in undocumented_openai_like: + error_msg += f" - {provider}\n" + + error_msg += "\n" + "=" * 70 + "\n" + error_msg += f"\n💡 To fix: Add entries for these {len(undocumented_openai_like)} provider(s)\n" + error_msg += ( + " in the 'providers' section of provider_endpoints_support.json\n" + ) + error_msg += "\nExample format:\n" + error_msg += ' "providers": {\n' + for provider in undocumented_openai_like[:3]: + normalized = provider.replace("-", "_") + error_msg += f' "{normalized}": {{\n' + error_msg += f' "display_name": "{provider.replace("-", " ").replace("_", " ").title()} (`{normalized}`)",\n' + error_msg += ( + f' "url": "https://docs.litellm.ai/docs/providers/{normalized}",\n' + ) + error_msg += ' "endpoints": {\n' + error_msg += ' "chat_completions": true,\n' + error_msg += ' "messages": true,\n' + error_msg += ' "responses": true,\n' + error_msg += ' "embeddings": false,\n' + error_msg += " ...\n" + error_msg += " }\n" + error_msg += " },\n" + if len(undocumented_openai_like) > 3: + error_msg += " ...\n" + error_msg += " }\n" + + print(error_msg) + error_messages.append( + f"Found {len(undocumented_openai_like)} undocumented openai_like providers: {', '.join(undocumented_openai_like)}" + ) + + # Raise exception if there are any errors + if has_errors: + error_summary = " AND ".join(error_messages) + raise UndocumentedProviderError( + f"Provider documentation validation failed: {error_summary}" + ) + + print(f"\n✅ All {len(provider_folders)} provider folders are documented!") + print(f"✅ All {len(openai_like_providers)} openai_like providers are documented!") + print("\n🎉 All provider documentation checks passed!") + return 0 + + +if __name__ == "__main__": + try: + sys.exit(main()) + except UndocumentedProviderError as e: + print(f"\n🚨 CRITICAL ERROR: {e}\n") + sys.exit(1) + except Exception as e: + print(f"\n🚨 UNEXPECTED ERROR: {e}\n") + import traceback + + traceback.print_exc() + sys.exit(1) diff --git a/tests/code_coverage_tests/liccheck.ini b/tests/code_coverage_tests/liccheck.ini index 01d8bc4aa09..cd73f3fe4ab 100644 --- a/tests/code_coverage_tests/liccheck.ini +++ b/tests/code_coverage_tests/liccheck.ini @@ -138,4 +138,5 @@ pondpond: >=1.4.1 # Apache 2.0 License fastuuid: >=0.13.0 # BSD-3-Clause license llm-sandbox: >=0.3.31 # MIT License - https://github.com/vndee/llm-sandbox nodejs-wheel-binaries: >=24.12.0 # MIT license manually verified +grpcio: >=1.69.0 # Apache License 2.0 diff --git a/tests/image_gen_tests/test_image_edits.py b/tests/image_gen_tests/test_image_edits.py index 68acb7ac7fc..810bd80a5b0 100644 --- a/tests/image_gen_tests/test_image_edits.py +++ b/tests/image_gen_tests/test_image_edits.py @@ -143,6 +143,23 @@ class TestOpenAIImageEditDallE2(BaseLLMImageEditTest): } +class TestAzureAIFlux2ImageEdit(BaseLLMImageEditTest): + """ + Concrete implementation of BaseLLMImageEditTest for Azure AI FLUX 2 image edits. + FLUX 2 uses JSON with base64 image instead of multipart/form-data. + """ + + def get_base_image_edit_call_args(self) -> dict: + """Return base call args for Azure AI FLUX 2 image edit""" + return { + "model": "azure_ai/flux.2-pro", + "image": SINGLE_TEST_IMAGE, + "api_base": os.getenv("AZURE_AI_API_BASE", "https://litellm-ci-cd-prod.services.ai.azure.com"), + "api_key": os.getenv("AZURE_AI_API_KEY"), + "api_version": "preview", + } + + @pytest.mark.flaky(retries=3, delay=2) @pytest.mark.asyncio async def test_openai_image_edit_litellm_router(): @@ -322,14 +339,23 @@ async def test_openai_image_edit_cost_tracking(): litellm.logging_callback_manager._reset_all_callbacks() litellm.callbacks = [test_custom_logger] - # Mock response for Azure image edit + # Mock response for Azure image edit with usage data for cost tracking mock_response = { "created": 1589478378, "data": [ { "b64_json": "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8/5+hHgAHggJ/PchI7wAAAABJRU5ErkJggg==" } - ] + ], + "usage": { + "total_tokens": 1100, + "input_tokens": 100, + "input_tokens_details": { + "image_tokens": 50, + "text_tokens": 50 + }, + "output_tokens": 1000 + } } class MockResponse: @@ -401,14 +427,23 @@ async def test_azure_image_edit_cost_tracking(): litellm.logging_callback_manager._reset_all_callbacks() litellm.callbacks = [test_custom_logger] - # Mock response for Azure image edit + # Mock response for Azure image edit with usage data for cost tracking mock_response = { "created": 1589478378, "data": [ { "b64_json": "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8/5+hHgAHggJ/PchI7wAAAABJRU5ErkJggg==" } - ] + ], + "usage": { + "total_tokens": 1100, + "input_tokens": 100, + "input_tokens_details": { + "image_tokens": 50, + "text_tokens": 50 + }, + "output_tokens": 1000 + } } class MockResponse: diff --git a/tests/llm_translation/test_bedrock_completion.py b/tests/llm_translation/test_bedrock_completion.py index 78c9f94239b..ec510b8f953 100644 --- a/tests/llm_translation/test_bedrock_completion.py +++ b/tests/llm_translation/test_bedrock_completion.py @@ -322,10 +322,7 @@ def process_stream_response(res, messages): return res -@pytest.mark.skipif( - os.environ.get("CIRCLE_OIDC_TOKEN_V2") is None, - reason="Cannot run without being in CircleCI Runner", -) +@pytest.mark.skip(reason="Cannot run without being in CircleCI Runner") def test_completion_bedrock_claude_aws_session_token(bedrock_session_token_creds): print("\ncalling bedrock claude with aws_session_token auth") @@ -406,10 +403,7 @@ def test_completion_bedrock_claude_aws_session_token(bedrock_session_token_creds pytest.fail(f"Error occurred: {e}") -@pytest.mark.skipif( - os.environ.get("CIRCLE_OIDC_TOKEN_V2") is None, - reason="Cannot run without being in CircleCI Runner", -) +@pytest.mark.skip(reason="Cannot run without being in CircleCI Runner") def test_completion_bedrock_claude_aws_bedrock_client(bedrock_session_token_creds): print("\ncalling bedrock claude with aws_session_token auth") diff --git a/tests/local_testing/test_arize_ai.py b/tests/local_testing/test_arize_ai.py index 6a773521435..3b497d638ae 100644 --- a/tests/local_testing/test_arize_ai.py +++ b/tests/local_testing/test_arize_ai.py @@ -71,6 +71,7 @@ def test_get_arize_config(mock_env_vars): assert config.api_key == "test_api_key" assert config.endpoint == "https://otlp.arize.com/v1" assert config.protocol == "otlp_grpc" + assert config.project_name is None def test_get_arize_config_with_endpoints(mock_env_vars, monkeypatch): @@ -79,10 +80,12 @@ def test_get_arize_config_with_endpoints(mock_env_vars, monkeypatch): """ monkeypatch.setenv("ARIZE_ENDPOINT", "grpc://test.endpoint") monkeypatch.setenv("ARIZE_HTTP_ENDPOINT", "http://test.endpoint") + monkeypatch.setenv("ARIZE_PROJECT_NAME", "custom-project") config = ArizeLogger.get_arize_config() assert config.endpoint == "grpc://test.endpoint" assert config.protocol == "otlp_grpc" + assert config.project_name == "custom-project" @pytest.mark.skip( diff --git a/tests/local_testing/test_completion.py b/tests/local_testing/test_completion.py index 5e92c10fbdc..8d815829d40 100644 --- a/tests/local_testing/test_completion.py +++ b/tests/local_testing/test_completion.py @@ -3059,7 +3059,6 @@ def response_format_tests(response: litellm.ModelResponse): "bedrock/cohere.command-r-plus-v1:0", "anthropic.claude-3-sonnet-20240229-v1:0", "mistral.mistral-7b-instruct-v0:2", - # "bedrock/amazon.titan-tg1-large", "meta.llama3-8b-instruct-v1:0", ], ) @@ -3101,31 +3100,6 @@ async def test_completion_bedrock_httpx_models(sync_mode, model): pytest.fail(f"An error occurred - {str(e)}") -def test_completion_bedrock_titan_null_response(): - try: - # amazon.titan-text-lite-v1 is deprecated, using titan-text-express-v1 instead - response = completion( - model="bedrock/amazon.titan-text-express-v1", - messages=[ - { - "role": "user", - "content": "Hello!", - }, - { - "role": "assistant", - "content": "Hello! How can I help you?", - }, - { - "role": "user", - "content": "What model are you?", - }, - ], - ) - # Add any assertions here to check the response - print(f"response: {response}") - except Exception as e: - pytest.fail(f"An error occurred - {str(e)}") - # test_completion_bedrock_titan() @@ -3916,26 +3890,7 @@ async def test_dynamic_azure_params(stream, sync_mode): raise e -@pytest.mark.asyncio() -@pytest.mark.flaky(retries=3, delay=1) -async def test_completion_ai21_chat(): - litellm.set_verbose = True - try: - response = await litellm.acompletion( - model="ai21_chat/jamba-mini", - user="ishaan", - tool_choice="auto", - seed=123, - messages=[{"role": "user", "content": "what does the document say"}], - documents=[ - { - "content": "hello world", - "metadata": {"source": "google", "author": "ishaan"}, - } - ], - ) - except litellm.InternalServerError: - pytest.skip("Model is overloaded") + @pytest.mark.parametrize( diff --git a/tests/local_testing/test_streaming.py b/tests/local_testing/test_streaming.py index b9b5d0fdb07..00732a12cfe 100644 --- a/tests/local_testing/test_streaming.py +++ b/tests/local_testing/test_streaming.py @@ -552,36 +552,6 @@ async def test_completion_predibase_streaming(sync_mode): pytest.fail(f"Error occurred: {e}") -@pytest.mark.asyncio() -@pytest.mark.flaky(retries=3, delay=1) -async def test_completion_ai21_stream(): - litellm.set_verbose = True - response = await litellm.acompletion( - model="ai21_chat/jamba-mini", - user="ishaan", - stream=True, - seed=123, - messages=[{"role": "user", "content": "hi my name is ishaan"}], - ) - complete_response = "" - idx = 0 - async for init_chunk in response: - chunk, finished = streaming_format_tests(idx, init_chunk) - complete_response += chunk - custom_llm_provider = init_chunk._hidden_params["custom_llm_provider"] - print(f"custom_llm_provider: {custom_llm_provider}") - assert custom_llm_provider == "ai21_chat" - idx += 1 - if finished: - assert isinstance(init_chunk.choices[0], litellm.utils.StreamingChoices) - break - if complete_response.strip() == "": - raise Exception("Empty response received") - - print(f"complete_response: {complete_response}") - - pass - def test_completion_azure_function_calling_stream(): try: @@ -1318,7 +1288,6 @@ async def test_completion_replicate_llama3_streaming(sync_mode): # ["bedrock/cohere.command-r-plus-v1:0", None], ["anthropic.claude-3-sonnet-20240229-v1:0", None], # ["mistral.mistral-7b-instruct-v0:2", None], - ["bedrock/amazon.titan-tg1-large", None], # ["meta.llama3-8b-instruct-v1:0", None], ], ) diff --git a/tests/logging_callback_tests/test_gcs_pub_sub.py b/tests/logging_callback_tests/test_gcs_pub_sub.py index d45110b3277..8ffbc8eedd5 100644 --- a/tests/logging_callback_tests/test_gcs_pub_sub.py +++ b/tests/logging_callback_tests/test_gcs_pub_sub.py @@ -40,6 +40,7 @@ ignored_keys = [ "metadata.usage_object", "metadata.cold_storage_object_key", "metadata.litellm_overhead_time_ms", + "metadata.cost_breakdown", ] diff --git a/tests/logging_callback_tests/test_otel_logging.py b/tests/logging_callback_tests/test_otel_logging.py index 8d1da0439d3..3350c6c2dbd 100644 --- a/tests/logging_callback_tests/test_otel_logging.py +++ b/tests/logging_callback_tests/test_otel_logging.py @@ -138,64 +138,6 @@ def validate_raw_gen_ai_request_openai_streaming(span): assert span._attributes[attr] is not None, f"Attribute {attr} has None" -@pytest.mark.parametrize( - "model", - ["anthropic/claude-3-opus-20240229"], -) -@pytest.mark.flaky(retries=6, delay=2) -def test_completion_claude_3_function_call_with_otel(model): - litellm.set_verbose = True - - litellm.callbacks = [OpenTelemetry(config=OpenTelemetryConfig(exporter=exporter))] - tools = [ - { - "type": "function", - "function": { - "name": "get_current_weather", - "description": "Get the current weather in a given location", - "parameters": { - "type": "object", - "properties": { - "location": { - "type": "string", - "description": "The city and state, e.g. San Francisco, CA", - }, - "unit": {"type": "string", "enum": ["celsius", "fahrenheit"]}, - }, - "required": ["location"], - }, - }, - } - ] - messages = [ - { - "role": "user", - "content": "What's the weather like in Boston today in Fahrenheit?", - } - ] - try: - # test without max tokens - response = litellm.completion( - model=model, - messages=messages, - tools=tools, - tool_choice={ - "type": "function", - "function": {"name": "get_current_weather"}, - }, - drop_params=True, - ) - - print("response from LiteLLM", response) - except litellm.InternalServerError: - pass - except Exception as e: - pytest.fail(f"Error occurred: {e}") - finally: - # clear in memory exporter - exporter.clear() - - @pytest.mark.asyncio @pytest.mark.parametrize("streaming", [True, False]) @pytest.mark.parametrize("global_redact", [True, False]) diff --git a/tests/pass_through_tests/test_anthropic_passthrough_basic.py b/tests/pass_through_tests/test_anthropic_passthrough_basic.py index 86d93818249..21e53994dcc 100644 --- a/tests/pass_through_tests/test_anthropic_passthrough_basic.py +++ b/tests/pass_through_tests/test_anthropic_passthrough_basic.py @@ -21,7 +21,7 @@ class TestAnthropicMessagesEndpoint(BaseAnthropicMessagesTest): def test_anthropic_messages_to_wildcard_model(self): client = self.get_client() response = client.messages.create( - model="anthropic/claude-3-opus-20240229", + model="anthropic/claude-haiku-4-5-20251001", messages=[{"role": "user", "content": "Hello, world!"}], max_tokens=100, ) diff --git a/tests/proxy_unit_tests/test_key_generate_prisma.py b/tests/proxy_unit_tests/test_key_generate_prisma.py index 52481806fea..e0d6b7e81bb 100644 --- a/tests/proxy_unit_tests/test_key_generate_prisma.py +++ b/tests/proxy_unit_tests/test_key_generate_prisma.py @@ -3504,6 +3504,7 @@ async def test_list_keys(prisma_client): include_created_by_keys=False, sort_by=None, sort_order="desc", + expand=None, ) print("response=", response) assert "keys" in response @@ -3528,6 +3529,7 @@ async def test_list_keys(prisma_client): include_created_by_keys=False, sort_by=None, sort_order="desc", + expand=None, ) print("pagination response=", response) assert len(response["keys"]) == 2 @@ -3568,6 +3570,7 @@ async def test_list_keys(prisma_client): include_created_by_keys=False, sort_by=None, sort_order="desc", + expand=None, ) print("filtered user_id response=", response) assert len(response["keys"]) == 1 @@ -3589,6 +3592,7 @@ async def test_list_keys(prisma_client): include_created_by_keys=False, sort_by=None, sort_order="desc", + expand=None, ) assert len(response["keys"]) == 1 assert _key in response["keys"] diff --git a/tests/router_unit_tests/test_router_helper_utils.py b/tests/router_unit_tests/test_router_helper_utils.py index 45aae3b9aee..073433cb9e5 100644 --- a/tests/router_unit_tests/test_router_helper_utils.py +++ b/tests/router_unit_tests/test_router_helper_utils.py @@ -73,7 +73,7 @@ def test_routing_strategy_init(model_list): from litellm.types.router import RoutingStrategy router = Router(model_list=model_list) - for strategy in RoutingStrategy._member_names_: + for strategy in RoutingStrategy: router.routing_strategy_init( routing_strategy=strategy, routing_strategy_args={} ) diff --git a/tests/test_litellm/containers/test_container_api.py b/tests/test_litellm/containers/test_container_api.py index d4c42b0b3d6..c7bb68e79cb 100644 --- a/tests/test_litellm/containers/test_container_api.py +++ b/tests/test_litellm/containers/test_container_api.py @@ -134,80 +134,6 @@ class TestContainerAPI: assert response.id == "cntr_async_123" assert response.name == "Async Test Container" - def test_list_containers_basic(self): - """Test basic container listing functionality.""" - mock_response = ContainerListResponse( - object="list", - data=[ - ContainerObject( - id="cntr_1", - object="container", - created_at=1747857508, - status="running", - expires_after={"anchor": "last_active_at", "minutes": 20}, - last_active_at=1747857508, - name="Container 1" - ), - ContainerObject( - id="cntr_2", - object="container", - created_at=1747857600, - status="running", - expires_after={"anchor": "last_active_at", "minutes": 15}, - last_active_at=1747857600, - name="Container 2" - ) - ], - first_id="cntr_1", - last_id="cntr_2", - has_more=False - ) - - with patch('litellm.containers.main.base_llm_http_handler') as mock_handler: - mock_handler.container_list_handler.return_value = mock_response - - response = list_containers( - custom_llm_provider="openai" - ) - - assert isinstance(response, ContainerListResponse) - assert len(response.data) == 2 - assert response.data[0].id == "cntr_1" - assert response.data[1].id == "cntr_2" - assert response.has_more == False - - def test_list_containers_with_params(self): - """Test container listing with parameters.""" - mock_response = ContainerListResponse( - object="list", - data=[ - ContainerObject( - id="cntr_limited", - object="container", - created_at=1747857508, - status="running", - expires_after={"anchor": "last_active_at", "minutes": 20}, - last_active_at=1747857508, - name="Limited Container" - ) - ], - first_id="cntr_limited", - last_id="cntr_limited", - has_more=True - ) - - with patch('litellm.containers.main.base_llm_http_handler') as mock_handler: - mock_handler.container_list_handler.return_value = mock_response - - response = list_containers( - limit=1, - order="desc", - after="cntr_prev", - custom_llm_provider="openai" - ) - - assert len(response.data) == 1 - assert response.has_more == True @pytest.mark.asyncio async def test_alist_containers_basic(self): diff --git a/tests/test_litellm/integrations/arize/test_arize_health_check.py b/tests/test_litellm/integrations/arize/test_arize_health_check.py index 91d0b42d48d..8d86b7dc097 100644 --- a/tests/test_litellm/integrations/arize/test_arize_health_check.py +++ b/tests/test_litellm/integrations/arize/test_arize_health_check.py @@ -123,7 +123,8 @@ class TestArizeIntegrationWithProxy: with patch.dict(os.environ, { "ARIZE_SPACE_KEY": "test-space-123", "ARIZE_API_KEY": "test-api-456", - "ARIZE_ENDPOINT": "https://custom.arize.com/v1" + "ARIZE_ENDPOINT": "https://custom.arize.com/v1", + "ARIZE_PROJECT_NAME": "custom-project", }): config = ArizeLogger.get_arize_config() @@ -131,13 +132,15 @@ class TestArizeIntegrationWithProxy: assert config.api_key == "test-api-456" assert config.endpoint == "https://custom.arize.com/v1" assert config.protocol == "otlp_grpc" + assert config.project_name == "custom-project" def test_arize_get_config_defaults(self): """Test ArizeLogger.get_arize_config() with default endpoint.""" with patch.dict(os.environ, { "ARIZE_SPACE_KEY": "test-space-default", - "ARIZE_API_KEY": "test-api-default" + "ARIZE_API_KEY": "test-api-default", + "ARIZE_PROJECT_NAME": "default-project", }, clear=True): config = ArizeLogger.get_arize_config() @@ -145,6 +148,7 @@ class TestArizeIntegrationWithProxy: assert config.api_key == "test-api-default" assert config.endpoint == "https://otlp.arize.com/v1" # Default endpoint assert config.protocol == "otlp_grpc" # Default protocol + assert config.project_name == "default-project" def test_arize_construct_dynamic_headers(self): """Test dynamic OTEL headers construction for team/key logging.""" @@ -180,4 +184,4 @@ class TestArizeIntegrationWithProxy: if __name__ == "__main__": - pytest.main([__file__, "-v"]) \ No newline at end of file + pytest.main([__file__, "-v"]) diff --git a/tests/test_litellm/integrations/gitlab/test_gitlab_prompt_manager.py b/tests/test_litellm/integrations/gitlab/test_gitlab_prompt_manager.py index 8475252cfc2..d623dba0c34 100644 --- a/tests/test_litellm/integrations/gitlab/test_gitlab_prompt_manager.py +++ b/tests/test_litellm/integrations/gitlab/test_gitlab_prompt_manager.py @@ -1,18 +1,19 @@ import os import sys from unittest.mock import MagicMock, patch + import pytest sys.path.insert(0, os.path.abspath("../../..")) # Adds the parent directory to the system path from litellm.integrations.gitlab.gitlab_client import GitLabClient from litellm.integrations.gitlab.gitlab_prompt_manager import ( + GitLabPromptCache, GitLabPromptManager, GitLabPromptTemplate, GitLabTemplateManager, - GitLabPromptCache, - encode_prompt_id, decode_prompt_id, + encode_prompt_id, ) # ----------------------- @@ -817,22 +818,3 @@ def test_cache_get_by_file_returns_exact_entry(mock_pm_cls, fake_managers): assert beta and beta["id"] == "nested/beta" -@patch("litellm.integrations.gitlab.gitlab_prompt_manager.GitLabPromptManager") -def test_encode_decode_helpers_roundtrip_in_cache_context(mock_pm_cls, fake_managers): - tm, wrapper = fake_managers - tm._discoverable_ids = ["dir1/dir2/item"] - mock_pm_cls.return_value = wrapper - - cache = GitLabPromptCache({"project": "g/s/r", "access_token": "tkn"}) - cache.load_all() - - encoded = encode_prompt_id("dir1/dir2/item") - assert encoded in cache.list_ids() - - # decode → encode → lookup should still work - decoded = decode_prompt_id(encoded) - assert decoded == "dir1/dir2/item" - - got = cache.get_by_id(decoded) - assert got is not None - assert got["id"] == "dir1/dir2/item" \ No newline at end of file diff --git a/tests/test_litellm/integrations/levo/test_levo.py b/tests/test_litellm/integrations/levo/test_levo.py index 5d042cbc060..3c89f8eeba2 100644 --- a/tests/test_litellm/integrations/levo/test_levo.py +++ b/tests/test_litellm/integrations/levo/test_levo.py @@ -9,10 +9,10 @@ from litellm.integrations.opentelemetry import OpenTelemetryConfig # Try to import OpenTelemetry packages, skip tests if not available try: from opentelemetry.sdk.trace import TracerProvider + from opentelemetry.sdk.trace.export import SimpleSpanProcessor from opentelemetry.sdk.trace.export.in_memory_span_exporter import ( InMemorySpanExporter, ) - from opentelemetry.sdk.trace.export import SimpleSpanProcessor OPENTELEMETRY_AVAILABLE = True except ImportError: @@ -150,53 +150,6 @@ class TestLevoConfig(unittest.TestCase): class TestLevoIntegration(unittest.TestCase): """Integration tests for LevoLogger.""" - - @patch.dict( - "os.environ", - { - "LEVOAI_API_KEY": "test-api-key", - "LEVOAI_ORG_ID": "test-org-id", - "LEVOAI_WORKSPACE_ID": "test-workspace-id", - "LEVOAI_COLLECTOR_URL": "https://collector.levo.ai", - }, - ) - @pytest.mark.skipif( - not OPENTELEMETRY_AVAILABLE, reason="OpenTelemetry packages not installed" - ) - @patch( - "litellm.integrations.opentelemetry.OpenTelemetry._init_otel_logger_on_litellm_proxy" - ) - def test_levo_logger_instantiation(self, mock_init_proxy): - """Test that LevoLogger can be instantiated with proper config.""" - # Mock the proxy initialization to avoid importing proxy code - mock_init_proxy.return_value = None - - config = LevoLogger.get_levo_config() - otel_config = OpenTelemetryConfig( - exporter=config.protocol, - endpoint=config.endpoint, - headers=config.otlp_auth_headers, - ) - - # Create a tracer provider with in-memory exporter to avoid requiring OTLP packages - tracer_provider = TracerProvider() - tracer_provider.add_span_processor(SimpleSpanProcessor(InMemorySpanExporter())) - - # Create LevoLogger instance with mocked tracer provider - levo_logger = LevoLogger( - config=otel_config, callback_name="levo", tracer_provider=tracer_provider - ) - - # Verify it's an instance of OpenTelemetry - self.assertIsInstance(levo_logger, LevoLogger) - # Check it extends OpenTelemetry by checking base classes - from litellm.integrations.opentelemetry import OpenTelemetry - - self.assertIsInstance(levo_logger, OpenTelemetry) - - # Verify callback_name is set - self.assertEqual(levo_logger.callback_name, "levo") - @patch.dict( "os.environ", { diff --git a/tests/test_litellm/integrations/test_opentelemetry.py b/tests/test_litellm/integrations/test_opentelemetry.py index 6c17570e135..55b65fbb92a 100644 --- a/tests/test_litellm/integrations/test_opentelemetry.py +++ b/tests/test_litellm/integrations/test_opentelemetry.py @@ -258,6 +258,22 @@ class TestOpenTelemetry(unittest.TestCase): MODEL = "arn:aws:bedrock:us-west-2:1234567890123:inference-profile/us.anthropic.claude-3-7-sonnet-20250219-v1:0" HERE = os.path.dirname(__file__) + @patch.dict(os.environ, {}, clear=True) + def test_open_telemetry_config_manual_defaults(self): + """Manual OpenTelemetryConfig creation should populate default identifiers.""" + config = OpenTelemetryConfig(exporter="console", endpoint="http://collector") + self.assertEqual(config.service_name, "litellm") + self.assertEqual(config.deployment_environment, "production") + self.assertEqual(config.model_id, "litellm") + + @patch.dict(os.environ, {}, clear=True) + def test_open_telemetry_config_custom_service_name(self): + """Model ID should inherit provided service name when not explicitly set.""" + config = OpenTelemetryConfig(service_name="custom-service", exporter="console") + self.assertEqual(config.service_name, "custom-service") + self.assertEqual(config.deployment_environment, "production") + self.assertEqual(config.model_id, "custom-service") + def wait_for_spans(self, exporter: InMemorySpanExporter, prefix: str): """Poll until we see at least one span with an attribute key starting with `prefix`.""" deadline = time.time() + self.POLL_TIMEOUT @@ -504,8 +520,6 @@ class TestOpenTelemetry(unittest.TestCase): self, mock_detector_cls, mock_resource_create ): """Test _get_litellm_resource with default values when no environment variables are set.""" - from litellm.integrations.opentelemetry import _get_litellm_resource - # Mock the Resource.create method mock_base_resource = MagicMock() mock_resource_create.return_value = mock_base_resource @@ -520,8 +534,8 @@ class TestOpenTelemetry(unittest.TestCase): mock_merged_resource = MagicMock() mock_base_resource.merge.return_value = mock_merged_resource - # Call the function - result = _get_litellm_resource() + config = OpenTelemetryConfig() + result = OpenTelemetry._get_litellm_resource(config) # Verify Resource.create was called with correct default attributes expected_attributes = { @@ -549,8 +563,6 @@ class TestOpenTelemetry(unittest.TestCase): self, mock_detector_cls, mock_resource_create ): """Test _get_litellm_resource with LiteLLM-specific environment variables.""" - from litellm.integrations.opentelemetry import _get_litellm_resource - # Mock the Resource.create method mock_base_resource = MagicMock() mock_resource_create.return_value = mock_base_resource @@ -565,8 +577,8 @@ class TestOpenTelemetry(unittest.TestCase): mock_merged_resource = MagicMock() mock_base_resource.merge.return_value = mock_merged_resource - # Call the function - result = _get_litellm_resource() + config = OpenTelemetryConfig.from_env() + result = OpenTelemetry._get_litellm_resource(config) # Verify Resource.create was called with environment variable values expected_attributes = { @@ -593,8 +605,6 @@ class TestOpenTelemetry(unittest.TestCase): self, mock_detector_cls, mock_resource_create ): """Test _get_litellm_resource with OTEL_RESOURCE_ATTRIBUTES environment variable.""" - from litellm.integrations.opentelemetry import _get_litellm_resource - # Mock the Resource.create method to simulate the actual behavior # In reality, Resource.create() would parse OTEL_RESOURCE_ATTRIBUTES and merge it mock_base_resource = MagicMock() @@ -610,8 +620,8 @@ class TestOpenTelemetry(unittest.TestCase): mock_merged_resource = MagicMock() mock_base_resource.merge.return_value = mock_merged_resource - # Call the function - result = _get_litellm_resource() + config = OpenTelemetryConfig.from_env() + result = OpenTelemetry._get_litellm_resource(config) # Verify Resource.create was called with the base attributes # The actual OTEL_RESOURCE_ATTRIBUTES parsing is handled by OpenTelemetry SDK @@ -628,10 +638,8 @@ class TestOpenTelemetry(unittest.TestCase): @patch.dict(os.environ, {}, clear=True) def test_get_litellm_resource_integration_with_real_resource(self): """Integration test to verify _get_litellm_resource works with actual OpenTelemetry Resource.""" - from litellm.integrations.opentelemetry import _get_litellm_resource - - # This test uses the real OpenTelemetry Resource.create() method - result = _get_litellm_resource() + config = OpenTelemetryConfig() + result = OpenTelemetry._get_litellm_resource(config) # Verify the result is a Resource instance from opentelemetry.sdk.resources import Resource @@ -653,10 +661,8 @@ class TestOpenTelemetry(unittest.TestCase): ) def test_get_litellm_resource_real_otel_resource_attributes(self): """Integration test to verify OTEL_RESOURCE_ATTRIBUTES is properly handled.""" - from litellm.integrations.opentelemetry import _get_litellm_resource - - # This test uses the real OpenTelemetry Resource.create() method - result = _get_litellm_resource() + config = OpenTelemetryConfig.from_env() + result = OpenTelemetry._get_litellm_resource(config) print("RESULT", result) @@ -683,10 +689,8 @@ class TestOpenTelemetry(unittest.TestCase): ) def test_get_litellm_resource_precedence(self): """Test that OTEL_SERVICE_NAME takes precedence over OTEL_RESOURCE_ATTRIBUTES according to OpenTelemetry spec.""" - from litellm.integrations.opentelemetry import _get_litellm_resource - - # This test verifies the OpenTelemetry standard behavior - result = _get_litellm_resource() + config = OpenTelemetryConfig.from_env() + result = OpenTelemetry._get_litellm_resource(config) # Verify the result is a Resource instance from opentelemetry.sdk.resources import Resource diff --git a/tests/test_litellm/integrations/test_prometheus_invalid_key_filtering.py b/tests/test_litellm/integrations/test_prometheus_invalid_key_filtering.py new file mode 100644 index 00000000000..ff433480d5e --- /dev/null +++ b/tests/test_litellm/integrations/test_prometheus_invalid_key_filtering.py @@ -0,0 +1,161 @@ +""" +Unit tests for Prometheus invalid API key request filtering. + +Tests functionality that prevents invalid API key requests (401 status codes) +from being recorded in Prometheus metrics. +""" + +import os +import sys +from unittest.mock import Mock, patch + +import pytest +from prometheus_client import REGISTRY + +sys.path.insert(0, os.path.abspath("../../..")) + +from litellm.integrations.prometheus import PrometheusLogger +from litellm.proxy._types import UserAPIKeyAuth + + +@pytest.fixture(scope="function") +def prometheus_logger(): + """Create a PrometheusLogger instance for testing.""" + collectors = list(REGISTRY._collector_to_names.keys()) + for collector in collectors: + REGISTRY.unregister(collector) + return PrometheusLogger() + + +class ExceptionWithCode: + """Exception-like object with 'code' attribute (ProxyException pattern).""" + def __init__(self, code): + self.code = code + + +class ExceptionWithStatusCode: + """Exception-like object with 'status_code' attribute.""" + def __init__(self, status_code): + self.status_code = status_code + + +class TestExtractStatusCode: + """Test status code extraction from various sources.""" + + @pytest.mark.parametrize("exception_class,code_value,expected", [ + (ExceptionWithCode, "401", 401), + (ExceptionWithStatusCode, 401, 401), + ]) + def test_extract_from_exception(self, prometheus_logger, exception_class, code_value, expected): + exception = exception_class(code_value) + assert prometheus_logger._extract_status_code(exception=exception) == expected + + def test_extract_from_kwargs(self, prometheus_logger): + exception = ExceptionWithCode("401") + assert prometheus_logger._extract_status_code(kwargs={"exception": exception}) == 401 + + def test_extract_from_enum_values(self, prometheus_logger): + enum_values = Mock(status_code="401") + assert prometheus_logger._extract_status_code(enum_values=enum_values) == 401 + + +class TestInvalidAPIKeyDetection: + """Test invalid API key request detection logic.""" + + @pytest.mark.parametrize("status_code,expected", [ + (401, True), + (200, False), + (500, False), + (None, False), + ]) + def test_status_code_detection(self, prometheus_logger, status_code, expected): + assert prometheus_logger._is_invalid_api_key_request(status_code=status_code) == expected + + def test_auth_error_message_detection(self, prometheus_logger): + exception = AssertionError("LiteLLM Virtual Key expected. Received=invalid-key-12345, expected to start with 'sk-'.") + assert prometheus_logger._is_invalid_api_key_request(status_code=None, exception=exception) is True + + def test_non_auth_exception_not_detected(self, prometheus_logger): + exception = ValueError("Some other error") + assert prometheus_logger._is_invalid_api_key_request(status_code=None, exception=exception) is False + + +class TestSkipMetricsValidation: + """Test high-level validation method that orchestrates detection and extraction.""" + + def test_skip_for_401_exception(self, prometheus_logger): + """Test full flow: extraction -> detection -> skip decision.""" + exception = ExceptionWithCode("401") + assert prometheus_logger._should_skip_metrics_for_invalid_key(exception=exception) is True + + def test_skip_for_auth_error_message(self, prometheus_logger): + """Test full flow: exception message -> detection -> skip decision.""" + exception = AssertionError("expected to start with 'sk-'") + assert prometheus_logger._should_skip_metrics_for_invalid_key(exception=exception) is True + + def test_no_skip_for_valid_request(self, prometheus_logger): + assert prometheus_logger._should_skip_metrics_for_invalid_key() is False + + +class TestAsyncHooks: + """Test async hook methods skip metrics for invalid API keys.""" + + @pytest.fixture + def mock_user_api_key(self): + """Create a mock UserAPIKeyAuth object.""" + user_key = Mock(spec=UserAPIKeyAuth) + user_key.api_key = "test-key" + user_key.end_user_id = None + user_key.user_id = None + user_key.user_email = None + user_key.key_alias = None + user_key.team_id = None + user_key.team_alias = None + user_key.request_route = "/test" + return user_key + + @pytest.mark.asyncio + async def test_post_call_failure_hook_skips_401(self, prometheus_logger, mock_user_api_key): + exception = ExceptionWithCode("401") + exception.__class__.__name__ = "ProxyException" + + with patch.object(prometheus_logger, 'litellm_proxy_failed_requests_metric') as mock_failed, \ + patch.object(prometheus_logger, 'litellm_proxy_total_requests_metric') as mock_total: + + await prometheus_logger.async_post_call_failure_hook( + request_data={"model": "test-model"}, + original_exception=exception, + user_api_key_dict=mock_user_api_key + ) + + mock_failed.labels.assert_not_called() + mock_total.labels.assert_not_called() + + @pytest.mark.asyncio + async def test_log_failure_event_skips_401(self, prometheus_logger): + exception = ExceptionWithCode("401") + kwargs = { + "model": "test-model", + "standard_logging_object": { + "metadata": { + "user_api_key_hash": "test-key", + "user_api_key_user_id": "test-user", + }, + "model_group": "test-model", + }, + "exception": exception, + "litellm_params": {}, + } + + with patch.object(prometheus_logger, 'litellm_llm_api_failed_requests_metric') as mock_failed, \ + patch.object(prometheus_logger, 'set_llm_deployment_failure_metrics') as mock_deployment: + + await prometheus_logger.async_log_failure_event( + kwargs=kwargs, + response_obj=None, + start_time=None, + end_time=None + ) + + mock_failed.labels.assert_not_called() + mock_deployment.assert_not_called() diff --git a/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py b/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py index c8fe6efeaa1..4914ec0bfb7 100644 --- a/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py +++ b/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py @@ -1137,3 +1137,94 @@ def test_bedrock_create_bedrock_block_different_document_formats(): assert f"DocumentPDFmessages_" in block["document"]["name"] assert block["document"]["name"].endswith(f"_{format_type}") assert block["document"]["format"] == format_type + + +def test_anthropic_messages_pt_server_tool_use_passthrough(): + """ + Test that anthropic_messages_pt passes through server_tool_use and + tool_search_tool_result blocks in assistant message content. + + These are Anthropic-native content types used for tool search functionality + that need to be preserved when reconstructing multi-turn conversations. + + Fixes: https://github.com/BerriAI/litellm/issues/XXXXX + """ + from litellm.litellm_core_utils.prompt_templates.factory import anthropic_messages_pt + + messages = [ + { + "role": "user", + "content": "I need help with time information." + }, + { + "role": "assistant", + "content": [ + { + "type": "server_tool_use", + "id": "srvtoolu_01ABC123", + "name": "tool_search_tool_regex", + "input": {"query": ".*time.*"} + }, + { + "type": "tool_search_tool_result", + "tool_use_id": "srvtoolu_01ABC123", + "content": { + "type": "tool_search_tool_search_result", + "tool_references": [ + {"type": "tool_reference", "tool_name": "get_time"} + ] + } + }, + { + "type": "text", + "text": "I found the time tool. How can I help you?" + } + ], + }, + { + "role": "user", + "content": "What's the time in New York?" + }, + ] + + result = anthropic_messages_pt( + messages=messages, + model="claude-sonnet-4-5-20250929", + llm_provider="anthropic", + ) + + # Verify we have 3 messages (user, assistant, user) + assert len(result) == 3 + + # Verify the assistant message content + assistant_msg = result[1] + assert assistant_msg["role"] == "assistant" + assert isinstance(assistant_msg["content"], list) + + # Find the different content block types + content_types = [block.get("type") for block in assistant_msg["content"]] + + # Verify server_tool_use block is preserved + assert "server_tool_use" in content_types + server_tool_use_block = next( + b for b in assistant_msg["content"] if b.get("type") == "server_tool_use" + ) + assert server_tool_use_block["id"] == "srvtoolu_01ABC123" + assert server_tool_use_block["name"] == "tool_search_tool_regex" + assert server_tool_use_block["input"] == {"query": ".*time.*"} + + # Verify tool_search_tool_result block is preserved + assert "tool_search_tool_result" in content_types + tool_result_block = next( + b for b in assistant_msg["content"] if b.get("type") == "tool_search_tool_result" + ) + assert tool_result_block["tool_use_id"] == "srvtoolu_01ABC123" + assert tool_result_block["content"]["type"] == "tool_search_tool_search_result" + assert tool_result_block["content"]["tool_references"][0]["tool_name"] == "get_time" + + # Verify text block is also preserved + assert "text" in content_types + text_block = next( + b for b in assistant_msg["content"] if b.get("type") == "text" + ) + assert text_block["text"] == "I found the time tool. How can I help you?" diff --git a/tests/test_litellm/llms/azure/test_azure_common_utils.py b/tests/test_litellm/llms/azure/test_azure_common_utils.py index 3050e8e20d1..a0216be77f7 100644 --- a/tests/test_litellm/llms/azure/test_azure_common_utils.py +++ b/tests/test_litellm/llms/azure/test_azure_common_utils.py @@ -570,6 +570,7 @@ async def test_ensure_initialize_azure_sdk_client_always_used(call_type): or call_type == CallTypes.acreate_container or call_type == CallTypes.adelete_container or call_type == CallTypes.alist_container_files + or call_type == CallTypes.aupload_container_file ): # Skip container call types as they're not supported for Azure (only OpenAI) pytest.skip(f"Skipping {call_type.value} because Azure doesn't support container operations") diff --git a/tests/test_litellm/llms/vertex_ai/test_vertex_ai_common_utils.py b/tests/test_litellm/llms/vertex_ai/test_vertex_ai_common_utils.py index 591b33911dc..b5637db3e52 100644 --- a/tests/test_litellm/llms/vertex_ai/test_vertex_ai_common_utils.py +++ b/tests/test_litellm/llms/vertex_ai/test_vertex_ai_common_utils.py @@ -1175,6 +1175,30 @@ def test_vertex_ai_moonshot_uses_openai_handler(): ) +def test_vertex_ai_zai_uses_openai_handler(): + """ + Ensure ZAI partner models re-use the OpenAI-format handler. + """ + from litellm.llms.vertex_ai.vertex_ai_partner_models.main import ( + VertexAIPartnerModels, + ) + + assert VertexAIPartnerModels.should_use_openai_handler( + "zai-org/glm-4.7-maas" + ) + + +def test_vertex_ai_zai_is_partner_model(): + """ + Ensure ZAI models are detected as Vertex AI partner models. + """ + from litellm.llms.vertex_ai.vertex_ai_partner_models.main import ( + VertexAIPartnerModels, + ) + + assert VertexAIPartnerModels.is_vertex_partner_model("zai-org/glm-4.7-maas") + + def test_build_vertex_schema_empty_properties(): """ Test _build_vertex_schema handles empty properties objects correctly. diff --git a/tests/test_litellm/proxy/db/test_db_spend_update_writer.py b/tests/test_litellm/proxy/db/test_db_spend_update_writer.py index e9d2313ece6..72403b0ba7b 100644 --- a/tests/test_litellm/proxy/db/test_db_spend_update_writer.py +++ b/tests/test_litellm/proxy/db/test_db_spend_update_writer.py @@ -107,7 +107,7 @@ async def test_update_daily_spend_with_null_entity_id(): entity_type="user", entity_id_field="user_id", table_name="litellm_dailyuserspend", - unique_constraint_name="user_id_date_api_key_model_custom_llm_provider", + unique_constraint_name="user_id_date_api_key_model_custom_llm_provider_mcp_namespaced_tool_name_endpoint", ) # Verify that table.upsert was called @@ -115,12 +115,14 @@ async def test_update_daily_spend_with_null_entity_id(): # Verify the where clause contains null entity_id call_args = mock_table.upsert.call_args[1] - where_clause = call_args["where"]["user_id_date_api_key_model_custom_llm_provider"] + where_clause = call_args["where"]["user_id_date_api_key_model_custom_llm_provider_mcp_namespaced_tool_name_endpoint"] assert where_clause["user_id"] is None assert where_clause["date"] == "2024-01-01" assert where_clause["api_key"] == "test-api-key" assert where_clause["model"] == "gpt-4" assert where_clause["custom_llm_provider"] == "openai" + assert where_clause["mcp_namespaced_tool_name"] == "" + assert where_clause["endpoint"] == "" # Verify the create data contains null entity_id create_data = call_args["data"]["create"] @@ -129,6 +131,8 @@ async def test_update_daily_spend_with_null_entity_id(): assert create_data["api_key"] == "test-api-key" assert create_data["model"] == "gpt-4" assert create_data["custom_llm_provider"] == "openai" + assert create_data["mcp_namespaced_tool_name"] == "" + assert create_data["endpoint"] is None assert create_data["prompt_tokens"] == 10 assert create_data["completion_tokens"] == 20 assert create_data["spend"] == 0.1 @@ -171,13 +175,14 @@ async def test_update_daily_spend_sorting(): } upsert_calls.append(call( where={ - "user_id_date_api_key_model_custom_llm_provider": { + "user_id_date_api_key_model_custom_llm_provider_mcp_namespaced_tool_name_endpoint": { "user_id": f"user{i+11}", # user11 ... user60, sorted order "date": "2024-01-01", "api_key": "test-api-key", "model": "gpt-4", "custom_llm_provider": "openai", "mcp_namespaced_tool_name": "", + "endpoint": "", } }, data={ @@ -189,6 +194,7 @@ async def test_update_daily_spend_sorting(): "model_group": None, "mcp_namespaced_tool_name": "", "custom_llm_provider": "openai", + "endpoint": None, "prompt_tokens": 10, "completion_tokens": 20, "spend": 0.1, @@ -203,6 +209,7 @@ async def test_update_daily_spend_sorting(): "api_requests": {"increment": 1}, "successful_requests": {"increment": 1}, "failed_requests": {"increment": 0}, + "endpoint": "", }, }, )) @@ -216,7 +223,7 @@ async def test_update_daily_spend_sorting(): entity_type="user", entity_id_field="user_id", table_name="litellm_dailyuserspend", - unique_constraint_name="user_id_date_api_key_model_custom_llm_provider", + unique_constraint_name="user_id_date_api_key_model_custom_llm_provider_mcp_namespaced_tool_name_endpoint", ) # Verify that table.upsert was called @@ -372,7 +379,7 @@ async def test_update_daily_spend_with_none_values_in_sorting_fields(): entity_type="user", entity_id_field="user_id", table_name="litellm_dailyuserspend", - unique_constraint_name="user_id_date_api_key_model_custom_llm_provider", + unique_constraint_name="user_id_date_api_key_model_custom_llm_provider_mcp_namespaced_tool_name_endpoint", ) # Verify that table.upsert was called (should be called 5 times, once for each transaction) @@ -588,7 +595,7 @@ async def test_add_spend_log_transaction_to_daily_org_transaction_injects_org_id update_dict = call_args["update"] assert len(update_dict) == 1 for key, transaction in update_dict.items(): - assert key == f"{org_id}_2024-01-01_test-key_gpt-4_openai" + assert key == f"{org_id}_2024-01-01_test-key_gpt-4_openai_" assert transaction["organization_id"] == org_id assert transaction["date"] == "2024-01-01" assert transaction["api_key"] == "test-key" @@ -665,7 +672,7 @@ async def test_add_spend_log_transaction_to_daily_end_user_transaction_injects_e update_dict = call_args["update"] assert len(update_dict) == 1 for key, transaction in update_dict.items(): - assert key == f"{end_user_id}_2024-01-01_test-key_gpt-4_openai" + assert key == f"{end_user_id}_2024-01-01_test-key_gpt-4_openai_" assert transaction["end_user_id"] == end_user_id assert transaction["date"] == "2024-01-01" assert transaction["api_key"] == "test-key" @@ -741,7 +748,7 @@ async def test_add_spend_log_transaction_to_daily_agent_transaction_injects_agen update_dict = call_args["update"] assert len(update_dict) == 1 for key, transaction in update_dict.items(): - assert key == f"{agent_id}_2024-01-01_test-key_gpt-4_openai" + assert key == f"{agent_id}_2024-01-01_test-key_gpt-4_openai_" assert transaction["agent_id"] == agent_id assert transaction["date"] == "2024-01-01" assert transaction["api_key"] == "test-key" @@ -780,4 +787,55 @@ async def test_add_spend_log_transaction_to_daily_agent_transaction_skips_when_a prisma_client=mock_prisma, ) - writer.daily_agent_spend_update_queue.add_update.assert_not_called() \ No newline at end of file + writer.daily_agent_spend_update_queue.add_update.assert_not_called() + + +@pytest.mark.asyncio +async def test_endpoint_field_is_correctly_mapped_from_call_type(): + """ + Test that the endpoint field is correctly mapped from call_type using ROUTE_ENDPOINT_MAPPING. + Verifies that when call_type is provided, the endpoint is set in the transaction and included in the key. + """ + writer = DBSpendUpdateWriter() + mock_prisma = MagicMock() + mock_prisma.get_request_status = MagicMock(return_value="success") + + payload = { + "request_id": "req-endpoint-test", + "user": "test-user", + "call_type": "acompletion", # Maps to "/chat/completions" + "startTime": "2024-01-01T12:00:00", + "api_key": "test-key", + "model": "gpt-4", + "custom_llm_provider": "openai", + "model_group": "gpt-4-group", + "prompt_tokens": 100, + "completion_tokens": 50, + "spend": 0.15, + "metadata": '{"usage_object": {}}', + } + + writer.daily_spend_update_queue.add_update = AsyncMock() + + await writer.add_spend_log_transaction_to_daily_user_transaction( + payload=payload, + prisma_client=mock_prisma, + ) + + writer.daily_spend_update_queue.add_update.assert_called_once() + + call_args = writer.daily_spend_update_queue.add_update.call_args[1] + update_dict = call_args["update"] + assert len(update_dict) == 1 + + for key, transaction in update_dict.items(): + # Verify endpoint is included in the key + assert key == f"test-user_2024-01-01_test-key_gpt-4_openai_/chat/completions" + + # Verify endpoint is set in the transaction + assert transaction["endpoint"] == "/chat/completions" + assert transaction["user_id"] == "test-user" + assert transaction["date"] == "2024-01-01" + assert transaction["api_key"] == "test-key" + assert transaction["model"] == "gpt-4" + assert transaction["custom_llm_provider"] == "openai" \ No newline at end of file diff --git a/tests/test_litellm/proxy/management_endpoints/test_common_daily_activity.py b/tests/test_litellm/proxy/management_endpoints/test_common_daily_activity.py index bbdc4b1edf4..93457631d2d 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_common_daily_activity.py +++ b/tests/test_litellm/proxy/management_endpoints/test_common_daily_activity.py @@ -12,6 +12,7 @@ from litellm.proxy.management_endpoints.common_daily_activity import ( _is_user_agent_tag, compute_tag_metadata_totals, get_daily_activity, + get_daily_activity_aggregated, ) @@ -124,3 +125,86 @@ def test_compute_tag_metadata_totals(): result = compute_tag_metadata_totals([]) assert result.spend == 0.0 assert result.prompt_tokens == 0 + + +@pytest.mark.asyncio +async def test_get_daily_activity_aggregated_with_endpoint_breakdown(): + """Test that endpoint breakdown is included in aggregated daily activity.""" + # Mock PrismaClient + mock_prisma = MagicMock() + mock_prisma.db = MagicMock() + + # Create mock records with endpoint fields + class MockRecord: + def __init__(self, date, endpoint, api_key, model, spend, prompt_tokens, completion_tokens): + self.date = date + self.endpoint = endpoint + self.api_key = api_key + self.model = model + self.model_group = None + self.custom_llm_provider = "openai" + self.mcp_namespaced_tool_name = None + self.spend = spend + self.prompt_tokens = prompt_tokens + self.completion_tokens = completion_tokens + self.total_tokens = prompt_tokens + completion_tokens + self.cache_read_input_tokens = 0 + self.cache_creation_input_tokens = 0 + self.api_requests = 1 + self.successful_requests = 1 + self.failed_requests = 0 + + mock_records = [ + MockRecord("2024-01-01", "/v1/chat/completions", "key-1", "gpt-4", 10.0, 100, 50), + MockRecord("2024-01-01", "/v1/chat/completions", "key-1", "gpt-4", 5.0, 50, 25), + MockRecord("2024-01-01", "/v1/embeddings", "key-2", "text-embedding-ada-002", 3.0, 30, 0), + ] + + # Mock the table methods + mock_table = MagicMock() + mock_table.find_many = AsyncMock(return_value=mock_records) + mock_prisma.db.litellm_dailyuserspend = mock_table + mock_prisma.db.litellm_verificationtoken = MagicMock() + mock_prisma.db.litellm_verificationtoken.find_many = AsyncMock(return_value=[]) + + # Call the function + result = await get_daily_activity_aggregated( + prisma_client=mock_prisma, + table_name="litellm_dailyuserspend", + entity_id_field="user_id", + entity_id=None, + entity_metadata_field=None, + start_date="2024-01-01", + end_date="2024-01-01", + model=None, + api_key=None, + ) + + # Verify the results + assert len(result.results) == 1 + daily_data = result.results[0] + assert daily_data.date.strftime("%Y-%m-%d") == "2024-01-01" + + # Verify endpoint breakdown exists + assert "endpoints" in daily_data.breakdown.model_fields + assert len(daily_data.breakdown.endpoints) == 2 + + # Verify /v1/chat/completions endpoint breakdown + assert "/v1/chat/completions" in daily_data.breakdown.endpoints + chat_endpoint = daily_data.breakdown.endpoints["/v1/chat/completions"] + assert chat_endpoint.metrics.spend == 15.0 # 10.0 + 5.0 + assert chat_endpoint.metrics.prompt_tokens == 150 # 100 + 50 + assert chat_endpoint.metrics.completion_tokens == 75 # 50 + 25 + + # Verify /v1/embeddings endpoint breakdown + assert "/v1/embeddings" in daily_data.breakdown.endpoints + embeddings_endpoint = daily_data.breakdown.endpoints["/v1/embeddings"] + assert embeddings_endpoint.metrics.spend == 3.0 + assert embeddings_endpoint.metrics.prompt_tokens == 30 + assert embeddings_endpoint.metrics.completion_tokens == 0 + + # Verify API key breakdowns within endpoints + assert "key-1" in chat_endpoint.api_key_breakdown + assert chat_endpoint.api_key_breakdown["key-1"].metrics.spend == 15.0 + assert "key-2" in embeddings_endpoint.api_key_breakdown + assert embeddings_endpoint.api_key_breakdown["key-2"].metrics.spend == 3.0 diff --git a/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py index 33fa7fc7bde..c9a10e3c4d0 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py @@ -3,6 +3,7 @@ import os import sys import pytest +import yaml from fastapi.testclient import TestClient sys.path.insert( @@ -3642,3 +3643,152 @@ async def test_update_key_negative_max_budget(): # Should not raise any errors at model level request = UpdateKeyRequest(key="test-key", max_budget=-5.0) assert request.max_budget == -5.0 + + +@pytest.mark.asyncio +async def test_generate_key_with_router_settings(monkeypatch): + """ + Test that /key/generate correctly handles router_settings by: + 1. Accepting router_settings as a dict parameter + 2. Serializing router_settings to JSON when saving to database + 3. Storing router_settings in the key record + """ + mock_prisma_client = AsyncMock() + mock_prisma_client.jsonify_object = lambda data: data + + # Mock prisma_client.insert_data for both user and key tables + async def _insert_data_side_effect(*args, **kwargs): + table_name = kwargs.get("table_name") + if table_name == "user": + return MagicMock(models=[], spend=0) + elif table_name == "key": + return MagicMock( + token="hashed_token_router", + litellm_budget_table=None, + object_permission=None, + ) + return MagicMock() + + mock_prisma_client.insert_data = AsyncMock(side_effect=_insert_data_side_effect) + mock_prisma_client.db = MagicMock() + mock_prisma_client.db.litellm_verificationtoken = MagicMock() + mock_prisma_client.db.litellm_verificationtoken.find_unique = AsyncMock( + return_value=None + ) + mock_prisma_client.db.litellm_verificationtoken.find_many = AsyncMock( + return_value=[] + ) + mock_prisma_client.db.litellm_verificationtoken.count = AsyncMock(return_value=0) + + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", mock_prisma_client) + + from litellm.proxy._types import GenerateKeyRequest, LitellmUserRoles + from litellm.proxy.auth.user_api_key_auth import UserAPIKeyAuth + from litellm.proxy.management_endpoints.key_management_endpoints import ( + generate_key_fn, + ) + + # Test router_settings with sample data + # Using valid UpdateRouterConfig fields (retry_policy is not a valid field, + # but model_group_retry_policy is, which also tests nested dict serialization) + router_settings_data = { + "routing_strategy": "usage-based", + "num_retries": 3, + "model_group_retry_policy": {"max_retries": 5}, + } + + request_data = GenerateKeyRequest( + models=["gpt-4"], + router_settings=router_settings_data, + ) + + await generate_key_fn( + data=request_data, + user_api_key_dict=UserAPIKeyAuth( + user_role=LitellmUserRoles.PROXY_ADMIN, + api_key="sk-1234", + user_id="user-router-1", + ), + ) + + # Verify key insertion was called + assert mock_prisma_client.insert_data.call_count >= 1 + key_insert_calls = [ + call.kwargs + for call in mock_prisma_client.insert_data.call_args_list + if call.kwargs.get("table_name") == "key" + ] + assert len(key_insert_calls) >= 1 + key_data = key_insert_calls[0]["data"] + + # Verify router_settings is present + assert "router_settings" in key_data + + # router_settings should be present in the data passed to insert_data + # The code uses safe_dumps to serialize router_settings, so it will be a JSON string + router_settings_value = key_data["router_settings"] + + # Get the actual settings value for comparison + # The code uses safe_dumps to serialize and yaml.safe_load to deserialize + if isinstance(router_settings_value, str): + # If it's a JSON string (from safe_dumps), deserialize it using json.loads + # (safe_dumps produces JSON, and json.loads is the correct way to deserialize it) + actual_settings = json.loads(router_settings_value) + elif isinstance(router_settings_value, dict): + # If it's still a dict, use it directly + actual_settings = router_settings_value + else: + raise AssertionError( + f"router_settings should be str or dict, got {type(router_settings_value)}" + ) + + # Verify router_settings matches input (regardless of serialization state) + assert actual_settings == router_settings_data + + +@pytest.mark.asyncio +async def test_update_key_with_router_settings(monkeypatch): + """ + Test that /key/update correctly handles router_settings by: + 1. Accepting router_settings as a dict parameter + 2. Serializing router_settings to JSON when updating database + 3. Updating router_settings in the key record + """ + from litellm.proxy._types import LiteLLM_VerificationToken, UpdateKeyRequest + from litellm.proxy.management_endpoints.key_management_endpoints import ( + prepare_key_update_data, + ) + + # Mock existing key + existing_key = LiteLLM_VerificationToken( + token="test-token-router", + key_alias="test-key", + models=["gpt-3.5-turbo"], + user_id="test-user", + team_id=None, + auto_rotate=False, + rotation_interval=None, + metadata={}, + ) + + # Test updating router_settings + router_settings_data = { + "routing_strategy": "latency-based", + "num_retries": 2, + } + + update_request = UpdateKeyRequest( + key="test-token-router", router_settings=router_settings_data + ) + + result = await prepare_key_update_data( + data=update_request, existing_key_row=existing_key + ) + + # Verify router_settings is serialized to JSON string + assert "router_settings" in result + assert isinstance(result["router_settings"], str) + + # Verify router_settings can be deserialized and matches input + deserialized_settings = json.loads(result["router_settings"]) + assert deserialized_settings == router_settings_data diff --git a/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py index 57064586afb..e296066b998 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py @@ -4393,3 +4393,162 @@ async def test_new_team_positive_budgets_accepted(): ) assert request.max_budget == 100.0 assert request.team_member_budget == 50.0 + + +@pytest.mark.asyncio +async def test_new_team_with_router_settings(mock_db_client, mock_admin_auth): + """ + Test that /team/new correctly handles router_settings by: + 1. Accepting router_settings as a dict parameter + 2. Serializing router_settings to JSON when saving to database + 3. Storing router_settings in the team record + """ + # Configure mocked prisma client + mock_db_client.jsonify_team_object = lambda db_data: db_data + mock_db_client.get_data = AsyncMock(return_value=None) + mock_db_client.update_data = AsyncMock(return_value=MagicMock()) + mock_db_client.db = MagicMock() + + # Mock model table creation + mock_db_client.db.litellm_modeltable = MagicMock() + mock_db_client.db.litellm_modeltable.create = AsyncMock( + return_value=MagicMock(id="model123") + ) + + # Capture team table creation + team_create_result = MagicMock( + team_id="team-router-456", + ) + team_create_result.model_dump.return_value = { + "team_id": "team-router-456", + } + mock_team_create = AsyncMock(return_value=team_create_result) + mock_team_count = AsyncMock(return_value=0) + mock_db_client.db.litellm_teamtable = MagicMock() + mock_db_client.db.litellm_teamtable.create = mock_team_create + mock_db_client.db.litellm_teamtable.count = mock_team_count + mock_db_client.db.litellm_teamtable.update = AsyncMock( + return_value=team_create_result + ) + + # Mock user table + mock_db_client.db.litellm_usertable = MagicMock() + mock_db_client.db.litellm_usertable.update = AsyncMock(return_value=MagicMock()) + + from fastapi import Request + + from litellm.proxy._types import NewTeamRequest + from litellm.proxy.management_endpoints.team_endpoints import new_team + + # Test router_settings with sample data + router_settings_data = { + "routing_strategy": "usage-based", + "num_retries": 3, + "retry_policy": {"max_retries": 5}, + } + + # Build request with router_settings + team_request = NewTeamRequest( + team_alias="my-team-router", + router_settings=router_settings_data, + ) + + dummy_request = MagicMock(spec=Request) + + # Execute the endpoint function + await new_team( + data=team_request, + http_request=dummy_request, + user_api_key_dict=mock_admin_auth, + ) + + # Verify team creation was called + assert mock_team_create.call_count == 1 + created_team_kwargs = mock_team_create.call_args.kwargs + team_data = created_team_kwargs["data"] + + # Verify router_settings is serialized to JSON string + assert "router_settings" in team_data + assert isinstance(team_data["router_settings"], str) + + # Verify router_settings can be deserialized and matches input + deserialized_settings = json.loads(team_data["router_settings"]) + assert deserialized_settings == router_settings_data + + +@pytest.mark.asyncio +async def test_update_team_with_router_settings(mock_db_client, mock_admin_auth): + """ + Test that /team/update correctly handles router_settings by: + 1. Accepting router_settings as a dict parameter + 2. Serializing router_settings to JSON when updating database + 3. Updating router_settings in the team record + """ + # Configure mocked prisma client + mock_db_client.jsonify_team_object = lambda db_data: db_data + mock_db_client.db = MagicMock() + + # Mock existing team row + existing_team_mock = MagicMock() + existing_team_mock.team_id = "team-router-update-789" + existing_team_mock.organization_id = None + existing_team_mock.models = [] + existing_team_mock.members_with_roles = [] + existing_team_mock.model_dump.return_value = { + "team_id": "team-router-update-789", + "organization_id": None, + "models": [], + "members_with_roles": [], + } + + # Mock team table find_unique and update + updated_team_result = MagicMock( + team_id="team-router-update-789", + ) + updated_team_result.model_dump.return_value = { + "team_id": "team-router-update-789", + } + mock_team_find_unique = AsyncMock(return_value=existing_team_mock) + mock_team_update = AsyncMock(return_value=updated_team_result) + mock_db_client.db.litellm_teamtable = MagicMock() + mock_db_client.db.litellm_teamtable.find_unique = mock_team_find_unique + mock_db_client.db.litellm_teamtable.update = mock_team_update + + from fastapi import Request + + from litellm.proxy._types import UpdateTeamRequest + from litellm.proxy.management_endpoints.team_endpoints import update_team + + # Test router_settings with updated data + router_settings_data = { + "routing_strategy": "latency-based", + "num_retries": 2, + } + + # Build update request with router_settings + team_update_request = UpdateTeamRequest( + team_id="team-router-update-789", + router_settings=router_settings_data, + ) + + dummy_request = MagicMock(spec=Request) + + # Execute the endpoint function + await update_team( + data=team_update_request, + http_request=dummy_request, + user_api_key_dict=mock_admin_auth, + ) + + # Verify team update was called + assert mock_team_update.call_count == 1 + updated_team_kwargs = mock_team_update.call_args.kwargs + team_data = updated_team_kwargs["data"] + + # Verify router_settings is serialized to JSON string + assert "router_settings" in team_data + assert isinstance(team_data["router_settings"], str) + + # Verify router_settings can be deserialized and matches input + deserialized_settings = json.loads(team_data["router_settings"]) + assert deserialized_settings == router_settings_data diff --git a/tests/test_litellm/test_responses_id_security.py b/tests/test_litellm/test_responses_id_security.py index e72a09ee0d3..6b04479326e 100644 --- a/tests/test_litellm/test_responses_id_security.py +++ b/tests/test_litellm/test_responses_id_security.py @@ -136,10 +136,9 @@ class TestEncryptResponseId: ) with patch( - "litellm.proxy.hooks.responses_id_security.encrypt_value_helper" - ) as mock_encrypt: - mock_encrypt.return_value = "encrypted_value_456" - + "litellm.proxy.common_utils.encrypt_decrypt_utils._get_salt_key", + return_value="test-salt-key" + ): with patch.object( responses_id_security, "_get_signing_key", return_value="test-key" ): @@ -148,6 +147,8 @@ class TestEncryptResponseId: ) assert result.id.startswith("resp_") + # The encrypted ID should be different from the original + assert result.id != "resp_456" class TestCheckUserAccessToResponseId: diff --git a/ui/litellm-dashboard/e2e_tests/tests/modelsPage/addModel.spec.ts b/ui/litellm-dashboard/e2e_tests/tests/modelsPage/addModel.spec.ts index 5fa11a98ef6..c0619cfa845 100644 --- a/ui/litellm-dashboard/e2e_tests/tests/modelsPage/addModel.spec.ts +++ b/ui/litellm-dashboard/e2e_tests/tests/modelsPage/addModel.spec.ts @@ -5,7 +5,7 @@ test.describe("Add Model", () => { test.use({ storageState: ADMIN_STORAGE_PATH }); test("Able to see all models for a specific provider in the model dropdown", async ({ page }) => { - await page.goto("http://localhost:4000/ui"); + await page.goto("/ui"); await page.getByText("Models + Endpoints").click(); await page.getByRole("tab", { name: "Add Model" }).click(); diff --git a/ui/litellm-dashboard/e2e_tests/tests/navigation/sidebar.spec.ts b/ui/litellm-dashboard/e2e_tests/tests/navigation/sidebar.spec.ts index 6801f891e87..c90be698ae1 100644 --- a/ui/litellm-dashboard/e2e_tests/tests/navigation/sidebar.spec.ts +++ b/ui/litellm-dashboard/e2e_tests/tests/navigation/sidebar.spec.ts @@ -24,7 +24,7 @@ for (const { role, storage } of roles) { test.use({ storageState: storage }); test("can see and navigate all sidebar buttons", async ({ page }) => { - await page.goto("http://localhost:4000/ui"); + await page.goto("/ui"); for (const button of sidebarButtons[role as keyof typeof sidebarButtons]) { const tab = page.getByRole("menuitem", { name: button }); await expect(tab).toBeVisible(); diff --git a/ui/litellm-dashboard/e2e_tests/tests/settings/adminSettings.spec.ts b/ui/litellm-dashboard/e2e_tests/tests/settings/adminSettings.spec.ts new file mode 100644 index 00000000000..f61532b05a5 --- /dev/null +++ b/ui/litellm-dashboard/e2e_tests/tests/settings/adminSettings.spec.ts @@ -0,0 +1,14 @@ +import { test, expect } from "@playwright/test"; +import { ADMIN_STORAGE_PATH } from "../../constants"; + +test.describe("Add Model", () => { + test.use({ storageState: ADMIN_STORAGE_PATH }); + + test("admin settings test", async ({ page }) => { + await page.goto("/ui"); + await page.getByRole("menuitem", { name: /Settings/ }).click(); + await page.getByRole("menuitem", { name: /Admin Settings/ }).click(); + await page.getByRole("tab", { name: "UI Settings" }).click(); + await expect(page.getByText("Configuration for UI-specific")).toBeVisible(); + }); +}); diff --git a/ui/litellm-dashboard/e2e_tests/tests/users/searchUsers.spec.ts b/ui/litellm-dashboard/e2e_tests/tests/users/searchUsers.spec.ts index 5873bb3125c..01c1e68f1ee 100644 --- a/ui/litellm-dashboard/e2e_tests/tests/users/searchUsers.spec.ts +++ b/ui/litellm-dashboard/e2e_tests/tests/users/searchUsers.spec.ts @@ -1,9 +1,10 @@ import { test, expect, Page } from "@playwright/test"; +import { ADMIN_STORAGE_PATH } from "../../constants"; test.describe("Internal Users Search", () => { - test.use({ storageState: "admin.storageState.json" }); + test.use({ storageState: ADMIN_STORAGE_PATH }); async function goToInternalUsers(page: Page) { - await page.goto("http://localhost:4000/ui"); + await page.goto("/ui"); const tab = page.getByRole("menuitem", { name: "Internal User" }); await expect(tab).toBeVisible(); diff --git a/ui/litellm-dashboard/e2e_tests/tests/users/viewInternalUsers.spec.ts b/ui/litellm-dashboard/e2e_tests/tests/users/viewInternalUsers.spec.ts index 65797028edc..4dfd79c9dff 100644 --- a/ui/litellm-dashboard/e2e_tests/tests/users/viewInternalUsers.spec.ts +++ b/ui/litellm-dashboard/e2e_tests/tests/users/viewInternalUsers.spec.ts @@ -1,10 +1,11 @@ import { test, expect, Page } from "@playwright/test"; +import { ADMIN_STORAGE_PATH } from "../../constants"; test.describe("Internal Users Page", () => { - test.use({ storageState: "admin.storageState.json" }); + test.use({ storageState: ADMIN_STORAGE_PATH }); async function goToInternalUsers(page: Page) { - await page.goto("http://localhost:4000/ui"); + await page.goto("/ui"); const internalUserTab = page.getByRole("menuitem", { name: "Internal User" }); await expect(internalUserTab).toBeVisible(); diff --git a/ui/litellm-dashboard/src/components/Settings/AdminSettings/SSOSettings/Modals/BaseSSOSettingsForm.test.tsx b/ui/litellm-dashboard/src/components/Settings/AdminSettings/SSOSettings/Modals/BaseSSOSettingsForm.test.tsx new file mode 100644 index 00000000000..a885bffa710 --- /dev/null +++ b/ui/litellm-dashboard/src/components/Settings/AdminSettings/SSOSettings/Modals/BaseSSOSettingsForm.test.tsx @@ -0,0 +1,185 @@ +import { Form } from "antd"; +import { act, fireEvent, screen, waitFor } from "@testing-library/react"; +import { renderWithProviders } from "../../../../../../tests/test-utils"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import BaseSSOSettingsForm, { renderProviderFields } from "./BaseSSOSettingsForm"; + +describe("BaseSSOSettingsForm", () => { + afterEach(() => { + vi.clearAllMocks(); + }); + + it("should render", () => { + const TestWrapper = () => { + const [form] = Form.useForm(); + const handleSubmit = vi.fn(); + + return ; + }; + + renderWithProviders(); + + expect(screen.getByText("SSO Provider")).toBeInTheDocument(); + expect(screen.getByText("Proxy Admin Email")).toBeInTheDocument(); + expect(screen.getByText("Proxy Base URL")).toBeInTheDocument(); + }); + + it("should render provider fields when provider is selected", async () => { + const TestWrapper = () => { + const [form] = Form.useForm(); + const handleSubmit = vi.fn(); + + return ; + }; + + renderWithProviders(); + + const providerSelect = screen.getByLabelText("SSO Provider"); + await act(async () => { + fireEvent.mouseDown(providerSelect); + }); + + await waitFor(() => { + const googleOption = screen.getByText(/google sso/i); + fireEvent.click(googleOption); + }); + + await waitFor(() => { + expect(screen.getByText("Google Client ID")).toBeInTheDocument(); + expect(screen.getByText("Google Client Secret")).toBeInTheDocument(); + }); + }); + + it("should show role mappings fields for okta provider", async () => { + const TestWrapper = () => { + const [form] = Form.useForm(); + const handleSubmit = vi.fn(); + + return ; + }; + + renderWithProviders(); + + const providerSelect = screen.getByLabelText("SSO Provider"); + await act(async () => { + fireEvent.mouseDown(providerSelect); + }); + + await waitFor(() => { + const oktaOption = screen.getByText(/okta/i); + fireEvent.click(oktaOption); + }); + + await waitFor(() => { + expect(screen.getByText("Use Role Mappings")).toBeInTheDocument(); + }); + }); + + it("should validate proxy base url format", async () => { + const TestWrapper = () => { + const [form] = Form.useForm(); + const handleSubmit = vi.fn(); + + return ; + }; + + renderWithProviders(); + + const urlInput = screen.getByPlaceholderText("https://example.com"); + await act(async () => { + fireEvent.change(urlInput, { target: { value: "invalid-url" } }); + fireEvent.blur(urlInput); + }); + + await waitFor(() => { + expect(screen.getByText(/URL must start with http:\/\/ or https:\/\//i)).toBeInTheDocument(); + }); + }); + + it("should validate proxy base url trailing slash", async () => { + const TestWrapper = () => { + const [form] = Form.useForm(); + const handleSubmit = vi.fn(); + + return ; + }; + + renderWithProviders(); + + const urlInput = screen.getByPlaceholderText("https://example.com"); + await act(async () => { + fireEvent.change(urlInput, { target: { value: "https://example.com/" } }); + fireEvent.blur(urlInput); + }); + + await waitFor(() => { + expect(screen.getByText(/URL must not end with a trailing slash/i)).toBeInTheDocument(); + }); + }); + + it("should show role mappings fields when use_role_mappings is checked for generic provider", async () => { + const TestWrapper = () => { + const [form] = Form.useForm(); + const handleSubmit = vi.fn(); + + return ; + }; + + renderWithProviders(); + + const providerSelect = screen.getByLabelText("SSO Provider"); + await act(async () => { + fireEvent.mouseDown(providerSelect); + }); + + await waitFor(() => { + const genericOption = screen.getByText(/generic sso/i); + fireEvent.click(genericOption); + }); + + await waitFor(() => { + expect(screen.getByText("Use Role Mappings")).toBeInTheDocument(); + }); + + const checkbox = screen.getByLabelText("Use Role Mappings"); + await act(async () => { + fireEvent.click(checkbox); + }); + + await waitFor(() => { + expect(screen.getByText("Group Claim")).toBeInTheDocument(); + expect(screen.getByText("Default Role")).toBeInTheDocument(); + }); + }); +}); + +describe("renderProviderFields", () => { + it("should return null for unknown provider", () => { + const result = renderProviderFields("unknown"); + expect(result).toBeNull(); + }); + + it("should return fields for google provider", () => { + const result = renderProviderFields("google"); + expect(result).not.toBeNull(); + expect(result?.length).toBe(2); + }); + + it("should return fields for microsoft provider", () => { + const result = renderProviderFields("microsoft"); + expect(result).not.toBeNull(); + expect(result?.length).toBe(3); + }); + + it("should return fields for okta provider", () => { + const result = renderProviderFields("okta"); + expect(result).not.toBeNull(); + expect(result?.length).toBe(5); + }); + + it("should return fields for generic provider", () => { + const result = renderProviderFields("generic"); + expect(result).not.toBeNull(); + expect(result?.length).toBe(5); + }); +}); diff --git a/ui/litellm-dashboard/src/components/team/available_teams.test.tsx b/ui/litellm-dashboard/src/components/team/available_teams.test.tsx new file mode 100644 index 00000000000..af2a0247b47 --- /dev/null +++ b/ui/litellm-dashboard/src/components/team/available_teams.test.tsx @@ -0,0 +1,138 @@ +import * as networking from "@/components/networking"; +import { act, fireEvent, screen, waitFor } from "@testing-library/react"; +import { renderWithProviders } from "../../../tests/test-utils"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import AvailableTeamsPanel from "./available_teams"; + +vi.mock("@/components/networking", () => ({ + availableTeamListCall: vi.fn(), + teamMemberAddCall: vi.fn(), +})); + +describe("AvailableTeamsPanel", () => { + afterEach(() => { + vi.clearAllMocks(); + }); + + it("should render", async () => { + vi.mocked(networking.availableTeamListCall).mockResolvedValue([]); + + renderWithProviders(); + + await waitFor(() => { + expect(screen.getByText("Team Name")).toBeInTheDocument(); + }); + }); + + it("should display teams when available", async () => { + const mockTeams = [ + { + team_id: "team-1", + team_alias: "Test Team 1", + description: "Test Description 1", + models: ["gpt-4"], + members_with_roles: [{ user_id: "user-1", user_email: "user1@test.com", role: "admin" }], + }, + { + team_id: "team-2", + team_alias: "Test Team 2", + description: "Test Description 2", + models: [], + members_with_roles: [{ user_id: "user-2", user_email: "user2@test.com", role: "user" }], + }, + ]; + + vi.mocked(networking.availableTeamListCall).mockResolvedValue(mockTeams); + + renderWithProviders(); + + await waitFor(() => { + expect(screen.getByText("Test Team 1")).toBeInTheDocument(); + expect(screen.getByText("Test Team 2")).toBeInTheDocument(); + }); + }); + + it("should display empty state when no teams are available", async () => { + vi.mocked(networking.availableTeamListCall).mockResolvedValue([]); + + renderWithProviders(); + + await waitFor(() => { + expect(screen.getByText("No available teams to join")).toBeInTheDocument(); + }); + }); + + it("should call teamMemberAddCall when join team button is clicked", async () => { + const mockTeams = [ + { + team_id: "team-1", + team_alias: "Test Team 1", + description: "Test Description 1", + models: ["gpt-4"], + members_with_roles: [{ user_id: "user-1", user_email: "user1@test.com", role: "admin" }], + }, + ]; + + vi.mocked(networking.availableTeamListCall).mockResolvedValue(mockTeams); + vi.mocked(networking.teamMemberAddCall).mockResolvedValue({}); + + renderWithProviders(); + + await waitFor(() => { + expect(screen.getByText("Test Team 1")).toBeInTheDocument(); + }); + + const joinButtons = screen.getAllByRole("button", { name: /join team/i }); + await act(async () => { + fireEvent.click(joinButtons[0]); + }); + + await waitFor(() => { + expect(networking.teamMemberAddCall).toHaveBeenCalledWith("token-123", "team-1", { + user_id: "user-123", + role: "user", + }); + }); + }); + + it("should display All Proxy Models badge when team has no models", async () => { + const mockTeams = [ + { + team_id: "team-1", + team_alias: "Test Team 1", + description: "Test Description 1", + models: [], + members_with_roles: [{ user_id: "user-1", user_email: "user1@test.com", role: "admin" }], + }, + ]; + + vi.mocked(networking.availableTeamListCall).mockResolvedValue(mockTeams); + + renderWithProviders(); + + await waitFor(() => { + expect(screen.getByText("All Proxy Models")).toBeInTheDocument(); + }); + }); + + it("should display model badges when team has models", async () => { + const mockTeams = [ + { + team_id: "team-1", + team_alias: "Test Team 1", + description: "Test Description 1", + models: ["gpt-4", "gpt-3.5-turbo"], + members_with_roles: [{ user_id: "user-1", user_email: "user1@test.com", role: "admin" }], + }, + ]; + + vi.mocked(networking.availableTeamListCall).mockResolvedValue(mockTeams); + + renderWithProviders(); + + await waitFor(() => { + expect(screen.getByText("gpt-4")).toBeInTheDocument(); + expect(screen.getByText("gpt-3.5-turbo")).toBeInTheDocument(); + }); + }); +}); diff --git a/ui/litellm-dashboard/src/components/team/member_permissions.test.tsx b/ui/litellm-dashboard/src/components/team/member_permissions.test.tsx new file mode 100644 index 00000000000..cee2b8d587a --- /dev/null +++ b/ui/litellm-dashboard/src/components/team/member_permissions.test.tsx @@ -0,0 +1,160 @@ +import * as networking from "@/components/networking"; +import { act, fireEvent, screen, waitFor } from "@testing-library/react"; +import { renderWithProviders } from "../../../tests/test-utils"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import MemberPermissions from "./member_permissions"; + +vi.mock("@/components/networking", () => ({ + getTeamPermissionsCall: vi.fn(), + teamPermissionsUpdateCall: vi.fn(), +})); + +describe("MemberPermissions", () => { + afterEach(() => { + vi.clearAllMocks(); + }); + + it("should render", async () => { + vi.mocked(networking.getTeamPermissionsCall).mockResolvedValue({ + all_available_permissions: ["/key/generate", "/key/list"], + team_member_permissions: ["/key/generate"], + }); + + renderWithProviders(); + + await waitFor(() => { + expect(screen.getByText("Member Permissions")).toBeInTheDocument(); + }); + }); + + it("should display permissions table when permissions are available", async () => { + vi.mocked(networking.getTeamPermissionsCall).mockResolvedValue({ + all_available_permissions: ["/key/generate", "/key/list"], + team_member_permissions: ["/key/generate"], + }); + + renderWithProviders(); + + await waitFor(() => { + expect(screen.getByText("Method")).toBeInTheDocument(); + expect(screen.getByText("Endpoint")).toBeInTheDocument(); + expect(screen.getByText("Description")).toBeInTheDocument(); + expect(screen.getByText("Allow Access")).toBeInTheDocument(); + }); + }); + + it("should display empty state when no permissions are available", async () => { + vi.mocked(networking.getTeamPermissionsCall).mockResolvedValue({ + all_available_permissions: [], + team_member_permissions: [], + }); + + renderWithProviders(); + + await waitFor(() => { + expect(screen.getByText("No permissions available")).toBeInTheDocument(); + }); + }); + + it("should save permissions when save button is clicked", async () => { + vi.mocked(networking.getTeamPermissionsCall).mockResolvedValue({ + all_available_permissions: ["/key/generate", "/key/list"], + team_member_permissions: ["/key/generate"], + }); + vi.mocked(networking.teamPermissionsUpdateCall).mockResolvedValue({}); + + renderWithProviders(); + + await waitFor(() => { + expect(screen.getByText("Member Permissions")).toBeInTheDocument(); + }); + + const checkboxes = screen.getAllByRole("checkbox"); + const unselectedCheckbox = checkboxes.find((cb) => !(cb as HTMLInputElement).checked); + + if (unselectedCheckbox) { + await act(async () => { + fireEvent.click(unselectedCheckbox); + }); + + await waitFor(() => { + const saveButton = screen.getByRole("button", { name: /save changes/i }); + expect(saveButton).toBeInTheDocument(); + }); + + const saveButton = screen.getByRole("button", { name: /save changes/i }); + await act(async () => { + fireEvent.click(saveButton); + }); + + await waitFor(() => { + expect(networking.teamPermissionsUpdateCall).toHaveBeenCalledWith( + "token-123", + "team-123", + expect.arrayContaining(["/key/generate", "/key/list"]), + ); + }); + } + }); + + it("should not show save button when canEditTeam is false", async () => { + vi.mocked(networking.getTeamPermissionsCall).mockResolvedValue({ + all_available_permissions: ["/key/generate", "/key/list"], + team_member_permissions: ["/key/generate"], + }); + + renderWithProviders(); + + await waitFor(() => { + expect(screen.getByText("Member Permissions")).toBeInTheDocument(); + }); + + const checkboxes = screen.getAllByRole("checkbox"); + checkboxes.forEach((checkbox) => { + expect(checkbox).toBeDisabled(); + }); + + expect(screen.queryByRole("button", { name: /save changes/i })).not.toBeInTheDocument(); + }); + + it("should handle reset button click", async () => { + vi.mocked(networking.getTeamPermissionsCall).mockResolvedValue({ + all_available_permissions: ["/key/generate", "/key/list"], + team_member_permissions: ["/key/generate"], + }); + + renderWithProviders(); + + await waitFor(() => { + expect(screen.getByText("Member Permissions")).toBeInTheDocument(); + }); + + const checkboxes = screen.getAllByRole("checkbox"); + const unselectedCheckbox = checkboxes.find((cb) => !(cb as HTMLInputElement).checked); + + if (unselectedCheckbox) { + await act(async () => { + fireEvent.click(unselectedCheckbox); + }); + + await waitFor(() => { + const resetButton = screen.getByRole("button", { name: /reset/i }); + expect(resetButton).toBeInTheDocument(); + }); + + vi.mocked(networking.getTeamPermissionsCall).mockResolvedValueOnce({ + all_available_permissions: ["/key/generate", "/key/list"], + team_member_permissions: ["/key/generate"], + }); + + const resetButton = screen.getByRole("button", { name: /reset/i }); + await act(async () => { + fireEvent.click(resetButton); + }); + + await waitFor(() => { + expect(networking.getTeamPermissionsCall).toHaveBeenCalledTimes(2); + }); + } + }); +}); diff --git a/ui/litellm-dashboard/src/components/team/permission_definitions.test.tsx b/ui/litellm-dashboard/src/components/team/permission_definitions.test.tsx new file mode 100644 index 00000000000..2d8aeabf383 --- /dev/null +++ b/ui/litellm-dashboard/src/components/team/permission_definitions.test.tsx @@ -0,0 +1,59 @@ +import { describe, expect, it } from "vitest"; +import { getMethodForEndpoint, getPermissionInfo, PERMISSION_DESCRIPTIONS } from "./permission_definitions"; + +describe("permission_definitions", () => { + describe("getMethodForEndpoint", () => { + it("should return GET for info endpoints", () => { + expect(getMethodForEndpoint("/key/info")).toBe("GET"); + }); + + it("should return GET for list endpoints", () => { + expect(getMethodForEndpoint("/key/list")).toBe("GET"); + }); + + it("should return POST for other endpoints", () => { + expect(getMethodForEndpoint("/key/generate")).toBe("POST"); + expect(getMethodForEndpoint("/key/update")).toBe("POST"); + expect(getMethodForEndpoint("/key/delete")).toBe("POST"); + }); + }); + + describe("getPermissionInfo", () => { + it("should return correct info for exact match permission", () => { + const result = getPermissionInfo("/key/generate"); + expect(result.method).toBe("POST"); + expect(result.endpoint).toBe("/key/generate"); + expect(result.description).toBe(PERMISSION_DESCRIPTIONS["/key/generate"]); + expect(result.route).toBe("/key/generate"); + }); + + it("should return GET method for info endpoint", () => { + const result = getPermissionInfo("/key/info"); + expect(result.method).toBe("GET"); + expect(result.endpoint).toBe("/key/info"); + expect(result.description).toBe(PERMISSION_DESCRIPTIONS["/key/info"]); + }); + + it("should return GET method for list endpoint", () => { + const result = getPermissionInfo("/key/list"); + expect(result.method).toBe("GET"); + expect(result.endpoint).toBe("/key/list"); + expect(result.description).toBe(PERMISSION_DESCRIPTIONS["/key/list"]); + }); + + it("should find partial match for permission with pattern", () => { + const result = getPermissionInfo("/key/service-account/generate"); + expect(result.method).toBe("POST"); + expect(result.endpoint).toBe("/key/service-account/generate"); + expect(result.description).toBe(PERMISSION_DESCRIPTIONS["/key/service-account/generate"]); + }); + + it("should return fallback description for unknown permission", () => { + const result = getPermissionInfo("/unknown/endpoint"); + expect(result.method).toBe("POST"); + expect(result.endpoint).toBe("/unknown/endpoint"); + expect(result.description).toBe("Access /unknown/endpoint"); + expect(result.route).toBe("/unknown/endpoint"); + }); + }); +});