Merge branch 'main' into add-digitalocean-provider

.circleci/config.yml

@@ -2,6 +2,7 @@ version: 2.1
 orbs:
   codecov: codecov/codecov@4.0.1
   node: circleci/node@5.1.0  # Add this line to declare the node orb
+  win: circleci/windows@5.0  # Add Windows orb
 
 commands:
   setup_google_dns:
@@ -15,8 +16,41 @@ commands:
             echo "nameserver 127.0.0.11" | sudo tee /etc/resolv.conf
             echo "nameserver 8.8.8.8" | sudo tee -a /etc/resolv.conf
             echo "nameserver 8.8.4.4" | sudo tee -a /etc/resolv.conf
+  setup_litellm_enterprise_pip:
+    steps:
+      - run:
+          name: "Install local version of litellm-enterprise"
+          command: |
+            cd enterprise
+            python -m pip install -e .
+            cd ..
 
 jobs:
+  # Add Windows testing job
+  using_litellm_on_windows:
+    executor:
+      name: win/default
+      shell: powershell.exe
+    working_directory: ~/project
+    steps:
+      - checkout
+      - run:
+          name: Install Python
+          command: |
+            choco install python --version=3.11.0 -y
+            refreshenv
+            python --version
+      - run:
+          name: Install Dependencies
+          command: |
+            python -m pip install --upgrade pip
+            pip install pytest
+            pip install .
+      - run:
+          name: Run Windows-specific test
+          command: |
+            python -m pytest tests/windows_tests/test_litellm_on_windows.py -v
+    
   local_testing:
     docker:
       - image: cimg/python:3.11
@@ -45,7 +79,7 @@ jobs:
             pip install "pytest-retry==1.6.3"
             pip install "pytest-asyncio==0.21.1"
             pip install "pytest-cov==5.0.0"
-            pip install mypy
+            pip install "mypy==1.15.0"
             pip install "google-generativeai==0.3.2"
             pip install "google-cloud-aiplatform==1.43.0"
             pip install pyarrow
@@ -61,11 +95,11 @@ jobs:
             pip install opentelemetry-api==1.25.0
             pip install opentelemetry-sdk==1.25.0
             pip install opentelemetry-exporter-otlp==1.25.0
-            pip install openai==1.68.2
+            pip install openai==1.81.0
             pip install prisma==0.11.0
             pip install "detect_secrets==1.5.0"
             pip install "httpx==0.24.1"
-            pip install "respx==0.21.1"
+            pip install "respx==0.22.0"
             pip install fastapi
             pip install "gunicorn==21.2.0"
             pip install "anyio==4.2.0"
@@ -85,6 +119,7 @@ jobs:
             pip install "pytest-xdist==3.6.1"
             pip install "websockets==13.1.0"
             pip uninstall posthog -y
+      - setup_litellm_enterprise_pip
       - save_cache:
           paths:
             - ./venv
@@ -107,10 +142,13 @@ jobs:
           name: Linting Testing
           command: |
             cd litellm
+            pip install "cryptography<40.0.0"
             python -m pip install types-requests types-setuptools types-redis types-PyYAML
-            if ! python -m mypy . --ignore-missing-imports; then
-              echo "mypy detected errors"
-              exit 1
+            if ! python -m mypy . \
+                --config-file mypy.ini \
+                --ignore-missing-imports; then
+                echo "mypy detected errors"
+                exit 1
             fi
             cd ..
 
@@ -180,11 +218,11 @@ jobs:
             pip install opentelemetry-api==1.25.0
             pip install opentelemetry-sdk==1.25.0
             pip install opentelemetry-exporter-otlp==1.25.0
-            pip install openai==1.68.2
+            pip install openai==1.81.0
             pip install prisma==0.11.0
             pip install "detect_secrets==1.5.0"
             pip install "httpx==0.24.1"
-            pip install "respx==0.21.1"
+            pip install "respx==0.22.0"
             pip install fastapi
             pip install "gunicorn==21.2.0"
             pip install "anyio==4.2.0"
@@ -202,6 +240,7 @@ jobs:
             pip install "Pillow==10.3.0"
             pip install "jsonschema==4.22.0"
             pip install "websockets==13.1.0"
+      - setup_litellm_enterprise_pip
       - save_cache:
           paths:
             - ./venv
@@ -286,11 +325,11 @@ jobs:
             pip install opentelemetry-api==1.25.0
             pip install opentelemetry-sdk==1.25.0
             pip install opentelemetry-exporter-otlp==1.25.0
-            pip install openai==1.68.2
+            pip install openai==1.81.0
             pip install prisma==0.11.0
             pip install "detect_secrets==1.5.0"
             pip install "httpx==0.24.1"
-            pip install "respx==0.21.1"
+            pip install "respx==0.22.0"
             pip install fastapi
             pip install "gunicorn==21.2.0"
             pip install "anyio==4.2.0"
@@ -308,6 +347,7 @@ jobs:
             pip install "Pillow==10.3.0"
             pip install "jsonschema==4.22.0"
             pip install "websockets==13.1.0"
+      - setup_litellm_enterprise_pip
       - save_cache:
           paths:
             - ./venv
@@ -414,11 +454,12 @@ jobs:
             python -m pip install --upgrade pip
             python -m pip install -r requirements.txt
             pip install "pytest==7.3.1"
-            pip install "respx==0.21.1"
+            pip install "respx==0.22.0"
             pip install "pytest-cov==5.0.0"
             pip install "pytest-retry==1.6.3"
             pip install "pytest-asyncio==0.21.1"
       # Run pytest and generate JUnit XML report
+      - setup_litellm_enterprise_pip
       - run:
           name: Run tests
           command: |
@@ -440,7 +481,7 @@ jobs:
           paths:
             - litellm_router_coverage.xml
             - litellm_router_coverage
-  litellm_proxy_security_tests:
+  litellm_security_tests:
     docker:
       - image: cimg/python:3.11
         auth:
@@ -463,6 +504,23 @@ jobs:
             pip install "pytest-retry==1.6.3"
             pip install "pytest-asyncio==0.21.1"
             pip install "pytest-cov==5.0.0"
+      - run:
+          name: Install Trivy
+          command: |
+            sudo apt-get update
+            sudo apt-get install wget apt-transport-https gnupg lsb-release
+            wget -qO - https://aquasecurity.github.io/trivy-repo/deb/public.key | sudo apt-key add -
+            echo "deb https://aquasecurity.github.io/trivy-repo/deb $(lsb_release -sc) main" | sudo tee -a /etc/apt/sources.list.d/trivy.list
+            sudo apt-get update
+            sudo apt-get install trivy
+      - run:
+          name: Run Trivy scan on LiteLLM Docs
+          command: |
+            trivy fs --scanners vuln --dependency-tree --exit-code 1 --severity HIGH,CRITICAL,MEDIUM ./docs/
+      - run:
+          name: Run Trivy scan on LiteLLM UI
+          command: |
+            trivy fs --scanners vuln --dependency-tree --exit-code 1 --severity HIGH,CRITICAL,MEDIUM ./ui/
       - run:
           name: Run prisma ./docker/entrypoint.sh
           command: |
@@ -481,16 +539,16 @@ jobs:
       - run:
           name: Rename the coverage files
           command: |
-            mv coverage.xml litellm_proxy_security_tests_coverage.xml
-            mv .coverage litellm_proxy_security_tests_coverage
+            mv coverage.xml litellm_security_tests_coverage.xml
+            mv .coverage litellm_security_tests_coverage
       # Store test results
       - store_test_results:
           path: test-results
       - persist_to_workspace:
           root: .
           paths:
-            - litellm_proxy_security_tests_coverage.xml
-            - litellm_proxy_security_tests_coverage
+            - litellm_security_tests_coverage.xml
+            - litellm_security_tests_coverage
   litellm_proxy_unit_testing: # Runs all tests with the "proxy", "key", "jwt" filenames
     docker:
       - image: cimg/python:3.11
@@ -540,11 +598,11 @@ jobs:
             pip install opentelemetry-api==1.25.0
             pip install opentelemetry-sdk==1.25.0
             pip install opentelemetry-exporter-otlp==1.25.0
-            pip install openai==1.68.2
+            pip install openai==1.81.0
             pip install prisma==0.11.0
             pip install "detect_secrets==1.5.0"
             pip install "httpx==0.24.1"
-            pip install "respx==0.21.1"
+            pip install "respx==0.22.0"
             pip install fastapi
             pip install "gunicorn==21.2.0"
             pip install "anyio==4.2.0"
@@ -562,6 +620,9 @@ jobs:
             pip install "Pillow==10.3.0"
             pip install "jsonschema==4.22.0"
             pip install "pytest-postgresql==7.0.1"
+            pip install "fakeredis==2.28.1"
+            pip install "pytest-xdist==3.6.1"
+      - setup_litellm_enterprise_pip
       - save_cache:
           paths:
             - ./venv
@@ -579,7 +640,7 @@ jobs:
           command: |
             pwd
             ls
-            python -m pytest tests/proxy_unit_tests --cov=litellm --cov-report=xml -vv -x -v --junitxml=test-results/junit.xml --durations=5
+            python -m pytest tests/proxy_unit_tests --cov=litellm --cov-report=xml -vv -x -v --junitxml=test-results/junit.xml --durations=5 -n 4
           no_output_timeout: 120m
       - run:
           name: Rename the coverage files
@@ -614,11 +675,12 @@ jobs:
             pip install --upgrade pip wheel setuptools
             python -m pip install -r requirements.txt
             pip install "pytest==7.3.1"
-            pip install "respx==0.21.1"
+            pip install "respx==0.22.0"
             pip install "pytest-retry==1.6.3"
             pip install "pytest-asyncio==0.21.1"
             pip install "pytest-cov==5.0.0"
       # Run pytest and generate JUnit XML report
+      - setup_litellm_enterprise_pip
       - run:
           name: Run tests
           command: |
@@ -659,7 +721,7 @@ jobs:
             pip install "pytest-retry==1.6.3"
             pip install "pytest-cov==5.0.0"
             pip install "pytest-asyncio==0.21.1"
-            pip install "respx==0.21.1"
+            pip install "respx==0.22.0"
       - run:
           name: Show current pydantic version
           command: |
@@ -696,14 +758,15 @@ jobs:
             pip install "pytest-retry==1.6.3"
             pip install "pytest-cov==5.0.0"
             pip install "pytest-asyncio==0.21.1"
-            pip install "respx==0.21.1"
+            pip install "respx==0.22.0"
+            pip install "pytest-xdist==3.6.1"
       # Run pytest and generate JUnit XML report
       - run:
           name: Run tests
           command: |
             pwd
             ls
-            python -m pytest -vv tests/llm_translation --cov=litellm --cov-report=xml -x -v --junitxml=test-results/junit.xml --durations=5
+            python -m pytest -vv tests/llm_translation --cov=litellm --cov-report=xml -x -v --junitxml=test-results/junit.xml --durations=5 -n 4
           no_output_timeout: 120m
       - run:
           name: Rename the coverage files
@@ -739,9 +802,9 @@ jobs:
             pip install "pytest-retry==1.6.3"
             pip install "pytest-cov==5.0.0"
             pip install "pytest-asyncio==0.21.1"
-            pip install "respx==0.21.1"
+            pip install "respx==0.22.0"
             pip install "pydantic==2.10.2"
-            pip install "mcp==1.5.0"
+            pip install "mcp==1.9.3"
       # Run pytest and generate JUnit XML report
       - run:
           name: Run tests
@@ -764,6 +827,51 @@ jobs:
           paths:
             - mcp_coverage.xml
             - mcp_coverage
+  guardrails_testing:
+    docker:
+      - image: cimg/python:3.11
+        auth:
+          username: ${DOCKERHUB_USERNAME}
+          password: ${DOCKERHUB_PASSWORD}
+    working_directory: ~/project
+
+    steps:
+      - checkout
+      - setup_google_dns
+      - run:
+          name: Install Dependencies
+          command: |
+            python -m pip install --upgrade pip
+            python -m pip install -r requirements.txt
+            pip install "pytest==7.3.1"
+            pip install "pytest-retry==1.6.3"
+            pip install "pytest-cov==5.0.0"
+            pip install "pytest-asyncio==0.21.1"
+            pip install "respx==0.22.0"
+            pip install "pydantic==2.10.2"
+            pip install "boto3==1.34.34"
+      # Run pytest and generate JUnit XML report
+      - run:
+          name: Run tests
+          command: |
+            pwd
+            ls
+            python -m pytest -vv tests/guardrails_tests --cov=litellm --cov-report=xml -x -s -v --junitxml=test-results/junit.xml --durations=5
+          no_output_timeout: 120m
+      - run:
+          name: Rename the coverage files
+          command: |
+            mv coverage.xml guardrails_coverage.xml
+            mv .coverage guardrails_coverage
+
+      # Store test results
+      - store_test_results:
+          path: test-results
+      - persist_to_workspace:
+          root: .
+          paths:
+            - guardrails_coverage.xml
+            - guardrails_coverage
   llm_responses_api_testing:
     docker:
       - image: cimg/python:3.11
@@ -784,7 +892,7 @@ jobs:
             pip install "pytest-retry==1.6.3"
             pip install "pytest-cov==5.0.0"
             pip install "pytest-asyncio==0.21.1"
-            pip install "respx==0.21.1"
+            pip install "respx==0.22.0"
       # Run pytest and generate JUnit XML report
       - run:
           name: Run tests
@@ -828,17 +936,28 @@ jobs:
             pip install "pytest-retry==1.6.3"
             pip install "pytest-cov==5.0.0"
             pip install "pytest-asyncio==0.21.1"
-            pip install "respx==0.21.1"
+            pip install "respx==0.22.0"
             pip install "hypercorn==0.17.3"
             pip install "pydantic==2.10.2"
-            pip install "mcp==1.5.0"
+            pip install "mcp==1.9.3"
+            pip install "requests-mock>=1.12.1"
+            pip install "responses==0.25.7"
+            pip install "pytest-xdist==3.6.1"
+      - setup_litellm_enterprise_pip
       # Run pytest and generate JUnit XML report
       - run:
-          name: Run tests
+          name: Run litellm tests
           command: |
             pwd
             ls
-            python -m pytest -vv tests/litellm --cov=litellm --cov-report=xml -x -s -v --junitxml=test-results/junit.xml --durations=5
+            python -m pytest -vv tests/test_litellm --cov=litellm --cov-report=xml -x -s -v --junitxml=test-results/junit-litellm.xml --durations=10 -n 4
+          no_output_timeout: 120m
+      - run:
+          name: Run enterprise tests
+          command: |
+            pwd
+            ls
+            python -m pytest -vv tests/enterprise --cov=litellm --cov-report=xml -x -s -v --junitxml=test-results/junit-enterprise.xml --durations=10 -n 4
           no_output_timeout: 120m
       - run:
           name: Rename the coverage files
@@ -870,7 +989,7 @@ jobs:
           command: |
             python -m pip install --upgrade pip
             python -m pip install -r requirements.txt
-            pip install "respx==0.21.1"
+            pip install "respx==0.22.0"
             pip install "pytest==7.3.1"
             pip install "pytest-retry==1.6.3"
             pip install "pytest-asyncio==0.21.1"
@@ -916,7 +1035,7 @@ jobs:
             python -m pip install --upgrade pip
             pip install numpydoc
             python -m pip install -r requirements.txt
-            pip install "respx==0.21.1"
+            pip install "respx==0.22.0"
             pip install "pytest==7.3.1"
             pip install "pytest-retry==1.6.3"
             pip install "pytest-asyncio==0.21.1"
@@ -966,7 +1085,7 @@ jobs:
             pip install "pytest-retry==1.6.3"
             pip install "pytest-cov==5.0.0"
             pip install "pytest-asyncio==0.21.1"
-            pip install "respx==0.21.1"
+            pip install "respx==0.22.0"
       # Run pytest and generate JUnit XML report
       - run:
           name: Run tests
@@ -1009,7 +1128,7 @@ jobs:
             pip install "pytest-retry==1.6.3"
             pip install "pytest-cov==5.0.0"
             pip install "pytest-asyncio==0.21.1"
-            pip install "respx==0.21.1"
+            pip install "respx==0.22.0"
       # Run pytest and generate JUnit XML report
       - run:
           name: Run tests
@@ -1053,11 +1172,14 @@ jobs:
             pip install "pytest-cov==5.0.0"
             pip install "pytest-asyncio==0.21.1"
             pip install pytest-mock
-            pip install "respx==0.21.1"
+            pip install "respx==0.22.0"
             pip install "google-generativeai==0.3.2"
             pip install "google-cloud-aiplatform==1.43.0"
             pip install "mlflow==2.17.2"
+            pip install "anthropic==0.52.0"
+            pip install "blockbuster==1.5.24"
       # Run pytest and generate JUnit XML report
+      - setup_litellm_enterprise_pip
       - run:
           name: Run tests
           command: |
@@ -1105,6 +1227,7 @@ jobs:
             pip install "tokenizers==0.20.0"
             pip install "uvloop==0.21.0"
             pip install jsonschema
+      - setup_litellm_enterprise_pip
       - run:
           name: Run tests
           command: |
@@ -1134,6 +1257,7 @@ jobs:
             pip install "pytest-asyncio==0.21.1"
             pip install "pytest-cov==5.0.0"
             pip install "tomli==2.2.1"
+            pip install "mcp==1.9.3"
       - run:
           name: Run tests
           command: |
@@ -1234,6 +1358,7 @@ jobs:
       # - run: python ./tests/documentation_tests/test_general_setting_keys.py
       - run: python ./tests/code_coverage_tests/check_licenses.py
       - run: python ./tests/code_coverage_tests/router_code_coverage.py
+      - run: python ./tests/code_coverage_tests/test_proxy_types_import.py
       - run: python ./tests/code_coverage_tests/callback_manager_test.py
       - run: python ./tests/code_coverage_tests/recursive_detector.py
       - run: python ./tests/code_coverage_tests/test_router_strategy_async.py
@@ -1246,6 +1371,7 @@ jobs:
       - run: python ./tests/code_coverage_tests/enforce_llms_folder_style.py
       - run: python ./tests/documentation_tests/test_circular_imports.py
       - run: python ./tests/code_coverage_tests/prevent_key_leaks_in_exceptions.py
+      - run: python ./tests/code_coverage_tests/check_unsafe_enterprise_import.py
       - run: helm lint ./deploy/charts/litellm-helm
 
   db_migration_disable_update_check:
@@ -1378,7 +1504,7 @@ jobs:
             pip install "aiodynamo==23.10.1"
             pip install "asyncio==3.4.3"
             pip install "PyGithub==1.59.1"
-            pip install "openai==1.68.2"
+            pip install "openai==1.81.0"
       - run:
           name: Install Grype
           command: |
@@ -1391,6 +1517,7 @@ jobs:
             docker build -t litellm-database:latest -f ./docker/Dockerfile.database .
             grype litellm-database:latest --fail-on high
 
+
             # Build and scan main Dockerfile
             echo "Building and scanning main Dockerfile..."
             docker build -t litellm:latest .
@@ -1453,7 +1580,7 @@ jobs:
           command: |
             pwd
             ls
-            python -m pytest -s -vv tests/*.py -x --junitxml=test-results/junit.xml --durations=5 --ignore=tests/otel_tests --ignore=tests/spend_tracking_tests --ignore=tests/pass_through_tests --ignore=tests/proxy_admin_ui_tests --ignore=tests/load_tests --ignore=tests/llm_translation --ignore=tests/llm_responses_api_testing --ignore=tests/mcp_tests --ignore=tests/image_gen_tests --ignore=tests/pass_through_unit_tests
+            python -m pytest -s -vv tests/*.py -x --junitxml=test-results/junit.xml --durations=5 --ignore=tests/otel_tests --ignore=tests/spend_tracking_tests --ignore=tests/pass_through_tests --ignore=tests/proxy_admin_ui_tests --ignore=tests/load_tests --ignore=tests/llm_translation --ignore=tests/llm_responses_api_testing --ignore=tests/mcp_tests --ignore=tests/guardrails_tests --ignore=tests/image_gen_tests --ignore=tests/pass_through_unit_tests
           no_output_timeout: 120m
 
       # Store test results
@@ -1516,7 +1643,7 @@ jobs:
             pip install "aiodynamo==23.10.1"
             pip install "asyncio==3.4.3"
             pip install "PyGithub==1.59.1"
-            pip install "openai==1.68.2"
+            pip install "openai==1.81.0"
             # Run pytest and generate JUnit XML report
       - run:
           name: Build Docker image
@@ -1639,7 +1766,7 @@ jobs:
             pip install "aiodynamo==23.10.1"
             pip install "asyncio==3.4.3"
             pip install "PyGithub==1.59.1"
-            pip install "openai==1.68.2"
+            pip install "openai==1.81.0"
       - run:
           name: Build Docker image
           command: docker build -t my-app:latest -f ./docker/Dockerfile.database .
@@ -2067,14 +2194,12 @@ jobs:
       - run:
           name: Build Docker image
           command: |
-            cd docker/build_from_pip
-            docker build -t my-app:latest -f Dockerfile.build_from_pip .
+            docker build -t my-app:latest -f docker/build_from_pip/Dockerfile.build_from_pip .
       - run:
           name: Run Docker container
           # intentionally give bad redis credentials here
           # the OTEL test - should get this as a trace
           command: |
-            cd docker/build_from_pip
             docker run -d \
               -p 4000:4000 \
               -e DATABASE_URL=$PROXY_DATABASE_URL \
@@ -2098,7 +2223,7 @@ jobs:
               -e DD_SITE=$DD_SITE \
               -e GCS_FLUSH_INTERVAL="1" \
               --name my-app \
-              -v $(pwd)/litellm_config.yaml:/app/config.yaml \
+              -v $(pwd)/docker/build_from_pip/litellm_config.yaml:/app/config.yaml \
               my-app:latest \
               --config /app/config.yaml \
               --port 4000 \
@@ -2162,7 +2287,7 @@ jobs:
             pip install "pytest-asyncio==0.21.1"
             pip install "google-cloud-aiplatform==1.43.0"
             pip install aiohttp
-            pip install "openai==1.68.2"
+            pip install "openai==1.81.0"
             pip install "assemblyai==0.37.0"
             python -m pip install --upgrade pip
             pip install "pydantic==2.10.2"
@@ -2181,7 +2306,7 @@ jobs:
             pip install "asyncio==3.4.3"
             pip install "PyGithub==1.59.1"
             pip install "google-cloud-aiplatform==1.59.0"
-            pip install "anthropic==0.49.0"
+            pip install "anthropic==0.52.0"
             pip install "langchain_mcp_adapters==0.0.5"
             pip install "langchain_openai==0.2.1"
             pip install "langgraph==0.3.18"
@@ -2312,7 +2437,7 @@ jobs:
             python -m venv venv
             . venv/bin/activate
             pip install coverage
-            coverage combine llm_translation_coverage llm_responses_api_coverage mcp_coverage logging_coverage litellm_router_coverage local_testing_coverage litellm_assistants_api_coverage auth_ui_unit_tests_coverage langfuse_coverage caching_coverage litellm_proxy_unit_tests_coverage image_gen_coverage pass_through_unit_tests_coverage batches_coverage litellm_proxy_security_tests_coverage
+            coverage combine llm_translation_coverage llm_responses_api_coverage mcp_coverage logging_coverage litellm_router_coverage local_testing_coverage litellm_assistants_api_coverage auth_ui_unit_tests_coverage langfuse_coverage caching_coverage litellm_proxy_unit_tests_coverage image_gen_coverage pass_through_unit_tests_coverage batches_coverage litellm_security_tests_coverage guardrails_coverage
             coverage xml
       - codecov/upload:
           file: ./coverage.xml
@@ -2550,7 +2675,7 @@ jobs:
             pip install "pytest-retry==1.6.3"
             pip install "pytest-asyncio==0.21.1"
             pip install aiohttp
-            pip install "openai==1.68.2"
+            pip install "openai==1.81.0"
             python -m pip install --upgrade pip
             pip install "pydantic==2.10.2"
             pip install "pytest==7.3.1"
@@ -2676,6 +2801,12 @@ workflows:
   version: 2
   build_and_test:
     jobs:
+      - using_litellm_on_windows:
+          filters:
+            branches:
+              only:
+                - main
+                - /litellm_.*/
       - local_testing:
           filters:
             branches:
@@ -2700,7 +2831,7 @@ workflows:
               only:
                 - main
                 - /litellm_.*/
-      - litellm_proxy_security_tests:
+      - litellm_security_tests:
           filters:
             branches:
               only:
@@ -2796,6 +2927,12 @@ workflows:
               only:
                 - main
                 - /litellm_.*/
+      - guardrails_testing:
+          filters:
+            branches:
+              only:
+                - main
+                - /litellm_.*/
       - llm_responses_api_testing:
           filters:
             branches:
@@ -2842,6 +2979,7 @@ workflows:
           requires:
             - llm_translation_testing
             - mcp_testing
+            - guardrails_testing
             - llm_responses_api_testing
             - litellm_mapped_tests
             - batches_testing
@@ -2852,7 +2990,7 @@ workflows:
             - litellm_router_testing
             - caching_unit_tests
             - litellm_proxy_unit_testing
-            - litellm_proxy_security_tests
+            - litellm_security_tests
             - langfuse_logging_unit_tests
             - local_testing
             - litellm_assistants_api_testing
@@ -2922,7 +3060,7 @@ workflows:
             - db_migration_disable_update_check
             - e2e_ui_testing
             - litellm_proxy_unit_testing
-            - litellm_proxy_security_tests
+            - litellm_security_tests
             - installing_litellm_on_python
             - installing_litellm_on_python_3_13
             - proxy_logging_guardrails_model_info_tests
@@ -2933,4 +3071,5 @@ workflows:
             - proxy_pass_through_endpoint_tests
             - check_code_and_doc_quality
             - publish_proxy_extras
-      
+            - guardrails_testing
+

.circleci/requirements.txt

@@ -1,5 +1,5 @@
 # used by CI/CD testing
-openai==1.68.2
+openai==1.81.0
 python-dotenv
 tiktoken
 importlib_metadata
@@ -7,9 +7,9 @@ cohere
 redis==5.2.1
 redisvl==0.4.1
 anthropic
-orjson==3.9.15
+orjson==3.10.12 # fast /embedding responses
 pydantic==2.10.2
 google-cloud-aiplatform==1.43.0
 fastapi-sso==0.16.0 
 uvloop==0.21.0
-mcp==1.5.0    # for MCP server  
+mcp==1.9.3    # for MCP server  

.env.example

@@ -1,6 +1,6 @@
 # OpenAI
 OPENAI_API_KEY = ""
-OPENAI_API_BASE = ""
+OPENAI_BASE_URL = ""
 # Cohere
 COHERE_API_KEY = ""
 # OpenRouter
@@ -20,10 +20,12 @@ REPLICATE_API_TOKEN = ""
 ANTHROPIC_API_KEY = ""
 # Infisical
 INFISICAL_TOKEN = ""
+# Novita AI
+NOVITA_API_KEY = ""
 # INFINITY
 INFINITY_API_KEY = ""
 
 # Development Configs
 LITELLM_MASTER_KEY = "sk-1234"
 DATABASE_URL = "postgresql://llmproxy:dbpassword9090@db:5432/litellm"
-STORE_MODEL_IN_DB = "True"
+STORE_MODEL_IN_DB = "True"

.github/ISSUE_TEMPLATE/feature_request.yml

@@ -23,10 +23,10 @@ body:
     validations:
       required: true
   - type: dropdown
-    id: ml-ops-team
+    id: hiring-interest
     attributes:
-      label: Are you a ML Ops Team?
-      description: This helps us prioritize your requests correctly 
+      label: LiteLLM is hiring a founding backend engineer, are you interested in joining us and shipping to all our users?
+      description: If yes, apply here - https://www.ycombinator.com/companies/litellm/jobs/6uvoBp3-founding-backend-engineer
       options:
         - "No"
         - "Yes"

.github/pull_request_template.md

@@ -12,7 +12,7 @@
 
 - [ ] I have Added testing in the [`tests/litellm/`](https://github.com/BerriAI/litellm/tree/main/tests/litellm) directory, **Adding at least 1 test is a hard requirement** - [see details](https://docs.litellm.ai/docs/extras/contributing_code)
 - [ ] I have added a screenshot of my new test passing locally 
-- [ ] My PR passes all unit tests on (`make test-unit`)[https://docs.litellm.ai/docs/extras/contributing_code]
+- [ ] My PR passes all unit tests on [`make test-unit`](https://docs.litellm.ai/docs/extras/contributing_code)
 - [ ] My PR's scope is as isolated as possible, it only solves 1 specific problem
 
 

.github/workflows/ghcr_deploy.yml

@@ -6,7 +6,7 @@ on:
       tag:
         description: "The tag version you want to build"
       release_type:
-        description: "The release type you want to build. Can be 'latest', 'stable', 'dev'"
+        description: "The release type you want to build. Can be 'latest', 'stable', 'dev', 'rc'"
         type: string
         default: "latest"
       commit_hash:
@@ -114,11 +114,55 @@ jobs:
           tags: |
             ${{ steps.meta.outputs.tags }}-${{ github.event.inputs.tag || 'latest' }},
             ${{ steps.meta.outputs.tags }}-${{ github.event.inputs.release_type }}
-            ${{ github.event.inputs.release_type == 'stable' && format('{0}/berriai/litellm:main-{1}', env.REGISTRY, github.event.inputs.tag) || '' }},
-            ${{ github.event.inputs.release_type == 'stable' && format('{0}/berriai/litellm:main-stable', env.REGISTRY) || '' }}
+            ${{ (github.event.inputs.release_type == 'stable'  || github.event.inputs.release_type == 'rc') && format('{0}/berriai/litellm:main-{1}', env.REGISTRY, github.event.inputs.tag) || '' }},
+            ${{ github.event.inputs.release_type == 'stable' && format('{0}/berriai/litellm:main-stable', env.REGISTRY) || '' }},
+            ${{ (github.event.inputs.release_type == 'stable'  || github.event.inputs.release_type == 'rc') && format('{0}/berriai/litellm:{1}', env.REGISTRY, github.event.inputs.tag) || '' }},
           labels: ${{ steps.meta.outputs.labels }}
           platforms: local,linux/amd64,linux/arm64,linux/arm64/v8
-          
+  
+  build-and-push-image-ee:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event.inputs.commit_hash }}
+
+      - name: Log in to the Container registry
+        uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Extract metadata (tags, labels) for EE Dockerfile
+        id: meta-ee
+        uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee
+      # Configure multi platform Docker builds
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@e0e4588fad221d38ee467c0bffd91115366dc0c5
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@edfb0fe6204400c56fbfd3feba3fe9ad1adfa345
+
+      - name: Build and push EE Docker image
+        uses: docker/build-push-action@f2a1d5e99d037542a71f64918e516c093c6f3fc4
+        with:
+          context: .
+          file: Dockerfile
+          push: true
+          tags: |
+            ${{ steps.meta-ee.outputs.tags }}-${{ github.event.inputs.tag || 'latest' }},
+            ${{ steps.meta-ee.outputs.tags }}-${{ github.event.inputs.release_type }}
+            ${{ (github.event.inputs.release_type == 'stable'  || github.event.inputs.release_type == 'rc') && format('{0}/berriai/litellm-ee:main-{1}', env.REGISTRY, github.event.inputs.tag) || '' }},
+            ${{ github.event.inputs.release_type == 'stable' && format('{0}/berriai/litellm-ee:main-stable', env.REGISTRY) || '' }}
+          labels: ${{ steps.meta-ee.outputs.labels }}
+          platforms: local,linux/amd64,linux/arm64,linux/arm64/v8
+   
   build-and-push-image-database:
     runs-on: ubuntu-latest
     permissions:
@@ -157,7 +201,7 @@ jobs:
           tags: |
             ${{ steps.meta-database.outputs.tags }}-${{ github.event.inputs.tag || 'latest' }},
             ${{ steps.meta-database.outputs.tags }}-${{ github.event.inputs.release_type }}
-            ${{ github.event.inputs.release_type == 'stable' && format('{0}/berriai/litellm-database:main-{1}', env.REGISTRY, github.event.inputs.tag) || '' }},
+            ${{ (github.event.inputs.release_type == 'stable'  || github.event.inputs.release_type == 'rc') && format('{0}/berriai/litellm-database:main-{1}', env.REGISTRY, github.event.inputs.tag) || '' }},
             ${{ github.event.inputs.release_type == 'stable' && format('{0}/berriai/litellm-database:main-stable', env.REGISTRY) || '' }}
           labels: ${{ steps.meta-database.outputs.labels }}
           platforms: local,linux/amd64,linux/arm64,linux/arm64/v8
@@ -200,7 +244,7 @@ jobs:
           tags: |
             ${{ steps.meta-non_root.outputs.tags }}-${{ github.event.inputs.tag || 'latest' }},
             ${{ steps.meta-non_root.outputs.tags }}-${{ github.event.inputs.release_type }}
-            ${{ github.event.inputs.release_type == 'stable' && format('{0}/berriai/litellm-non_root:main-{1}', env.REGISTRY, github.event.inputs.tag) || '' }},
+            ${{ (github.event.inputs.release_type == 'stable'  || github.event.inputs.release_type == 'rc') && format('{0}/berriai/litellm-non_root:main-{1}', env.REGISTRY, github.event.inputs.tag) || '' }},
             ${{ github.event.inputs.release_type == 'stable' && format('{0}/berriai/litellm-non_root:main-stable', env.REGISTRY) || '' }}
           labels: ${{ steps.meta-non_root.outputs.labels }} 
           platforms: local,linux/amd64,linux/arm64,linux/arm64/v8
@@ -243,7 +287,7 @@ jobs:
           tags: |
             ${{ steps.meta-spend-logs.outputs.tags }}-${{ github.event.inputs.tag || 'latest' }},
             ${{ steps.meta-spend-logs.outputs.tags }}-${{ github.event.inputs.release_type }}
-            ${{ github.event.inputs.release_type == 'stable' && format('{0}/berriai/litellm-spend_logs:main-{1}', env.REGISTRY, github.event.inputs.tag) || '' }},
+            ${{ (github.event.inputs.release_type == 'stable'  || github.event.inputs.release_type == 'rc') && format('{0}/berriai/litellm-spend_logs:main-{1}', env.REGISTRY, github.event.inputs.tag) || '' }},
             ${{ github.event.inputs.release_type == 'stable' && format('{0}/berriai/litellm-spend_logs:main-stable', env.REGISTRY) || '' }}
           platforms: local,linux/amd64,linux/arm64,linux/arm64/v8
 

.github/workflows/llm-translation-testing.yml

@@ -0,0 +1,89 @@
+name: LLM Translation Tests
+
+on:
+  workflow_dispatch:
+    inputs:
+      release_candidate_tag:
+        description: 'Release candidate tag/version'
+        required: true
+        type: string
+  push:
+    tags:
+      - 'v*-rc*'  # Triggers on release candidate tags like v1.0.0-rc1
+  
+jobs:
+  run-llm-translation-tests:
+    runs-on: ubuntu-latest
+    timeout-minutes: 90
+    
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event.inputs.release_candidate_tag || github.ref }}
+      
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.11'
+      
+      - name: Install Poetry
+        uses: snok/install-poetry@v1
+        with:
+          version: latest
+          virtualenvs-create: true
+          virtualenvs-in-project: true
+      
+      - name: Cache Poetry dependencies
+        uses: actions/cache@v3
+        with:
+          path: |
+            ~/.cache/pypoetry
+            .venv
+          key: ${{ runner.os }}-poetry-${{ hashFiles('**/poetry.lock') }}
+          restore-keys: |
+            ${{ runner.os }}-poetry-
+      
+      - name: Install dependencies
+        run: |
+          poetry install --with dev
+          poetry run pip install pytest-xdist pytest-timeout
+      
+      - name: Create test results directory
+        run: mkdir -p test-results
+      
+      - name: Run LLM Translation Tests
+        env:
+          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+          ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
+          COHERE_API_KEY: ${{ secrets.COHERE_API_KEY }}
+          GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }}
+          AZURE_API_KEY: ${{ secrets.AZURE_API_KEY }}
+          AZURE_API_BASE: ${{ secrets.AZURE_API_BASE }}
+          AZURE_API_VERSION: ${{ secrets.AZURE_API_VERSION }}
+          # Add other API keys as needed
+        run: |
+          python .github/workflows/run_llm_translation_tests.py \
+            --tag "${{ github.event.inputs.release_candidate_tag || github.ref_name }}" \
+            --commit "${{ github.sha }}" \
+            || true  # Continue even if tests fail
+      
+      - name: Display test summary
+        if: always()
+        run: |
+          if [ -f "test-results/llm_translation_report.md" ]; then
+            echo "Test report generated successfully!"
+            echo "Artifact will contain:"
+            echo "- test-results/junit.xml (JUnit XML results)"
+            echo "- test-results/llm_translation_report.md (Beautiful markdown report)"
+          else
+            echo "Warning: Test report was not generated"
+          fi
+      
+      - name: Upload test artifacts
+        uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: LLM-Translation-Artifact-${{ github.event.inputs.release_candidate_tag || github.ref_name }}
+          path: test-results/
+          retention-days: 30

.github/workflows/run_llm_translation_tests.py

@@ -0,0 +1,439 @@
+#!/usr/bin/env python3
+"""
+Run LLM Translation Tests and Generate Beautiful Markdown Report
+
+This script runs the LLM translation tests and generates a comprehensive
+markdown report with provider-specific breakdowns and test statistics.
+"""
+
+import os
+import sys
+import subprocess
+import xml.etree.ElementTree as ET
+from collections import defaultdict
+from datetime import datetime
+from pathlib import Path
+import json
+from typing import Dict, List, Tuple, Optional
+
+# ANSI color codes for terminal output
+class Colors:
+    GREEN = '\033[92m'
+    RED = '\033[91m'
+    YELLOW = '\033[93m'
+    BLUE = '\033[94m'
+    PURPLE = '\033[95m'
+    CYAN = '\033[96m'
+    RESET = '\033[0m'
+    BOLD = '\033[1m'
+
+def print_colored(message: str, color: str = Colors.RESET):
+    """Print colored message to terminal"""
+    print(f"{color}{message}{Colors.RESET}")
+
+def get_provider_from_test_file(test_file: str) -> str:
+    """Map test file names to provider names"""
+    provider_mapping = {
+        'test_anthropic': 'Anthropic',
+        'test_azure': 'Azure',
+        'test_bedrock': 'AWS Bedrock',
+        'test_openai': 'OpenAI',
+        'test_vertex': 'Google Vertex AI',
+        'test_gemini': 'Google Vertex AI',
+        'test_cohere': 'Cohere',
+        'test_databricks': 'Databricks',
+        'test_groq': 'Groq',
+        'test_together': 'Together AI',
+        'test_mistral': 'Mistral',
+        'test_deepseek': 'DeepSeek',
+        'test_replicate': 'Replicate',
+        'test_huggingface': 'HuggingFace',
+        'test_fireworks': 'Fireworks AI',
+        'test_perplexity': 'Perplexity',
+        'test_cloudflare': 'Cloudflare',
+        'test_voyage': 'Voyage AI',
+        'test_xai': 'xAI',
+        'test_nvidia': 'NVIDIA',
+        'test_watsonx': 'IBM watsonx',
+        'test_azure_ai': 'Azure AI',
+        'test_snowflake': 'Snowflake',
+        'test_infinity': 'Infinity',
+        'test_jina': 'Jina AI',
+        'test_deepgram': 'Deepgram',
+        'test_clarifai': 'Clarifai',
+        'test_triton': 'Triton',
+    }
+    
+    for key, provider in provider_mapping.items():
+        if key in test_file:
+            return provider
+    
+    # For cross-provider test files
+    if any(name in test_file for name in ['test_optional_params', 'test_prompt_factory', 
+                                           'test_router', 'test_text_completion']):
+        return f'Cross-Provider Tests ({test_file})'
+    
+    return 'Other Tests'
+
+def format_duration(seconds: float) -> str:
+    """Format duration in human-readable format"""
+    if seconds < 60:
+        return f"{seconds:.2f}s"
+    elif seconds < 3600:
+        minutes = int(seconds // 60)
+        secs = seconds % 60
+        return f"{minutes}m {secs:.0f}s"
+    else:
+        hours = int(seconds // 3600)
+        minutes = int((seconds % 3600) // 60)
+        return f"{hours}h {minutes}m"
+
+
+def generate_markdown_report(junit_xml_path: str, output_path: str, tag: str = None, commit: str = None):
+    """Generate a beautiful markdown report from JUnit XML"""
+    try:
+        tree = ET.parse(junit_xml_path)
+        root = tree.getroot()
+        
+        # Handle both testsuite and testsuites root
+        if root.tag == 'testsuites':
+            suites = root.findall('testsuite')
+        else:
+            suites = [root]
+        
+        # Overall statistics
+        total_tests = 0
+        total_failures = 0
+        total_errors = 0
+        total_skipped = 0
+        total_time = 0.0
+        
+        # Provider breakdown
+        provider_stats = defaultdict(lambda: {'passed': 0, 'failed': 0, 'skipped': 0, 'errors': 0, 'time': 0.0})
+        provider_tests = defaultdict(list)
+        
+        for suite in suites:
+            total_tests += int(suite.get('tests', 0))
+            total_failures += int(suite.get('failures', 0))
+            total_errors += int(suite.get('errors', 0))
+            total_skipped += int(suite.get('skipped', 0))
+            total_time += float(suite.get('time', 0))
+            
+            for testcase in suite.findall('testcase'):
+                classname = testcase.get('classname', '')
+                test_name = testcase.get('name', '')
+                test_time = float(testcase.get('time', 0))
+                
+                # Extract test file name from classname
+                if '.' in classname:
+                    parts = classname.split('.')
+                    test_file = parts[-2] if len(parts) > 1 else 'unknown'
+                else:
+                    test_file = 'unknown'
+                
+                provider = get_provider_from_test_file(test_file)
+                provider_stats[provider]['time'] += test_time
+                
+                # Check test status
+                if testcase.find('failure') is not None:
+                    provider_stats[provider]['failed'] += 1
+                    failure = testcase.find('failure')
+                    failure_msg = failure.get('message', '') if failure is not None else ''
+                    provider_tests[provider].append({
+                        'name': test_name,
+                        'status': 'FAILED',
+                        'time': test_time,
+                        'message': failure_msg
+                    })
+                elif testcase.find('error') is not None:
+                    provider_stats[provider]['errors'] += 1
+                    error = testcase.find('error')
+                    error_msg = error.get('message', '') if error is not None else ''
+                    provider_tests[provider].append({
+                        'name': test_name,
+                        'status': 'ERROR',
+                        'time': test_time,
+                        'message': error_msg
+                    })
+                elif testcase.find('skipped') is not None:
+                    provider_stats[provider]['skipped'] += 1
+                    skip = testcase.find('skipped')
+                    skip_msg = skip.get('message', '') if skip is not None else ''
+                    provider_tests[provider].append({
+                        'name': test_name,
+                        'status': 'SKIPPED',
+                        'time': test_time,
+                        'message': skip_msg
+                    })
+                else:
+                    provider_stats[provider]['passed'] += 1
+                    provider_tests[provider].append({
+                        'name': test_name,
+                        'status': 'PASSED',
+                        'time': test_time,
+                        'message': ''
+                    })
+        
+        passed = total_tests - total_failures - total_errors - total_skipped
+        
+        # Generate the markdown report
+        with open(output_path, 'w') as f:
+            # Header
+            f.write("# LLM Translation Test Results\n\n")
+            
+            # Metadata table
+            f.write("## Test Run Information\n\n")
+            f.write("| Field | Value |\n")
+            f.write("|-------|-------|\n")
+            f.write(f"| **Tag** | `{tag or 'N/A'}` |\n")
+            f.write(f"| **Date** | {datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S UTC')} |\n")
+            f.write(f"| **Commit** | `{commit or 'N/A'}` |\n")
+            f.write(f"| **Duration** | {format_duration(total_time)} |\n")
+            f.write("\n")
+            
+            # Overall statistics with visual elements
+            f.write("## Overall Statistics\n\n")
+            
+            # Summary box
+            f.write("```\n")
+            f.write(f"Total Tests: {total_tests}\n")
+            f.write(f"├── Passed:  {passed:>4} ({(passed/total_tests)*100 if total_tests > 0 else 0:.1f}%)\n")
+            f.write(f"├── Failed:  {total_failures:>4} ({(total_failures/total_tests)*100 if total_tests > 0 else 0:.1f}%)\n")
+            f.write(f"├── Errors:  {total_errors:>4} ({(total_errors/total_tests)*100 if total_tests > 0 else 0:.1f}%)\n")
+            f.write(f"└── Skipped: {total_skipped:>4} ({(total_skipped/total_tests)*100 if total_tests > 0 else 0:.1f}%)\n")
+            f.write("```\n\n")
+            
+            
+            # Provider summary table
+            f.write("## Results by Provider\n\n")
+            f.write("| Provider | Total | Pass | Fail | Error | Skip | Pass Rate | Duration |\n")
+            f.write("|----------|-------|------|------|-------|------|-----------|----------|")
+            
+            # Sort providers: specific providers first, then cross-provider tests
+            sorted_providers = []
+            cross_provider = []
+            for p in sorted(provider_stats.keys()):
+                if 'Cross-Provider' in p or p == 'Other Tests':
+                    cross_provider.append(p)
+                else:
+                    sorted_providers.append(p)
+            
+            all_providers = sorted_providers + cross_provider
+            
+            for provider in all_providers:
+                stats = provider_stats[provider]
+                total = stats['passed'] + stats['failed'] + stats['errors'] + stats['skipped']
+                pass_rate = (stats['passed'] / total * 100) if total > 0 else 0
+                
+                f.write(f"\n| {provider} | {total} | {stats['passed']} | {stats['failed']} | ")
+                f.write(f"{stats['errors']} | {stats['skipped']} | {pass_rate:.1f}% | ")
+                f.write(f"{format_duration(stats['time'])} |")
+            
+            # Detailed test results by provider
+            f.write("\n\n## Detailed Test Results\n\n")
+            
+            for provider in sorted_providers:
+                if provider_tests[provider]:
+                    stats = provider_stats[provider]
+                    total = stats['passed'] + stats['failed'] + stats['errors'] + stats['skipped']
+                    
+                    f.write(f"### {provider}\n\n")
+                    f.write(f"**Summary:** {stats['passed']}/{total} passed ")
+                    f.write(f"({(stats['passed']/total)*100 if total > 0 else 0:.1f}%) ")
+                    f.write(f"in {format_duration(stats['time'])}\n\n")
+                    
+                    # Group tests by status
+                    tests_by_status = defaultdict(list)
+                    for test in provider_tests[provider]:
+                        tests_by_status[test['status']].append(test)
+                    
+                    # Show failed tests first (if any)
+                    if tests_by_status['FAILED']:
+                        f.write("<details>\n<summary>Failed Tests</summary>\n\n")
+                        for test in tests_by_status['FAILED']:
+                            f.write(f"- `{test['name']}` ({test['time']:.2f}s)\n")
+                            if test['message']:
+                                # Truncate long error messages
+                                msg = test['message'][:200] + '...' if len(test['message']) > 200 else test['message']
+                                f.write(f"  > {msg}\n")
+                        f.write("\n</details>\n\n")
+                    
+                    # Show errors (if any)
+                    if tests_by_status['ERROR']:
+                        f.write("<details>\n<summary>Error Tests</summary>\n\n")
+                        for test in tests_by_status['ERROR']:
+                            f.write(f"- `{test['name']}` ({test['time']:.2f}s)\n")
+                        f.write("\n</details>\n\n")
+                    
+                    # Show passed tests in collapsible section
+                    if tests_by_status['PASSED']:
+                        f.write("<details>\n<summary>Passed Tests</summary>\n\n")
+                        for test in tests_by_status['PASSED']:
+                            f.write(f"- `{test['name']}` ({test['time']:.2f}s)\n")
+                        f.write("\n</details>\n\n")
+                    
+                    # Show skipped tests (if any)
+                    if tests_by_status['SKIPPED']:
+                        f.write("<details>\n<summary>Skipped Tests</summary>\n\n")
+                        for test in tests_by_status['SKIPPED']:
+                            f.write(f"- `{test['name']}`\n")
+                        f.write("\n</details>\n\n")
+            
+            # Cross-provider tests in a separate section
+            if cross_provider:
+                f.write("### Cross-Provider Tests\n\n")
+                for provider in cross_provider:
+                    if provider_tests[provider]:
+                        stats = provider_stats[provider]
+                        total = stats['passed'] + stats['failed'] + stats['errors'] + stats['skipped']
+                        
+                        f.write(f"#### {provider}\n\n")
+                        f.write(f"**Summary:** {stats['passed']}/{total} passed ")
+                        f.write(f"({(stats['passed']/total)*100 if total > 0 else 0:.1f}%)\n\n")
+                        
+                        # For cross-provider tests, just show counts
+                        f.write(f"- Passed: {stats['passed']}\n")
+                        if stats['failed'] > 0:
+                            f.write(f"- Failed: {stats['failed']}\n")
+                        if stats['errors'] > 0:
+                            f.write(f"- Errors: {stats['errors']}\n")
+                        if stats['skipped'] > 0:
+                            f.write(f"- Skipped: {stats['skipped']}\n")
+                        f.write("\n")
+            
+        
+        print_colored(f"Report generated: {output_path}", Colors.GREEN)
+        
+    except Exception as e:
+        print_colored(f"Error generating report: {e}", Colors.RED)
+        raise
+
+def run_tests(test_path: str = "tests/llm_translation/", 
+              junit_xml: str = "test-results/junit.xml",
+              report_path: str = "test-results/llm_translation_report.md",
+              tag: str = None,
+              commit: str = None) -> int:
+    """Run the LLM translation tests and generate report"""
+    
+    # Create test results directory
+    os.makedirs(os.path.dirname(junit_xml), exist_ok=True)
+    
+    print_colored("Starting LLM Translation Tests", Colors.BOLD + Colors.BLUE)
+    print_colored(f"Test directory: {test_path}", Colors.CYAN)
+    print_colored(f"Output: {junit_xml}", Colors.CYAN)
+    print()
+    
+    # Run pytest
+    cmd = [
+        "poetry", "run", "pytest", test_path,
+        f"--junitxml={junit_xml}",
+        "-v",
+        "--tb=short",
+        "--maxfail=500",
+        "-n", "auto"
+    ]
+    
+    # Add timeout if pytest-timeout is installed
+    try:
+        subprocess.run(["poetry", "run", "python", "-c", "import pytest_timeout"], 
+                      capture_output=True, check=True)
+        cmd.extend(["--timeout=300"])
+    except:
+        print_colored("Warning: pytest-timeout not installed, skipping timeout option", Colors.YELLOW)
+    
+    print_colored("Running pytest with command:", Colors.YELLOW)
+    print(f"   {' '.join(cmd)}")
+    print()
+    
+    # Run the tests
+    result = subprocess.run(cmd, capture_output=False)
+    
+    # Generate the report regardless of test outcome
+    if os.path.exists(junit_xml):
+        print()
+        print_colored("Generating test report...", Colors.BLUE)
+        generate_markdown_report(junit_xml, report_path, tag, commit)
+        
+        # Print summary to console
+        print()
+        print_colored("Test Summary:", Colors.BOLD + Colors.PURPLE)
+        
+        # Parse XML for quick summary
+        tree = ET.parse(junit_xml)
+        root = tree.getroot()
+        
+        if root.tag == 'testsuites':
+            suites = root.findall('testsuite')
+        else:
+            suites = [root]
+        
+        total = sum(int(s.get('tests', 0)) for s in suites)
+        failures = sum(int(s.get('failures', 0)) for s in suites)
+        errors = sum(int(s.get('errors', 0)) for s in suites)
+        skipped = sum(int(s.get('skipped', 0)) for s in suites)
+        passed = total - failures - errors - skipped
+        
+        print(f"   Total:   {total}")
+        print_colored(f"   Passed:  {passed}", Colors.GREEN)
+        if failures > 0:
+            print_colored(f"   Failed:  {failures}", Colors.RED)
+        if errors > 0:
+            print_colored(f"   Errors:  {errors}", Colors.RED)
+        if skipped > 0:
+            print_colored(f"   Skipped: {skipped}", Colors.YELLOW)
+        
+        if total > 0:
+            pass_rate = (passed / total) * 100
+            color = Colors.GREEN if pass_rate >= 80 else Colors.YELLOW if pass_rate >= 60 else Colors.RED
+            print_colored(f"   Pass Rate: {pass_rate:.1f}%", color)
+    else:
+        print_colored("No test results found!", Colors.RED)
+    
+    print()
+    print_colored("Test run complete!", Colors.BOLD + Colors.GREEN)
+    
+    return result.returncode
+
+if __name__ == "__main__":
+    import argparse
+    
+    parser = argparse.ArgumentParser(description="Run LLM Translation Tests")
+    parser.add_argument("--test-path", default="tests/llm_translation/", 
+                        help="Path to test directory")
+    parser.add_argument("--junit-xml", default="test-results/junit.xml",
+                        help="Path for JUnit XML output")
+    parser.add_argument("--report", default="test-results/llm_translation_report.md",
+                        help="Path for markdown report")
+    parser.add_argument("--tag", help="Git tag or version")
+    parser.add_argument("--commit", help="Git commit SHA")
+    
+    args = parser.parse_args()
+    
+    # Get git info if not provided
+    if not args.commit:
+        try:
+            result = subprocess.run(["git", "rev-parse", "HEAD"], 
+                                    capture_output=True, text=True)
+            if result.returncode == 0:
+                args.commit = result.stdout.strip()
+        except:
+            pass
+    
+    if not args.tag:
+        try:
+            result = subprocess.run(["git", "describe", "--tags", "--abbrev=0"], 
+                                    capture_output=True, text=True)
+            if result.returncode == 0:
+                args.tag = result.stdout.strip()
+        except:
+            pass
+    
+    exit_code = run_tests(
+        test_path=args.test_path,
+        junit_xml=args.junit_xml,
+        report_path=args.report,
+        tag=args.tag,
+        commit=args.commit
+    )
+    
+    sys.exit(exit_code)

.github/workflows/test-linting.yml

@@ -22,7 +22,11 @@ jobs:
 
     - name: Install dependencies
       run: |
+        pip install openai==1.81.0
         poetry install --with dev
+        pip install openai==1.81.0
+
+
 
     - name: Run Black formatting
       run: |

.github/workflows/test-litellm.yml

@@ -1,4 +1,4 @@
-name: LiteLLM Mock Tests (folder - tests/litellm)
+name: LiteLLM Mock Tests (folder - tests/test_litellm)
 
 on:
   pull_request:
@@ -7,7 +7,7 @@ on:
 jobs:
   test:
     runs-on: ubuntu-latest
-    timeout-minutes: 5
+    timeout-minutes: 15
 
     steps:
     - uses: actions/checkout@v4
@@ -28,8 +28,13 @@ jobs:
     - name: Install dependencies
       run: |
         poetry install --with dev,proxy-dev --extras proxy
+        poetry run pip install "pytest-retry==1.6.3"
         poetry run pip install pytest-xdist
-
+    - name: Setup litellm-enterprise as local package
+      run: |
+        cd enterprise
+        python -m pip install -e .
+        cd ..
     - name: Run tests
       run: |
-        poetry run pytest tests/litellm -x -vv -n 4 
+        poetry run pytest tests/test_litellm -x -vv -n 4 

.gitignore

@@ -88,3 +88,8 @@ litellm/proxy/migrations/*config.yaml
 litellm/proxy/migrations/*
 config.yaml
 tests/litellm/litellm_core_utils/llm_cost_calc/log.txt
+tests/test_custom_dir/*
+test.py
+
+litellm_config.yaml
+.cursor

.pre-commit-config.yaml

@@ -6,27 +6,27 @@ repos:
         entry: pyright
         language: system
         types: [python]
-        files: ^(litellm/|litellm_proxy_extras/)
+        files: ^(litellm/|litellm_proxy_extras/|enterprise/)
     -   id: isort
         name: isort
         entry: isort
         language: system
         types: [python]
-        files: (litellm/|litellm_proxy_extras/).*\.py
+        files: (litellm/|litellm_proxy_extras/|enterprise/).*\.py
         exclude: ^litellm/__init__.py$
-    -   id: black
-        name: black
-        entry: poetry run black
-        language: system
-        types: [python]
-        files: (litellm/|litellm_proxy_extras/).*\.py
+    # -   id: black
+    #     name: black
+    #     entry: poetry run black
+    #     language: system
+    #     types: [python]
+    #     files: (litellm/|litellm_proxy_extras/|enterprise/).*\.py
 -   repo: https://github.com/pycqa/flake8
     rev: 7.0.0  # The version of flake8 to use
     hooks:
     -  id: flake8
-       exclude: ^litellm/tests/|^litellm/proxy/tests/|^litellm/tests/litellm/|^tests/litellm/
+       exclude: ^litellm/tests/|^litellm/proxy/tests/|^litellm/tests/test_litellm/|^tests/test_litellm/|^tests/enterprise/
        additional_dependencies: [flake8-print]
-       files: (litellm/|litellm_proxy_extras/).*\.py
+       files: (litellm/|litellm_proxy_extras/|enterprise/).*\.py
 -   repo: https://github.com/python-poetry/poetry
     rev: 1.8.0
     hooks:

AGENTS.md

@@ -0,0 +1,144 @@
+# INSTRUCTIONS FOR LITELLM
+
+This document provides comprehensive instructions for AI agents working in the LiteLLM repository.
+
+## OVERVIEW
+
+LiteLLM is a unified interface for 100+ LLMs that:
+- Translates inputs to provider-specific completion, embedding, and image generation endpoints
+- Provides consistent OpenAI-format output across all providers
+- Includes retry/fallback logic across multiple deployments (Router)
+- Offers a proxy server (LLM Gateway) with budgets, rate limits, and authentication
+- Supports advanced features like function calling, streaming, caching, and observability
+
+## REPOSITORY STRUCTURE
+
+### Core Components
+- `litellm/` - Main library code
+  - `llms/` - Provider-specific implementations (OpenAI, Anthropic, Azure, etc.)
+  - `proxy/` - Proxy server implementation (LLM Gateway)
+  - `router_utils/` - Load balancing and fallback logic
+  - `types/` - Type definitions and schemas
+  - `integrations/` - Third-party integrations (observability, caching, etc.)
+
+### Key Directories
+- `tests/` - Comprehensive test suites
+- `docs/my-website/` - Documentation website
+- `ui/litellm-dashboard/` - Admin dashboard UI
+- `enterprise/` - Enterprise-specific features
+
+## DEVELOPMENT GUIDELINES
+
+### MAKING CODE CHANGES
+
+1. **Provider Implementations**: When adding/modifying LLM providers:
+   - Follow existing patterns in `litellm/llms/{provider}/`
+   - Implement proper transformation classes that inherit from `BaseConfig`
+   - Support both sync and async operations
+   - Handle streaming responses appropriately
+   - Include proper error handling with provider-specific exceptions
+
+2. **Type Safety**: 
+   - Use proper type hints throughout
+   - Update type definitions in `litellm/types/`
+   - Ensure compatibility with both Pydantic v1 and v2
+
+3. **Testing**:
+   - Add tests in appropriate `tests/` subdirectories
+   - Include both unit tests and integration tests
+   - Test provider-specific functionality thoroughly
+   - Consider adding load tests for performance-critical changes
+
+### IMPORTANT PATTERNS
+
+1. **Function/Tool Calling**:
+   - LiteLLM standardizes tool calling across providers
+   - OpenAI format is the standard, with transformations for other providers
+   - See `litellm/llms/anthropic/chat/transformation.py` for complex tool handling
+
+2. **Streaming**:
+   - All providers should support streaming where possible
+   - Use consistent chunk formatting across providers
+   - Handle both sync and async streaming
+
+3. **Error Handling**:
+   - Use provider-specific exception classes
+   - Maintain consistent error formats across providers
+   - Include proper retry logic and fallback mechanisms
+
+4. **Configuration**:
+   - Support both environment variables and programmatic configuration
+   - Use `BaseConfig` classes for provider configurations
+   - Allow dynamic parameter passing
+
+## PROXY SERVER (LLM GATEWAY)
+
+The proxy server is a critical component that provides:
+- Authentication and authorization
+- Rate limiting and budget management
+- Load balancing across multiple models/deployments
+- Observability and logging
+- Admin dashboard UI
+- Enterprise features
+
+Key files:
+- `litellm/proxy/proxy_server.py` - Main server implementation
+- `litellm/proxy/auth/` - Authentication logic
+- `litellm/proxy/management_endpoints/` - Admin API endpoints
+
+## MCP (MODEL CONTEXT PROTOCOL) SUPPORT
+
+LiteLLM supports MCP for agent workflows:
+- MCP server integration for tool calling
+- Transformation between OpenAI and MCP tool formats
+- Support for external MCP servers (Zapier, Jira, Linear, etc.)
+- See `litellm/experimental_mcp_client/` and `litellm/proxy/_experimental/mcp_server/`
+
+## TESTING CONSIDERATIONS
+
+1. **Provider Tests**: Test against real provider APIs when possible
+2. **Proxy Tests**: Include authentication, rate limiting, and routing tests
+3. **Performance Tests**: Load testing for high-throughput scenarios
+4. **Integration Tests**: End-to-end workflows including tool calling
+
+## DOCUMENTATION
+
+- Keep documentation in sync with code changes
+- Update provider documentation when adding new providers
+- Include code examples for new features
+- Update changelog and release notes
+
+## SECURITY CONSIDERATIONS
+
+- Handle API keys securely
+- Validate all inputs, especially for proxy endpoints
+- Consider rate limiting and abuse prevention
+- Follow security best practices for authentication
+
+## ENTERPRISE FEATURES
+
+- Some features are enterprise-only
+- Check `enterprise/` directory for enterprise-specific code
+- Maintain compatibility between open-source and enterprise versions
+
+## COMMON PITFALLS TO AVOID
+
+1. **Breaking Changes**: LiteLLM has many users - avoid breaking existing APIs
+2. **Provider Specifics**: Each provider has unique quirks - handle them properly
+3. **Rate Limits**: Respect provider rate limits in tests
+4. **Memory Usage**: Be mindful of memory usage in streaming scenarios
+5. **Dependencies**: Keep dependencies minimal and well-justified
+
+## HELPFUL RESOURCES
+
+- Main documentation: https://docs.litellm.ai/
+- Provider-specific docs in `docs/my-website/docs/providers/`
+- Admin UI for testing proxy features
+
+## WHEN IN DOUBT
+
+- Follow existing patterns in the codebase
+- Check similar provider implementations
+- Ensure comprehensive test coverage
+- Update documentation appropriately
+- Consider backward compatibility impact 

CLAUDE.md

@@ -0,0 +1,89 @@
+# CLAUDE.md
+
+This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository.
+
+## Development Commands
+
+### Installation
+- `make install-dev` - Install core development dependencies
+- `make install-proxy-dev` - Install proxy development dependencies with full feature set
+- `make install-test-deps` - Install all test dependencies
+
+### Testing
+- `make test` - Run all tests
+- `make test-unit` - Run unit tests (tests/test_litellm) with 4 parallel workers
+- `make test-integration` - Run integration tests (excludes unit tests)
+- `pytest tests/` - Direct pytest execution
+
+### Code Quality
+- `make lint` - Run all linting (Ruff, MyPy, Black, circular imports, import safety)
+- `make format` - Apply Black code formatting
+- `make lint-ruff` - Run Ruff linting only
+- `make lint-mypy` - Run MyPy type checking only
+
+### Single Test Files
+- `poetry run pytest tests/path/to/test_file.py -v` - Run specific test file
+- `poetry run pytest tests/path/to/test_file.py::test_function -v` - Run specific test
+
+## Architecture Overview
+
+LiteLLM is a unified interface for 100+ LLM providers with two main components:
+
+### Core Library (`litellm/`)
+- **Main entry point**: `litellm/main.py` - Contains core completion() function
+- **Provider implementations**: `litellm/llms/` - Each provider has its own subdirectory
+- **Router system**: `litellm/router.py` + `litellm/router_utils/` - Load balancing and fallback logic
+- **Type definitions**: `litellm/types/` - Pydantic models and type hints
+- **Integrations**: `litellm/integrations/` - Third-party observability, caching, logging
+- **Caching**: `litellm/caching/` - Multiple cache backends (Redis, in-memory, S3, etc.)
+
+### Proxy Server (`litellm/proxy/`)
+- **Main server**: `proxy_server.py` - FastAPI application
+- **Authentication**: `auth/` - API key management, JWT, OAuth2
+- **Database**: `db/` - Prisma ORM with PostgreSQL/SQLite support
+- **Management endpoints**: `management_endpoints/` - Admin APIs for keys, teams, models
+- **Pass-through endpoints**: `pass_through_endpoints/` - Provider-specific API forwarding
+- **Guardrails**: `guardrails/` - Safety and content filtering hooks
+- **UI Dashboard**: Served from `_experimental/out/` (Next.js build)
+
+## Key Patterns
+
+### Provider Implementation
+- Providers inherit from base classes in `litellm/llms/base.py`
+- Each provider has transformation functions for input/output formatting
+- Support both sync and async operations
+- Handle streaming responses and function calling
+
+### Error Handling
+- Provider-specific exceptions mapped to OpenAI-compatible errors
+- Fallback logic handled by Router system
+- Comprehensive logging through `litellm/_logging.py`
+
+### Configuration
+- YAML config files for proxy server (see `proxy/example_config_yaml/`)
+- Environment variables for API keys and settings
+- Database schema managed via Prisma (`proxy/schema.prisma`)
+
+## Development Notes
+
+### Code Style
+- Uses Black formatter, Ruff linter, MyPy type checker
+- Pydantic v2 for data validation
+- Async/await patterns throughout
+- Type hints required for all public APIs
+
+### Testing Strategy
+- Unit tests in `tests/test_litellm/`
+- Integration tests for each provider in `tests/llm_translation/`
+- Proxy tests in `tests/proxy_unit_tests/`
+- Load tests in `tests/load_tests/`
+
+### Database Migrations
+- Prisma handles schema migrations
+- Migration files auto-generated with `prisma migrate dev`
+- Always test migrations against both PostgreSQL and SQLite
+
+### Enterprise Features
+- Enterprise-specific code in `enterprise/` directory
+- Optional features enabled via environment variables
+- Separate licensing and authentication for enterprise features

CONTRIBUTING.md

@@ -0,0 +1,275 @@
+# Contributing to LiteLLM
+
+Thank you for your interest in contributing to LiteLLM! We welcome contributions of all kinds - from bug fixes and documentation improvements to new features and integrations.
+
+## **Checklist before submitting a PR**
+
+Here are the core requirements for any PR submitted to LiteLLM:
+
+- [ ] **Sign the Contributor License Agreement (CLA)** - [see details](#contributor-license-agreement-cla)
+- [ ] **Add testing** - Adding at least 1 test is a hard requirement - [see details](#adding-testing)
+- [ ] **Ensure your PR passes all checks**:
+  - [ ] [Unit Tests](#running-unit-tests) - `make test-unit`
+  - [ ] [Linting / Formatting](#running-linting-and-formatting-checks) - `make lint`
+- [ ] **Keep scope isolated** - Your changes should address 1 specific problem at a time
+
+## **Contributor License Agreement (CLA)**
+
+Before contributing code to LiteLLM, you must sign our [Contributor License Agreement (CLA)](https://cla-assistant.io/BerriAI/litellm). This is a legal requirement for all contributions to be merged into the main repository.
+
+**Important:** We strongly recommend reviewing and signing the CLA before starting work on your contribution to avoid any delays in the PR process.
+
+## Quick Start
+
+### 1. Setup Your Local Development Environment
+
+```bash
+# Clone the repository
+git clone https://github.com/BerriAI/litellm.git
+cd litellm
+
+# Create a new branch for your feature
+git checkout -b your-feature-branch
+
+# Install development dependencies
+make install-dev
+
+# Verify your setup works
+make help
+```
+
+That's it! Your local development environment is ready.
+
+### 2. Development Workflow
+
+Here's the recommended workflow for making changes:
+
+```bash
+# Make your changes to the code
+# ...
+
+# Format your code (auto-fixes formatting issues)
+make format
+
+# Run all linting checks (matches CI exactly)
+make lint
+
+# Run unit tests to ensure nothing is broken
+make test-unit
+
+# Commit your changes
+git add .
+git commit -m "Your descriptive commit message"
+
+# Push and create a PR
+git push origin your-feature-branch
+```
+
+## Adding Testing
+
+**Adding at least 1 test is a hard requirement for all PRs.**
+
+### Where to Add Tests
+
+Add your tests to the [`tests/test_litellm/` directory](https://github.com/BerriAI/litellm/tree/main/tests/test_litellm).
+
+- This directory mirrors the structure of the `litellm/` directory
+- **Only add mocked tests** - no real LLM API calls in this directory
+- For integration tests with real APIs, use the appropriate test directories
+
+### File Naming Convention
+
+The `tests/test_litellm/` directory follows the same structure as `litellm/`:
+
+- `litellm/proxy/caching_routes.py` → `tests/test_litellm/proxy/test_caching_routes.py`
+- `litellm/utils.py` → `tests/test_litellm/test_utils.py`
+
+### Example Test
+
+```python
+import pytest
+from litellm import completion
+
+def test_your_feature():
+    """Test your feature with a descriptive docstring."""
+    # Arrange
+    messages = [{"role": "user", "content": "Hello"}]
+    
+    # Act
+    # Use mocked responses, not real API calls
+    
+    # Assert
+    assert expected_result == actual_result
+```
+
+## Running Tests and Checks
+
+### Running Unit Tests
+
+Run all unit tests (uses parallel execution for speed):
+
+```bash
+make test-unit
+```
+
+Run specific test files:
+```bash
+poetry run pytest tests/test_litellm/test_your_file.py -v
+```
+
+### Running Linting and Formatting Checks
+
+Run all linting checks (matches CI exactly):
+
+```bash
+make lint
+```
+
+Individual linting commands:
+```bash
+make format-check       # Check Black formatting
+make lint-ruff          # Run Ruff linting
+make lint-mypy          # Run MyPy type checking
+make check-circular-imports    # Check for circular imports
+make check-import-safety       # Check import safety
+```
+
+Apply formatting (auto-fixes issues):
+```bash
+make format
+```
+
+### CI Compatibility
+
+To ensure your changes will pass CI, run the exact same checks locally:
+
+```bash
+# This runs the same checks as the GitHub workflows
+make lint
+make test-unit
+```
+
+For exact CI compatibility (pins OpenAI version like CI):
+```bash
+make install-dev-ci     # Installs exact CI dependencies
+```
+
+## Available Make Commands
+
+Run `make help` to see all available commands:
+
+```bash
+make help                       # Show all available commands
+make install-dev               # Install development dependencies
+make install-proxy-dev         # Install proxy development dependencies
+make install-test-deps         # Install test dependencies (for running tests)
+make format                    # Apply Black code formatting
+make format-check              # Check Black formatting (matches CI)
+make lint                      # Run all linting checks
+make test-unit                 # Run unit tests
+make test-integration          # Run integration tests
+make test-unit-helm            # Run Helm unit tests
+```
+
+## Code Quality Standards
+
+LiteLLM follows the [Google Python Style Guide](https://google.github.io/styleguide/pyguide.html).
+
+Our automated quality checks include:
+- **Black** for consistent code formatting
+- **Ruff** for linting and code quality
+- **MyPy** for static type checking
+- **Circular import detection**
+- **Import safety validation**
+
+All checks must pass before your PR can be merged.
+
+## Common Issues and Solutions
+
+### 1. Linting Failures
+
+If `make lint` fails:
+
+1. **Formatting issues**: Run `make format` to auto-fix
+2. **Ruff issues**: Check the output and fix manually
+3. **MyPy issues**: Add proper type hints
+4. **Circular imports**: Refactor import dependencies
+5. **Import safety**: Fix any unprotected imports
+
+### 2. Test Failures
+
+If `make test-unit` fails:
+
+1. Check if you broke existing functionality
+2. Add tests for your new code
+3. Ensure tests use mocks, not real API calls
+4. Check test file naming conventions
+
+### 3. Common Development Tips
+
+- **Use type hints**: MyPy requires proper type annotations
+- **Write descriptive commit messages**: Help reviewers understand your changes
+- **Keep PRs focused**: One feature/fix per PR
+- **Test edge cases**: Don't just test the happy path
+- **Update documentation**: If you change APIs, update docs
+
+## Building and Running Locally
+
+### LiteLLM Proxy Server
+
+To run the proxy server locally:
+
+```bash
+# Install proxy dependencies
+make install-proxy-dev
+
+# Start the proxy server
+poetry run litellm --config your_config.yaml
+```
+
+### Docker Development
+
+If you want to build the Docker image yourself:
+
+```bash
+# Build using the non-root Dockerfile
+docker build -f docker/Dockerfile.non_root -t litellm_dev .
+
+# Run with your config
+docker run \
+    -v $(pwd)/proxy_config.yaml:/app/config.yaml \
+    -e LITELLM_MASTER_KEY="sk-1234" \
+    -p 4000:4000 \
+    litellm_dev \
+    --config /app/config.yaml --detailed_debug
+```
+
+## Submitting Your PR
+
+1. **Push your branch**: `git push origin your-feature-branch`
+2. **Create a PR**: Go to GitHub and create a pull request
+3. **Fill out the PR template**: Provide clear description of changes
+4. **Wait for review**: Maintainers will review and provide feedback
+5. **Address feedback**: Make requested changes and push updates
+6. **Merge**: Once approved, your PR will be merged!
+
+## Getting Help
+
+If you need help:
+
+- 💬 [Join our Discord](https://discord.gg/wuPM9dRgDw)
+- 💬 [Join our Slack](https://join.slack.com/share/enQtOTE0ODczMzk2Nzk4NC01YjUxNjY2YjBlYTFmNDRiZTM3NDFiYTM3MzVkODFiMDVjOGRjMmNmZTZkZTMzOWQzZGQyZWIwYjQ0MWExYmE3)
+- 📧 Email us: ishaan@berri.ai / krrish@berri.ai
+- 🐛 [Create an issue](https://github.com/BerriAI/litellm/issues/new)
+
+## What to Contribute
+
+Looking for ideas? Check out:
+
+- 🐛 [Good first issues](https://github.com/BerriAI/litellm/labels/good%20first%20issue)
+- 🚀 [Feature requests](https://github.com/BerriAI/litellm/labels/enhancement)
+- 📚 Documentation improvements
+- 🧪 Test coverage improvements
+- 🔌 New LLM provider integrations
+
+Thank you for contributing to LiteLLM! 🚀 

Dockerfile

@@ -51,7 +51,7 @@ FROM $LITELLM_RUNTIME_IMAGE AS runtime
 USER root
 
 # Install runtime dependencies
-RUN apk add --no-cache openssl
+RUN apk add --no-cache openssl tzdata
 
 WORKDIR /app
 # Copy the current directory contents into the container at /app
@@ -74,5 +74,5 @@ EXPOSE 4000/tcp
 
 ENTRYPOINT ["docker/prod_entrypoint.sh"]
 
-# Append "--detailed_debug" to the end of CMD to view detailed debug logs 
+# Append "--detailed_debug" to the end of CMD to view detailed debug logs
 CMD ["--port", "4000"]

GEMINI.md

@@ -0,0 +1,89 @@
+# GEMINI.md
+
+This file provides guidance to Gemini when working with code in this repository.
+
+## Development Commands
+
+### Installation
+- `make install-dev` - Install core development dependencies
+- `make install-proxy-dev` - Install proxy development dependencies with full feature set
+- `make install-test-deps` - Install all test dependencies
+
+### Testing
+- `make test` - Run all tests
+- `make test-unit` - Run unit tests (tests/test_litellm) with 4 parallel workers
+- `make test-integration` - Run integration tests (excludes unit tests)
+- `pytest tests/` - Direct pytest execution
+
+### Code Quality
+- `make lint` - Run all linting (Ruff, MyPy, Black, circular imports, import safety)
+- `make format` - Apply Black code formatting
+- `make lint-ruff` - Run Ruff linting only
+- `make lint-mypy` - Run MyPy type checking only
+
+### Single Test Files
+- `poetry run pytest tests/path/to/test_file.py -v` - Run specific test file
+- `poetry run pytest tests/path/to/test_file.py::test_function -v` - Run specific test
+
+## Architecture Overview
+
+LiteLLM is a unified interface for 100+ LLM providers with two main components:
+
+### Core Library (`litellm/`)
+- **Main entry point**: `litellm/main.py` - Contains core completion() function
+- **Provider implementations**: `litellm/llms/` - Each provider has its own subdirectory
+- **Router system**: `litellm/router.py` + `litellm/router_utils/` - Load balancing and fallback logic
+- **Type definitions**: `litellm/types/` - Pydantic models and type hints
+- **Integrations**: `litellm/integrations/` - Third-party observability, caching, logging
+- **Caching**: `litellm/caching/` - Multiple cache backends (Redis, in-memory, S3, etc.)
+
+### Proxy Server (`litellm/proxy/`)
+- **Main server**: `proxy_server.py` - FastAPI application
+- **Authentication**: `auth/` - API key management, JWT, OAuth2
+- **Database**: `db/` - Prisma ORM with PostgreSQL/SQLite support
+- **Management endpoints**: `management_endpoints/` - Admin APIs for keys, teams, models
+- **Pass-through endpoints**: `pass_through_endpoints/` - Provider-specific API forwarding
+- **Guardrails**: `guardrails/` - Safety and content filtering hooks
+- **UI Dashboard**: Served from `_experimental/out/` (Next.js build)
+
+## Key Patterns
+
+### Provider Implementation
+- Providers inherit from base classes in `litellm/llms/base.py`
+- Each provider has transformation functions for input/output formatting
+- Support both sync and async operations
+- Handle streaming responses and function calling
+
+### Error Handling
+- Provider-specific exceptions mapped to OpenAI-compatible errors
+- Fallback logic handled by Router system
+- Comprehensive logging through `litellm/_logging.py`
+
+### Configuration
+- YAML config files for proxy server (see `proxy/example_config_yaml/`)
+- Environment variables for API keys and settings
+- Database schema managed via Prisma (`proxy/schema.prisma`)
+
+## Development Notes
+
+### Code Style
+- Uses Black formatter, Ruff linter, MyPy type checker
+- Pydantic v2 for data validation
+- Async/await patterns throughout
+- Type hints required for all public APIs
+
+### Testing Strategy
+- Unit tests in `tests/test_litellm/`
+- Integration tests for each provider in `tests/llm_translation/`
+- Proxy tests in `tests/proxy_unit_tests/`
+- Load tests in `tests/load_tests/`
+
+### Database Migrations
+- Prisma handles schema migrations
+- Migration files auto-generated with `prisma migrate dev`
+- Always test migrations against both PostgreSQL and SQLite
+
+### Enterprise Features
+- Enterprise-specific code in `enterprise/` directory
+- Optional features enabled via environment variables
+- Separate licensing and authentication for enterprise features

Makefile

@@ -1,35 +1,103 @@
 # LiteLLM Makefile
 # Simple Makefile for running tests and basic development tasks
 
-.PHONY: help test test-unit test-integration lint format
+.PHONY: help test test-unit test-integration test-unit-helm lint format install-dev install-proxy-dev install-test-deps install-helm-unittest check-circular-imports check-import-safety
 
 # Default target
 help:
 	@echo "Available commands:"
+	@echo "  make install-dev        - Install development dependencies"
+	@echo "  make install-proxy-dev  - Install proxy development dependencies"
+	@echo "  make install-dev-ci     - Install dev dependencies (CI-compatible, pins OpenAI)"
+	@echo "  make install-proxy-dev-ci - Install proxy dev dependencies (CI-compatible)"
+	@echo "  make install-test-deps  - Install test dependencies"
+	@echo "  make install-helm-unittest - Install helm unittest plugin"
+	@echo "  make format             - Apply Black code formatting"
+	@echo "  make format-check       - Check Black code formatting (matches CI)"
+	@echo "  make lint               - Run all linting (Ruff, MyPy, Black check, circular imports, import safety)"
+	@echo "  make lint-ruff          - Run Ruff linting only"
+	@echo "  make lint-mypy          - Run MyPy type checking only"
+	@echo "  make lint-black         - Check Black formatting (matches CI)"
+	@echo "  make check-circular-imports - Check for circular imports"
+	@echo "  make check-import-safety - Check import safety"
 	@echo "  make test               - Run all tests"
-	@echo "  make test-unit          - Run unit tests"
+	@echo "  make test-unit          - Run unit tests (tests/test_litellm)"
 	@echo "  make test-integration   - Run integration tests"
 	@echo "  make test-unit-helm     - Run helm unit tests"
 
+# Installation targets
 install-dev:
 	poetry install --with dev
 
 install-proxy-dev:
-	poetry install --with dev,proxy-dev
+	poetry install --with dev,proxy-dev --extras proxy
 
-lint: install-dev
+# CI-compatible installations (matches GitHub workflows exactly)
+install-dev-ci:
+	pip install openai==1.81.0
+	poetry install --with dev
+	pip install openai==1.81.0
+
+install-proxy-dev-ci:
+	poetry install --with dev,proxy-dev --extras proxy
+	pip install openai==1.81.0
+
+install-test-deps: install-proxy-dev
+	poetry run pip install "pytest-retry==1.6.3"
+	poetry run pip install pytest-xdist
+	cd enterprise && python -m pip install -e . && cd ..
+
+install-helm-unittest:
+	helm plugin install https://github.com/helm-unittest/helm-unittest --version v0.4.4
+
+# Formatting
+format: install-dev
+	cd litellm && poetry run black . && cd ..
+
+format-check: install-dev
+	cd litellm && poetry run black --check . && cd ..
+
+# Linting targets
+lint-ruff: install-dev
+	cd litellm && poetry run ruff check . && cd ..
+
+lint-mypy: install-dev
 	poetry run pip install types-requests types-setuptools types-redis types-PyYAML
-	cd litellm && poetry run mypy . --ignore-missing-imports
+	cd litellm && poetry run mypy . --ignore-missing-imports && cd ..
 
-# Testing
+lint-black: format-check
+
+check-circular-imports: install-dev
+	cd litellm && poetry run python ../tests/documentation_tests/test_circular_imports.py && cd ..
+
+check-import-safety: install-dev
+	poetry run python -c "from litellm import *" || (echo '🚨 import failed, this means you introduced unprotected imports! 🚨'; exit 1)
+
+# Combined linting (matches test-linting.yml workflow)
+lint: format-check lint-ruff lint-mypy check-circular-imports check-import-safety
+
+# Testing targets
 test:
 	poetry run pytest tests/
 
-test-unit:
-	poetry run pytest tests/litellm/
+test-unit: install-test-deps
+	poetry run pytest tests/test_litellm -x -vv -n 4
 
 test-integration:
-	poetry run pytest tests/ -k "not litellm"
+	poetry run pytest tests/ -k "not test_litellm"
 
-test-unit-helm:
-	helm unittest -f 'tests/*.yaml' deploy/charts/litellm-helm
+test-unit-helm: install-helm-unittest
+	helm unittest -f 'tests/*.yaml' deploy/charts/litellm-helm
+
+# LLM Translation testing targets
+test-llm-translation: install-test-deps
+	@echo "Running LLM translation tests..."
+	@python .github/workflows/run_llm_translation_tests.py
+
+test-llm-translation-single: install-test-deps
+	@echo "Running single LLM translation test file..."
+	@if [ -z "$(FILE)" ]; then echo "Usage: make test-llm-translation-single FILE=test_filename.py"; exit 1; fi
+	@mkdir -p test-results
+	poetry run pytest tests/llm_translation/$(FILE) \
+		--junitxml=test-results/junit.xml \
+		-v --tb=short --maxfail=100 --timeout=300

README.md

@@ -25,6 +25,9 @@
     <a href="https://discord.gg/wuPM9dRgDw">
         <img src="https://img.shields.io/static/v1?label=Chat%20on&message=Discord&color=blue&logo=Discord&style=flat-square" alt="Discord">
     </a>
+    <a href="https://join.slack.com/share/enQtOTE0ODczMzk2Nzk4NC01YjUxNjY2YjBlYTFmNDRiZTM3NDFiYTM3MzVkODFiMDVjOGRjMmNmZTZkZTMzOWQzZGQyZWIwYjQ0MWExYmE3">
+        <img src="https://img.shields.io/static/v1?label=Chat%20on&message=Slack&color=black&logo=Slack&style=flat-square" alt="Slack">
+    </a>
 </h4>
 
 LiteLLM manages:
@@ -261,7 +264,7 @@ echo 'LITELLM_MASTER_KEY="sk-1234"' > .env
 # It is used to encrypt / decrypt your LLM API Key credentials
 # We recommend - https://1password.com/password-generator/
 # password generator to get a random hash for litellm salt key
-echo 'LITELLM_SALT_KEY="sk-1234"' > .env
+echo 'LITELLM_SALT_KEY="sk-1234"' >> .env
 
 source .env
 
@@ -299,6 +302,7 @@ curl 'http://0.0.0.0:4000/key/generate' \
 | Provider                                                                            | [Completion](https://docs.litellm.ai/docs/#basic-usage) | [Streaming](https://docs.litellm.ai/docs/completion/stream#streaming-responses) | [Async Completion](https://docs.litellm.ai/docs/completion/stream#async-completion) | [Async Streaming](https://docs.litellm.ai/docs/completion/stream#async-streaming) | [Async Embedding](https://docs.litellm.ai/docs/embedding/supported_embedding) | [Async Image Generation](https://docs.litellm.ai/docs/image_generation) |
 |-------------------------------------------------------------------------------------|---------------------------------------------------------|---------------------------------------------------------------------------------|-------------------------------------------------------------------------------------|-----------------------------------------------------------------------------------|-------------------------------------------------------------------------------|-------------------------------------------------------------------------|
 | [openai](https://docs.litellm.ai/docs/providers/openai)                             | ✅                                                       | ✅                                                                               | ✅                                                                                   | ✅                                                                                 | ✅                                                                             | ✅                                                                       |
+| [Meta - Llama API](https://docs.litellm.ai/docs/providers/meta_llama)                               | ✅                                                       | ✅                                                                               | ✅                                                                                   | ✅                                                                                 |                                                                              |                                                                        |
 | [azure](https://docs.litellm.ai/docs/providers/azure)                               | ✅                                                       | ✅                                                                               | ✅                                                                                   | ✅                                                                                 | ✅                                                                             | ✅                                                                       |
 | [AI/ML API](https://docs.litellm.ai/docs/providers/aiml)                               | ✅                                                       | ✅                                                                               | ✅                                                                                   | ✅                                                                                 | ✅                                                                             | ✅                                                                       |
 | [aws - sagemaker](https://docs.litellm.ai/docs/providers/aws_sagemaker)             | ✅                                                       | ✅                                                                               | ✅                                                                                   | ✅                                                                                 | ✅                                                                             |                                                                         |
@@ -333,12 +337,19 @@ curl 'http://0.0.0.0:4000/key/generate' \
 | [FriendliAI](https://docs.litellm.ai/docs/providers/friendliai)                              | ✅                                                       | ✅                                                                               | ✅                                                                                   | ✅                                                                                 |                                                                               |                                                                         |
 | [Galadriel](https://docs.litellm.ai/docs/providers/galadriel)                              | ✅                                                       | ✅                                                                               | ✅                                                                                   | ✅                                                                                 |                                                                               |                                                                         |
 | [DigitalOcean](https://docs.litellm.ai/docs/providers/digitalocean)                              | ✅                                                       | ✅                                                                               |                                                                                   |                                                                                  |                                                                               |                                                                         |
+| [Novita AI](https://novita.ai/models/llm?utm_source=github_litellm&utm_medium=github_readme&utm_campaign=github_link)                     | ✅                                                       | ✅                                                                               | ✅                                                                                   | ✅                                                                                 |                                                                               |                                                                         |
+| [Featherless AI](https://docs.litellm.ai/docs/providers/featherless_ai)                              | ✅                                                       | ✅                                                                               | ✅                                                                                   | ✅                                                                                 |                                                                               |                                                                         |
+| [Nebius AI Studio](https://docs.litellm.ai/docs/providers/nebius)                             | ✅                                                       | ✅                                                                               | ✅                                                                                   | ✅                                                                                 | ✅                                                                             |                                                                         |
 
 [**Read the Docs**](https://docs.litellm.ai/docs/)
 
 ## Contributing
 
-Interested in contributing? Contributions to LiteLLM Python SDK, Proxy Server, and contributing LLM integrations are both accepted and highly encouraged! [See our Contribution Guide for more details](https://docs.litellm.ai/docs/extras/contributing_code)
+Interested in contributing? Contributions to LiteLLM Python SDK, Proxy Server, and LLM integrations are both accepted and highly encouraged! 
+
+**Quick start:** `git clone` → `make install-dev` → `make format` → `make lint` → `make test-unit`
+
+See our comprehensive [Contributing Guide (CONTRIBUTING.md)](CONTRIBUTING.md) for detailed instructions.
 
 # Enterprise
 For companies that need better security, user management and professional support
@@ -353,24 +364,48 @@ This covers:
 - ✅ **Custom SLAs**
 - ✅ **Secure access with Single Sign-On**
 
-# Code Quality / Linting
+# Contributing
+
+We welcome contributions to LiteLLM! Whether you're fixing bugs, adding features, or improving documentation, we appreciate your help.
+
+## Quick Start for Contributors
+
+```bash
+git clone https://github.com/BerriAI/litellm.git
+cd litellm
+make install-dev    # Install development dependencies
+make format         # Format your code
+make lint           # Run all linting checks
+make test-unit      # Run unit tests
+```
+
+For detailed contributing guidelines, see [CONTRIBUTING.md](CONTRIBUTING.md).
+
+## Code Quality / Linting
 
 LiteLLM follows the [Google Python Style Guide](https://google.github.io/styleguide/pyguide.html).
 
-We run:
-- Ruff for [formatting and linting checks](https://github.com/BerriAI/litellm/blob/e19bb55e3b4c6a858b6e364302ebbf6633a51de5/.circleci/config.yml#L320)
-- Mypy + Pyright for typing [1](https://github.com/BerriAI/litellm/blob/e19bb55e3b4c6a858b6e364302ebbf6633a51de5/.circleci/config.yml#L90), [2](https://github.com/BerriAI/litellm/blob/e19bb55e3b4c6a858b6e364302ebbf6633a51de5/.pre-commit-config.yaml#L4)
-- Black for [formatting](https://github.com/BerriAI/litellm/blob/e19bb55e3b4c6a858b6e364302ebbf6633a51de5/.circleci/config.yml#L79)
-- isort for [import sorting](https://github.com/BerriAI/litellm/blob/e19bb55e3b4c6a858b6e364302ebbf6633a51de5/.pre-commit-config.yaml#L10)
+Our automated checks include:
+- **Black** for code formatting
+- **Ruff** for linting and code quality
+- **MyPy** for type checking
+- **Circular import detection**
+- **Import safety checks**
 
+Run all checks locally:
+```bash
+make lint           # Run all linting (matches CI)
+make format-check   # Check formatting only
+```
 
-If you have suggestions on how to improve the code quality feel free to open an issue or a PR.
+All these checks must pass before your PR can be merged.
 
 
 # Support / talk with founders
 
 - [Schedule Demo 👋](https://calendly.com/d/4mp-gd3-k5k/berriai-1-1-onboarding-litellm-hosted-version)
 - [Community Discord 💭](https://discord.gg/wuPM9dRgDw)
+- [Community Slack 💭](https://join.slack.com/share/enQtOTE0ODczMzk2Nzk4NC01YjUxNjY2YjBlYTFmNDRiZTM3NDFiYTM3MzVkODFiMDVjOGRjMmNmZTZkZTMzOWQzZGQyZWIwYjQ0MWExYmE3)
 - Our numbers 📞 +1 (770) 8783-106 / ‭+1 (412) 618-6238‬
 - Our emails ✉️ ishaan@berri.ai / krrish@berri.ai
 

cookbook/LiteLLM_NovitaAI_Cookbook.ipynb

@@ -0,0 +1,97 @@
+{
+  "cells": [
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "iFEmsVJI_2BR"
+      },
+      "source": [
+        "# LiteLLM NovitaAI Cookbook"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "cBlUhCEP_xj4"
+      },
+      "outputs": [],
+      "source": [
+        "!pip install litellm"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "p-MQqWOT_1a7"
+      },
+      "outputs": [],
+      "source": [
+        "import os\n",
+        "\n",
+        "os.environ['NOVITA_API_KEY'] = \"\""
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "Ze8JqMqWAARO"
+      },
+      "outputs": [],
+      "source": [
+        "from litellm import completion\n",
+        "response = completion(\n",
+        "            model=\"novita/deepseek/deepseek-r1\",\n",
+        "            messages=[{\"role\": \"user\", \"content\": \"write code for saying hi\"}]\n",
+        ")\n",
+        "response"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "-LnhELrnAM_J"
+      },
+      "outputs": [],
+      "source": [
+        "response = completion(\n",
+        "            model=\"novita/deepseek/deepseek-r1\",\n",
+        "            messages=[{\"role\": \"user\", \"content\": \"write code for saying hi\"}]\n",
+        ")\n",
+        "response"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "dJBOUYdwCEn1"
+      },
+      "outputs": [],
+      "source": [
+        "response = completion(\n",
+        "            model=\"mistralai/mistral-7b-instruct\",\n",
+        "            messages=[{\"role\": \"user\", \"content\": \"write code for saying hi\"}]\n",
+        ")\n",
+        "response"
+      ]
+    }
+  ],
+  "metadata": {
+    "colab": {
+      "provenance": []
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    },
+    "language_info": {
+      "name": "python"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}

cookbook/LiteLLM_OpenRouter.ipynb

@@ -1,27 +1,13 @@
 {
-  "nbformat": 4,
-  "nbformat_minor": 0,
-  "metadata": {
-    "colab": {
-      "provenance": []
-    },
-    "kernelspec": {
-      "name": "python3",
-      "display_name": "Python 3"
-    },
-    "language_info": {
-      "name": "python"
-    }
-  },
   "cells": [
     {
       "cell_type": "markdown",
-      "source": [
-        "# LiteLLM OpenRouter Cookbook"
-      ],
       "metadata": {
         "id": "iFEmsVJI_2BR"
-      }
+      },
+      "source": [
+        "# LiteLLM OpenRouter Cookbook"
+      ]
     },
     {
       "cell_type": "code",
@@ -36,27 +22,20 @@
     },
     {
       "cell_type": "code",
+      "execution_count": 14,
+      "metadata": {
+        "id": "p-MQqWOT_1a7"
+      },
+      "outputs": [],
       "source": [
         "import os\n",
         "\n",
         "os.environ['OPENROUTER_API_KEY'] = \"\""
-      ],
-      "metadata": {
-        "id": "p-MQqWOT_1a7"
-      },
-      "execution_count": 14,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "code",
-      "source": [
-        "from litellm import completion\n",
-        "response = completion(\n",
-        "            model=\"openrouter/google/palm-2-chat-bison\",\n",
-        "            messages=[{\"role\": \"user\", \"content\": \"write code for saying hi\"}]\n",
-        ")\n",
-        "response"
-      ],
+      "execution_count": 11,
       "metadata": {
         "colab": {
           "base_uri": "https://localhost:8080/"
@@ -64,10 +43,8 @@
         "id": "Ze8JqMqWAARO",
         "outputId": "64f3e836-69fa-4f8e-fb35-088a913bbe98"
       },
-      "execution_count": 11,
       "outputs": [
         {
-          "output_type": "execute_result",
           "data": {
             "text/plain": [
               "<OpenAIObject id=gen-W8FTMSIEorCp3vG5iYIgNMR4IeBv at 0x7c3dcef1f060> JSON: {\n",
@@ -85,20 +62,23 @@
               "}"
             ]
           },
+          "execution_count": 11,
           "metadata": {},
-          "execution_count": 11
+          "output_type": "execute_result"
         }
+      ],
+      "source": [
+        "from litellm import completion\n",
+        "response = completion(\n",
+        "            model=\"openrouter/google/palm-2-chat-bison\",\n",
+        "            messages=[{\"role\": \"user\", \"content\": \"write code for saying hi\"}]\n",
+        ")\n",
+        "response"
       ]
     },
     {
       "cell_type": "code",
-      "source": [
-        "response = completion(\n",
-        "            model=\"openrouter/anthropic/claude-2\",\n",
-        "            messages=[{\"role\": \"user\", \"content\": \"write code for saying hi\"}]\n",
-        ")\n",
-        "response"
-      ],
+      "execution_count": 12,
       "metadata": {
         "colab": {
           "base_uri": "https://localhost:8080/"
@@ -106,10 +86,8 @@
         "id": "-LnhELrnAM_J",
         "outputId": "d51c7ab7-d761-4bd1-f849-1534d9df4cd0"
       },
-      "execution_count": 12,
       "outputs": [
         {
-          "output_type": "execute_result",
           "data": {
             "text/plain": [
               "<OpenAIObject id=gen-IiuV7ZNimDufVeutBHrl8ajPuzEh at 0x7c3dcea67560> JSON: {\n",
@@ -128,20 +106,22 @@
               "}"
             ]
           },
+          "execution_count": 12,
           "metadata": {},
-          "execution_count": 12
+          "output_type": "execute_result"
         }
+      ],
+      "source": [
+        "response = completion(\n",
+        "            model=\"openrouter/anthropic/claude-2\",\n",
+        "            messages=[{\"role\": \"user\", \"content\": \"write code for saying hi\"}]\n",
+        ")\n",
+        "response"
       ]
     },
     {
       "cell_type": "code",
-      "source": [
-        "response = completion(\n",
-        "            model=\"openrouter/meta-llama/llama-2-70b-chat\",\n",
-        "            messages=[{\"role\": \"user\", \"content\": \"write code for saying hi\"}]\n",
-        ")\n",
-        "response"
-      ],
+      "execution_count": 13,
       "metadata": {
         "colab": {
           "base_uri": "https://localhost:8080/"
@@ -149,10 +129,8 @@
         "id": "dJBOUYdwCEn1",
         "outputId": "ffa18679-ec15-4dad-fe2b-68665cdf36b0"
       },
-      "execution_count": 13,
       "outputs": [
         {
-          "output_type": "execute_result",
           "data": {
             "text/plain": [
               "<OpenAIObject id=gen-PyMd3yyJ0aQsCgIY9R8XGZoAtPbl at 0x7c3dceefcae0> JSON: {\n",
@@ -170,10 +148,32 @@
               "}"
             ]
           },
+          "execution_count": 13,
           "metadata": {},
-          "execution_count": 13
+          "output_type": "execute_result"
         }
+      ],
+      "source": [
+        "response = completion(\n",
+        "            model=\"openrouter/meta-llama/llama-2-70b-chat\",\n",
+        "            messages=[{\"role\": \"user\", \"content\": \"write code for saying hi\"}]\n",
+        ")\n",
+        "response"
       ]
     }
-  ]
-}
+  ],
+  "metadata": {
+    "colab": {
+      "provenance": []
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    },
+    "language_info": {
+      "name": "python"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}

cookbook/google_adk_litellm_tutorial.ipynb

@@ -0,0 +1,412 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "7aa8875d",
+   "metadata": {},
+   "source": [
+    "# Google ADK with LiteLLM\n",
+    "\n",
+    "Use Google ADK with LiteLLM Python SDK, LiteLLM Proxy.\n",
+    "\n",
+    "This tutorial shows you how to create intelligent agents using Agent Development Kit (ADK) with support for multiple Large Language Model (LLM) providers through LiteLLM."
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "a4d249c3",
+   "metadata": {},
+   "source": [
+    "## Overview\n",
+    "\n",
+    "ADK (Agent Development Kit) allows you to build intelligent agents powered by LLMs. By integrating with LiteLLM, you can:\n",
+    "\n",
+    "- Use multiple LLM providers (OpenAI, Anthropic, Google, etc.)\n",
+    "- Switch easily between models from different providers\n",
+    "- Connect to a LiteLLM proxy for centralized model management"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "a0bbb56b",
+   "metadata": {},
+   "source": [
+    "## Prerequisites\n",
+    "\n",
+    "- Python environment setup\n",
+    "- API keys for model providers (OpenAI, Anthropic, Google AI Studio)\n",
+    "- Basic understanding of LLMs and agent concepts"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "7fee50a8",
+   "metadata": {},
+   "source": [
+    "## Installation"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "44106a23",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Install dependencies\n",
+    "!pip install google-adk litellm"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "2171740a",
+   "metadata": {},
+   "source": [
+    "## 1. Setting Up Environment"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "6695807e",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Setup environment and API keys\n",
+    "import os\n",
+    "import asyncio\n",
+    "from google.adk.agents import Agent\n",
+    "from google.adk.models.lite_llm import LiteLlm  # For multi-model support\n",
+    "from google.adk.sessions import InMemorySessionService\n",
+    "from google.adk.runners import Runner\n",
+    "from google.genai import types\n",
+    "import litellm  # Import for proxy configuration\n",
+    "\n",
+    "# Set your API keys\n",
+    "os.environ['GOOGLE_API_KEY'] = 'your-google-api-key'  # For Gemini models\n",
+    "os.environ['OPENAI_API_KEY'] = 'your-openai-api-key'  # For OpenAI models\n",
+    "os.environ['ANTHROPIC_API_KEY'] = 'your-anthropic-api-key'  # For Claude models\n",
+    "\n",
+    "# Define model constants for cleaner code\n",
+    "MODEL_GEMINI_PRO = 'gemini-1.5-pro'\n",
+    "MODEL_GPT_4O = 'openai/gpt-4o'\n",
+    "MODEL_CLAUDE_SONNET = 'anthropic/claude-3-sonnet-20240229'"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "d2b1ed59",
+   "metadata": {},
+   "source": [
+    "## 2. Define a Simple Tool"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "04b3ef5b",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Weather tool implementation\n",
+    "def get_weather(city: str) -> dict:\n",
+    "    \"\"\"Retrieves the current weather report for a specified city.\"\"\"\n",
+    "    print(f'Tool: get_weather called for city: {city}')\n",
+    "\n",
+    "    # Mock weather data\n",
+    "    mock_weather_db = {\n",
+    "        'newyork': {\n",
+    "            'status': 'success',\n",
+    "            'report': 'The weather in New York is sunny with a temperature of 25°C.'\n",
+    "        },\n",
+    "        'london': {\n",
+    "            'status': 'success',\n",
+    "            'report': \"It's cloudy in London with a temperature of 15°C.\"\n",
+    "        },\n",
+    "        'tokyo': {\n",
+    "            'status': 'success',\n",
+    "            'report': 'Tokyo is experiencing light rain and a temperature of 18°C.'\n",
+    "        },\n",
+    "    }\n",
+    "\n",
+    "    city_normalized = city.lower().replace(' ', '')\n",
+    "\n",
+    "    if city_normalized in mock_weather_db:\n",
+    "        return mock_weather_db[city_normalized]\n",
+    "    else:\n",
+    "        return {\n",
+    "            'status': 'error',\n",
+    "            'error_message': f\"Sorry, I don't have weather information for '{city}'.\"\n",
+    "        }"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "727b15c9",
+   "metadata": {},
+   "source": [
+    "## 3. Helper Function for Agent Interaction"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "f77449bf",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Agent interaction helper function\n",
+    "async def call_agent_async(query: str, runner, user_id, session_id):\n",
+    "    \"\"\"Sends a query to the agent and prints the final response.\"\"\"\n",
+    "    print(f'\\n>>> User Query: {query}')\n",
+    "\n",
+    "    content = types.Content(role='user', parts=[types.Part(text=query)])\n",
+    "    final_response_text = 'Agent did not produce a final response.'\n",
+    "\n",
+    "    async for event in runner.run_async(\n",
+    "        user_id=user_id,\n",
+    "        session_id=session_id,\n",
+    "        new_message=content\n",
+    "    ):\n",
+    "        if event.is_final_response():\n",
+    "            if event.content and event.content.parts:\n",
+    "                final_response_text = event.content.parts[0].text\n",
+    "            break\n",
+    "    print(f'<<< Agent Response: {final_response_text}')"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "0ac87987",
+   "metadata": {},
+   "source": [
+    "## 4. Using Different Model Providers with ADK\n",
+    "\n",
+    "### 4.1 Using OpenAI Models"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "e167d557",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# OpenAI model implementation\n",
+    "weather_agent_gpt = Agent(\n",
+    "    name='weather_agent_gpt',\n",
+    "    model=LiteLlm(model=MODEL_GPT_4O),\n",
+    "    description='Provides weather information using OpenAI\\'s GPT.',\n",
+    "    instruction=(\n",
+    "        'You are a helpful weather assistant powered by GPT-4o. '\n",
+    "        \"Use the 'get_weather' tool for city weather requests. \"\n",
+    "        'Present information clearly.'\n",
+    "    ),\n",
+    "    tools=[get_weather],\n",
+    ")\n",
+    "\n",
+    "session_service_gpt = InMemorySessionService()\n",
+    "session_gpt = session_service_gpt.create_session(\n",
+    "    app_name='weather_app', user_id='user_1', session_id='session_gpt'\n",
+    ")\n",
+    "\n",
+    "runner_gpt = Runner(\n",
+    "    agent=weather_agent_gpt,\n",
+    "    app_name='weather_app',\n",
+    "    session_service=session_service_gpt,\n",
+    ")\n",
+    "\n",
+    "async def test_gpt_agent():\n",
+    "    print('\\n--- Testing GPT Agent ---')\n",
+    "    await call_agent_async(\n",
+    "        \"What's the weather in London?\",\n",
+    "        runner=runner_gpt,\n",
+    "        user_id='user_1',\n",
+    "        session_id='session_gpt',\n",
+    "    )\n",
+    "\n",
+    "# To execute in a notebook cell:\n",
+    "# await test_gpt_agent()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "f9cb0613",
+   "metadata": {},
+   "source": [
+    "### 4.2 Using Anthropic Models"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "1c653665",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Anthropic model implementation\n",
+    "weather_agent_claude = Agent(\n",
+    "    name='weather_agent_claude',\n",
+    "    model=LiteLlm(model=MODEL_CLAUDE_SONNET),\n",
+    "    description='Provides weather information using Anthropic\\'s Claude.',\n",
+    "    instruction=(\n",
+    "        'You are a helpful weather assistant powered by Claude Sonnet. '\n",
+    "        \"Use the 'get_weather' tool for city weather requests. \"\n",
+    "        'Present information clearly.'\n",
+    "    ),\n",
+    "    tools=[get_weather],\n",
+    ")\n",
+    "\n",
+    "session_service_claude = InMemorySessionService()\n",
+    "session_claude = session_service_claude.create_session(\n",
+    "    app_name='weather_app', user_id='user_1', session_id='session_claude'\n",
+    ")\n",
+    "\n",
+    "runner_claude = Runner(\n",
+    "    agent=weather_agent_claude,\n",
+    "    app_name='weather_app',\n",
+    "    session_service=session_service_claude,\n",
+    ")\n",
+    "\n",
+    "async def test_claude_agent():\n",
+    "    print('\\n--- Testing Claude Agent ---')\n",
+    "    await call_agent_async(\n",
+    "        \"What's the weather in Tokyo?\",\n",
+    "        runner=runner_claude,\n",
+    "        user_id='user_1',\n",
+    "        session_id='session_claude',\n",
+    "    )\n",
+    "\n",
+    "# To execute in a notebook cell:\n",
+    "# await test_claude_agent()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "bf9d863b",
+   "metadata": {},
+   "source": [
+    "### 4.3 Using Google's Gemini Models"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "83f49d0a",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Gemini model implementation\n",
+    "weather_agent_gemini = Agent(\n",
+    "    name='weather_agent_gemini',\n",
+    "    model=MODEL_GEMINI_PRO,\n",
+    "    description='Provides weather information using Google\\'s Gemini.',\n",
+    "    instruction=(\n",
+    "        'You are a helpful weather assistant powered by Gemini Pro. '\n",
+    "        \"Use the 'get_weather' tool for city weather requests. \"\n",
+    "        'Present information clearly.'\n",
+    "    ),\n",
+    "    tools=[get_weather],\n",
+    ")\n",
+    "\n",
+    "session_service_gemini = InMemorySessionService()\n",
+    "session_gemini = session_service_gemini.create_session(\n",
+    "    app_name='weather_app', user_id='user_1', session_id='session_gemini'\n",
+    ")\n",
+    "\n",
+    "runner_gemini = Runner(\n",
+    "    agent=weather_agent_gemini,\n",
+    "    app_name='weather_app',\n",
+    "    session_service=session_service_gemini,\n",
+    ")\n",
+    "\n",
+    "async def test_gemini_agent():\n",
+    "    print('\\n--- Testing Gemini Agent ---')\n",
+    "    await call_agent_async(\n",
+    "        \"What's the weather in New York?\",\n",
+    "        runner=runner_gemini,\n",
+    "        user_id='user_1',\n",
+    "        session_id='session_gemini',\n",
+    "    )\n",
+    "\n",
+    "# To execute in a notebook cell:\n",
+    "# await test_gemini_agent()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "93bc5fd0",
+   "metadata": {},
+   "source": [
+    "## 5. Using LiteLLM Proxy with ADK"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "b4275151",
+   "metadata": {},
+   "source": [
+    "| Variable | Description |\n",
+    "|----------|-------------|\n",
+    "| `LITELLM_PROXY_API_KEY` | The API key for the LiteLLM proxy |\n",
+    "| `LITELLM_PROXY_API_BASE` | The base URL for the LiteLLM proxy |\n",
+    "| `USE_LITELLM_PROXY` or `litellm.use_litellm_proxy` | When set to True, your request will be sent to LiteLLM proxy. |"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "256530a6",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# LiteLLM proxy integration\n",
+    "os.environ['LITELLM_PROXY_API_KEY'] = 'your-litellm-proxy-api-key'\n",
+    "os.environ['LITELLM_PROXY_API_BASE'] = 'your-litellm-proxy-url'  # e.g., 'http://localhost:4000'\n",
+    "litellm.use_litellm_proxy = True\n",
+    "\n",
+    "weather_agent_proxy_env = Agent(\n",
+    "    name='weather_agent_proxy_env',\n",
+    "    model=LiteLlm(model='gpt-4o'),\n",
+    "    description='Provides weather information using a model from LiteLLM proxy.',\n",
+    "    instruction=(\n",
+    "        'You are a helpful weather assistant. '\n",
+    "        \"Use the 'get_weather' tool for city weather requests. \"\n",
+    "        'Present information clearly.'\n",
+    "    ),\n",
+    "    tools=[get_weather],\n",
+    ")\n",
+    "\n",
+    "session_service_proxy_env = InMemorySessionService()\n",
+    "session_proxy_env = session_service_proxy_env.create_session(\n",
+    "    app_name='weather_app', user_id='user_1', session_id='session_proxy_env'\n",
+    ")\n",
+    "\n",
+    "runner_proxy_env = Runner(\n",
+    "    agent=weather_agent_proxy_env,\n",
+    "    app_name='weather_app',\n",
+    "    session_service=session_service_proxy_env,\n",
+    ")\n",
+    "\n",
+    "async def test_proxy_env_agent():\n",
+    "    print('\\n--- Testing Proxy-enabled Agent (Environment Variables) ---')\n",
+    "    await call_agent_async(\n",
+    "        \"What's the weather in London?\",\n",
+    "        runner=runner_proxy_env,\n",
+    "        user_id='user_1',\n",
+    "        session_id='session_proxy_env',\n",
+    "    )\n",
+    "\n",
+    "# To execute in a notebook cell:\n",
+    "# await test_proxy_env_agent()"
+   ]
+  }
+ ],
+ "metadata": {
+  "language_info": {
+   "name": "python"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

cookbook/litellm_proxy_server/grafana_dashboard/dashboard_1/grafana_dashboard.json

@@ -32,7 +32,7 @@
       {
         "datasource": {
           "type": "prometheus",
-          "uid": "rMzWaBvIk"
+          "uid": "${DS_PROMETHEUS}"
         },
         "fieldConfig": {
           "defaults": {
@@ -110,7 +110,7 @@
           {
             "datasource": {
               "type": "prometheus",
-              "uid": "rMzWaBvIk"
+              "uid": "${DS_PROMETHEUS}"
             },
             "editorMode": "code",
             "expr": "histogram_quantile(0.99, sum(rate(litellm_self_latency_bucket{self=\"self\"}[1m])) by (le))",
@@ -125,7 +125,7 @@
       {
         "datasource": {
           "type": "prometheus",
-          "uid": "rMzWaBvIk"
+          "uid": "${DS_PROMETHEUS}"
         },
         "fieldConfig": {
           "defaults": {
@@ -216,7 +216,7 @@
           {
             "datasource": {
               "type": "prometheus",
-              "uid": "rMzWaBvIk"
+              "uid": "${DS_PROMETHEUS}"
             },
             "editorMode": "code",
             "expr": "sum(increase(litellm_spend_metric_total[30d])) by (hashed_api_key)",
@@ -232,7 +232,7 @@
       {
         "datasource": {
           "type": "prometheus",
-          "uid": "rMzWaBvIk"
+          "uid": "${DS_PROMETHEUS}"
         },
         "fieldConfig": {
           "defaults": {
@@ -309,7 +309,7 @@
           {
             "datasource": {
               "type": "prometheus",
-              "uid": "rMzWaBvIk"
+              "uid": "${DS_PROMETHEUS}"
             },
             "editorMode": "code",
             "expr": "sum by (model) (increase(litellm_requests_metric_total[5m]))",
@@ -324,7 +324,7 @@
       {
         "datasource": {
           "type": "prometheus",
-          "uid": "rMzWaBvIk"
+          "uid": "${DS_PROMETHEUS}"
         },
         "fieldConfig": {
           "defaults": {
@@ -375,7 +375,7 @@
           {
             "datasource": {
               "type": "prometheus",
-              "uid": "rMzWaBvIk"
+              "uid": "${DS_PROMETHEUS}"
             },
             "editorMode": "code",
             "expr": "sum(increase(litellm_llm_api_failed_requests_metric_total[1h]))",
@@ -390,7 +390,7 @@
       {
         "datasource": {
           "type": "prometheus",
-          "uid": "rMzWaBvIk"
+          "uid": "${DS_PROMETHEUS}"
         },
         "fieldConfig": {
           "defaults": {
@@ -468,7 +468,7 @@
           {
             "datasource": {
               "type": "prometheus",
-              "uid": "rMzWaBvIk"
+              "uid": "${DS_PROMETHEUS}"
             },
             "editorMode": "code",
             "expr": "sum(increase(litellm_spend_metric_total[30d])) by (model)",
@@ -483,7 +483,7 @@
       {
         "datasource": {
           "type": "prometheus",
-          "uid": "rMzWaBvIk"
+          "uid": "${DS_PROMETHEUS}"
         },
         "fieldConfig": {
           "defaults": {
@@ -560,7 +560,7 @@
           {
             "datasource": {
               "type": "prometheus",
-              "uid": "rMzWaBvIk"
+              "uid": "${DS_PROMETHEUS}"
             },
             "editorMode": "code",
             "expr": "sum(increase(litellm_total_tokens_total[5m])) by (model)",
@@ -579,7 +579,27 @@
     "style": "dark",
     "tags": [],
     "templating": {
-      "list": []
+      "list": [
+        {
+          "current": {
+            "selected": false,
+            "text": "prometheus",
+            "value": "edx8memhpd9tsa"
+          },
+          "hide": 0,
+          "includeAll": false,
+          "label": "datasource",
+          "multi": false,
+          "name": "DS_PROMETHEUS",
+          "options": [],
+          "query": "prometheus",
+          "queryValue": "",
+          "refresh": 1,
+          "regex": "",
+          "skipUrlSync": false,
+          "type": "datasource"
+        }
+      ]
     },
     "time": {
       "from": "now-1h",

cookbook/litellm_proxy_server/grafana_dashboard/dashboard_v2/grafana_dashboard.json

@@ -37,7 +37,7 @@
       {
         "datasource": {
           "type": "prometheus",
-          "uid": "bdiyc60dco54we"
+          "uid": "${DS_PROMETHEUS}"
         },
         "description": "Total requests per second made to proxy - success + failure ",
         "fieldConfig": {
@@ -119,7 +119,7 @@
           {
             "datasource": {
               "type": "prometheus",
-              "uid": "bdiyc60dco54we"
+              "uid": "${DS_PROMETHEUS}"
             },
             "disableTextWrap": false,
             "editorMode": "code",
@@ -138,7 +138,7 @@
       {
         "datasource": {
           "type": "prometheus",
-          "uid": "bdiyc60dco54we"
+          "uid": "${DS_PROMETHEUS}"
         },
         "description": "Failures per second by Exception Class",
         "fieldConfig": {
@@ -220,7 +220,7 @@
           {
             "datasource": {
               "type": "prometheus",
-              "uid": "bdiyc60dco54we"
+              "uid": "${DS_PROMETHEUS}"
             },
             "disableTextWrap": false,
             "editorMode": "code",
@@ -239,7 +239,7 @@
       {
         "datasource": {
           "type": "prometheus",
-          "uid": "bdiyc60dco54we"
+          "uid": "${DS_PROMETHEUS}"
         },
         "description": "Average Response latency (seconds)",
         "fieldConfig": {
@@ -346,7 +346,7 @@
           {
             "datasource": {
               "type": "prometheus",
-              "uid": "bdiyc60dco54we"
+              "uid": "${DS_PROMETHEUS}"
             },
             "disableTextWrap": false,
             "editorMode": "code",
@@ -361,7 +361,7 @@
           {
             "datasource": {
               "type": "prometheus",
-              "uid": "bdiyc60dco54we"
+              "uid": "${DS_PROMETHEUS}"
             },
             "editorMode": "code",
             "expr": "histogram_quantile(0.5, sum(rate(litellm_request_total_latency_metric_bucket[2m])) by (le))",
@@ -391,7 +391,7 @@
       {
         "datasource": {
           "type": "prometheus",
-          "uid": "bdiyc60dco54we"
+          "uid": "${DS_PROMETHEUS}"
         },
         "description": "x-ratelimit-remaining-requests returning from LLM APIs",
         "fieldConfig": {
@@ -473,7 +473,7 @@
           {
             "datasource": {
               "type": "prometheus",
-              "uid": "bdiyc60dco54we"
+              "uid": "${DS_PROMETHEUS}"
             },
             "editorMode": "code",
             "expr": "topk(5, sort(litellm_remaining_requests))",
@@ -488,7 +488,7 @@
       {
         "datasource": {
           "type": "prometheus",
-          "uid": "bdiyc60dco54we"
+          "uid": "${DS_PROMETHEUS}"
         },
         "description": "x-ratelimit-remaining-tokens from LLM API ",
         "fieldConfig": {
@@ -570,7 +570,7 @@
           {
             "datasource": {
               "type": "prometheus",
-              "uid": "bdiyc60dco54we"
+              "uid": "${DS_PROMETHEUS}"
             },
             "editorMode": "code",
             "expr": "topk(5, sort(litellm_remaining_tokens))",
@@ -598,7 +598,7 @@
       {
         "datasource": {
           "type": "prometheus",
-          "uid": "bdiyc60dco54we"
+          "uid": "${DS_PROMETHEUS}"
         },
         "description": "Requests per second by Key Alias (keys are LiteLLM Virtual Keys). If key is None - means no Alias Set ",
         "fieldConfig": {
@@ -679,7 +679,7 @@
           {
             "datasource": {
               "type": "prometheus",
-              "uid": "bdiyc60dco54we"
+              "uid": "${DS_PROMETHEUS}"
             },
             "editorMode": "code",
             "expr": "sum(rate(litellm_proxy_total_requests_metric_total[2m])) by (api_key_alias)\n",
@@ -694,7 +694,7 @@
       {
         "datasource": {
           "type": "prometheus",
-          "uid": "bdiyc60dco54we"
+          "uid": "${DS_PROMETHEUS}"
         },
         "description": "Requests per second by Team Alias. If team is None - means no team alias Set ",
         "fieldConfig": {
@@ -775,7 +775,7 @@
           {
             "datasource": {
               "type": "prometheus",
-              "uid": "bdiyc60dco54we"
+              "uid": "${DS_PROMETHEUS}"
             },
             "editorMode": "code",
             "expr": "sum(rate(litellm_proxy_total_requests_metric_total[2m])) by (team_alias)\n",
@@ -792,7 +792,27 @@
     "schemaVersion": 40,
     "tags": [],
     "templating": {
-      "list": []
+      "list": [
+        {
+          "current": {
+            "selected": false,
+            "text": "prometheus",
+            "value": "edx8memhpd9tsb"
+          },
+          "hide": 0,
+          "includeAll": false,
+          "label": "datasource",
+          "multi": false,
+          "name": "DS_PROMETHEUS",
+          "options": [],
+          "query": "prometheus",
+          "queryValue": "",
+          "refresh": 1,
+          "regex": "",
+          "skipUrlSync": false,
+          "type": "datasource"
+        }
+      ]
     },
     "time": {
       "from": "now-6h",

deploy/charts/litellm-helm/Chart.yaml

@@ -18,7 +18,7 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.4.3
+version: 0.4.4
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to

deploy/charts/litellm-helm/README.md

@@ -34,6 +34,7 @@ If `db.useStackgresOperator` is used (not yet implemented):
 | `serviceAccount.create`                                    | Whether or not to create a Kubernetes Service Account for this deployment.  The default is `false` because LiteLLM has no need to access the Kubernetes API.                          | `false`  |
 | `service.type`                                             | Kubernetes Service type (e.g. `LoadBalancer`, `ClusterIP`, etc.)                                                                                                                      | `ClusterIP`  |
 | `service.port`                                             | TCP port that the Kubernetes Service will listen on.  Also the TCP port within the Pod that the proxy will listen on.                                                                 | `4000`  |
+| `service.loadBalancerClass`                                | Optional LoadBalancer implementation class (only used when `service.type` is `LoadBalancer`)                                                                                                          | `""`  |
 | `ingress.*`                                                | See [values.yaml](./values.yaml) for example settings                                                                                                                                 | N/A  |
 | `proxy_config.*`                                           | See [values.yaml](./values.yaml) for default settings.  See [example_config_yaml](../../../litellm/proxy/example_config_yaml/) for configuration examples.                            | N/A  |
 | `extraContainers[]`                                        | An array of additional containers to be deployed as sidecars alongside the LiteLLM Proxy.                                                                                             | `[]`  |

deploy/charts/litellm-helm/templates/deployment.yaml

@@ -1,6 +1,8 @@
 apiVersion: apps/v1
 kind: Deployment
 metadata:
+  annotations:
+  {{- toYaml .Values.deploymentAnnotations | nindent 4 }}
   name: {{ include "litellm.fullname" . }}
   labels:
     {{- include "litellm.labels" . | nindent 4 }}

deploy/charts/litellm-helm/templates/migrations-job.yaml

@@ -53,6 +53,9 @@ spec:
           volumeMounts:
             {{- toYaml . | nindent 12 }}
           {{- end }}
+      {{- with .Values.migrationJob.extraContainers }}
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
       {{- with .Values.volumes }}
       volumes:
         {{- toYaml . | nindent 8 }}

deploy/charts/litellm-helm/templates/service.yaml

@@ -10,6 +10,9 @@ metadata:
     {{- include "litellm.labels" . | nindent 4 }}
 spec:
   type: {{ .Values.service.type }}
+  {{- if and (eq .Values.service.type "LoadBalancer") .Values.service.loadBalancerClass }}
+  loadBalancerClass: {{ .Values.service.loadBalancerClass }}
+  {{- end }}
   ports:
     - port: {{ .Values.service.port }}
       targetPort: http

deploy/charts/litellm-helm/tests/service_tests.yaml

@@ -0,0 +1,116 @@
+suite: Service Configuration Tests
+templates:
+  - service.yaml
+tests:
+  - it: should create a default ClusterIP service
+    template: service.yaml
+    asserts:
+      - isKind:
+          of: Service
+      - equal:
+          path: spec.type
+          value: ClusterIP
+      - equal:
+          path: spec.ports[0].port
+          value: 4000
+      - equal:
+          path: spec.ports[0].targetPort
+          value: http
+      - equal:
+          path: spec.ports[0].protocol
+          value: TCP
+      - equal:
+          path: spec.ports[0].name
+          value: http
+      - isNull:
+          path: spec.loadBalancerClass
+
+  - it: should create a NodePort service when specified
+    template: service.yaml
+    set:
+      service.type: NodePort
+    asserts:
+      - isKind:
+          of: Service
+      - equal:
+          path: spec.type
+          value: NodePort
+      - isNull:
+          path: spec.loadBalancerClass
+
+  - it: should create a LoadBalancer service when specified
+    template: service.yaml
+    set:
+      service.type: LoadBalancer
+    asserts:
+      - isKind:
+          of: Service
+      - equal:
+          path: spec.type
+          value: LoadBalancer
+      - isNull:
+          path: spec.loadBalancerClass
+
+  - it: should add loadBalancerClass when specified with LoadBalancer type
+    template: service.yaml
+    set:
+      service.type: LoadBalancer
+      service.loadBalancerClass: tailscale
+    asserts:
+      - isKind:
+          of: Service
+      - equal:
+          path: spec.type
+          value: LoadBalancer
+      - equal:
+          path: spec.loadBalancerClass
+          value: tailscale
+
+  - it: should not add loadBalancerClass when specified with ClusterIP type
+    template: service.yaml
+    set:
+      service.type: ClusterIP
+      service.loadBalancerClass: tailscale
+    asserts:
+      - isKind:
+          of: Service
+      - equal:
+          path: spec.type
+          value: ClusterIP
+      - isNull:
+          path: spec.loadBalancerClass
+
+  - it: should use custom port when specified
+    template: service.yaml
+    set:
+      service.port: 8080
+    asserts:
+      - equal:
+          path: spec.ports[0].port
+          value: 8080
+
+  - it: should add service annotations when specified
+    template: service.yaml
+    set:
+      service.annotations:
+        cloud.google.com/load-balancer-type: "Internal"
+        service.beta.kubernetes.io/aws-load-balancer-internal: "true"
+    asserts:
+      - isKind:
+          of: Service
+      - equal:
+          path: metadata.annotations
+          value:
+            cloud.google.com/load-balancer-type: "Internal"
+            service.beta.kubernetes.io/aws-load-balancer-internal: "true"
+
+  - it: should use the correct selector labels
+    template: service.yaml
+    asserts:
+      - isNotNull:
+          path: spec.selector
+      - equal:
+          path: spec.selector
+          value:
+            app.kubernetes.io/name: litellm
+            app.kubernetes.io/instance: RELEASE-NAME

deploy/charts/litellm-helm/values.yaml

@@ -27,6 +27,9 @@ serviceAccount:
   # If not set and create is true, a name is generated using the fullname template
   name: ""
 
+# annotations for litellm deployment
+deploymentAnnotations: {}
+# annotations for litellm pods
 podAnnotations: {}
 podLabels: {}
 
@@ -56,6 +59,9 @@ environmentConfigMaps: []
 service:
   type: ClusterIP
   port: 4000
+  # If service type is `LoadBalancer` you can
+  # optionally specify loadBalancerClass
+  # loadBalancerClass: tailscale
 
 ingress:
   enabled: false
@@ -194,6 +200,7 @@ migrationJob:
   disableSchemaUpdate: false # Skip schema migrations for specific environments. When True, the job will exit with code 0.
   annotations: {}
   ttlSecondsAfterFinished: 120
+  extraContainers: []
 
 # Additional environment variables to be added to the deployment as a map of key-value pairs
 envVars: {

docker-compose.yml

@@ -16,23 +16,23 @@ services:
     ports:
       - "4000:4000" # Map the container port to the host, change the host port if necessary
     environment:
-        DATABASE_URL: "postgresql://llmproxy:dbpassword9090@db:5432/litellm"
-        STORE_MODEL_IN_DB: "True" # allows adding models to proxy via UI
+      DATABASE_URL: "postgresql://llmproxy:dbpassword9090@db:5432/litellm"
+      STORE_MODEL_IN_DB: "True" # allows adding models to proxy via UI
     env_file:
       - .env # Load local .env file
     depends_on:
       - db  # Indicates that this service depends on the 'db' service, ensuring 'db' starts first
     healthcheck:  # Defines the health check configuration for the container
-      test: [ "CMD", "curl", "-f", "http://localhost:4000/health/liveliness || exit 1" ]  # Command to execute for health check
+      test: [ "CMD-SHELL", "wget --no-verbose --tries=1 http://localhost:4000/health/liveliness || exit 1" ]  # Command to execute for health check
       interval: 30s  # Perform health check every 30 seconds
       timeout: 10s   # Health check command times out after 10 seconds
       retries: 3     # Retry up to 3 times if health check fails
       start_period: 40s  # Wait 40 seconds after container start before beginning health checks
 
- 
   db:
     image: postgres:16
     restart: always
+    container_name: litellm_db
     environment:
       POSTGRES_DB: litellm
       POSTGRES_USER: llmproxy
@@ -40,13 +40,13 @@ services:
     ports:
       - "5432:5432"
     volumes:
-      - postgres_data:/var/lib/postgresql/data  # Persists Postgres data across container restarts
+      - postgres_data:/var/lib/postgresql/data # Persists Postgres data across container restarts
     healthcheck:
       test: ["CMD-SHELL", "pg_isready -d litellm -U llmproxy"]
       interval: 1s
       timeout: 5s
       retries: 10
-  
+
   prometheus:
     image: prom/prometheus
     volumes:
@@ -55,14 +55,14 @@ services:
     ports:
       - "9090:9090"
     command:
-      - '--config.file=/etc/prometheus/prometheus.yml'
-      - '--storage.tsdb.path=/prometheus'
-      - '--storage.tsdb.retention.time=15d'
+      - "--config.file=/etc/prometheus/prometheus.yml"
+      - "--storage.tsdb.path=/prometheus"
+      - "--storage.tsdb.retention.time=15d"
     restart: always
 
 volumes:
   prometheus_data:
     driver: local
   postgres_data:
-    name: litellm_postgres_data  # Named volume for Postgres data persistence
+    name: litellm_postgres_data # Named volume for Postgres data persistence
 

docker/Dockerfile.dev

@@ -0,0 +1,87 @@
+# Base image for building
+ARG LITELLM_BUILD_IMAGE=python:3.11-slim
+
+# Runtime image
+ARG LITELLM_RUNTIME_IMAGE=python:3.11-slim
+
+# Builder stage
+FROM $LITELLM_BUILD_IMAGE AS builder
+
+# Set the working directory to /app
+WORKDIR /app
+
+USER root
+
+# Install build dependencies in one layer
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    gcc \
+    python3-dev \
+    libssl-dev \
+    pkg-config \
+    && rm -rf /var/lib/apt/lists/* \
+    && pip install --upgrade pip build
+
+# Copy requirements first for better layer caching
+COPY requirements.txt .
+
+# Install Python dependencies with cache mount for faster rebuilds
+RUN --mount=type=cache,target=/root/.cache/pip \
+    pip wheel --no-cache-dir --wheel-dir=/wheels/ -r requirements.txt
+
+# Fix JWT dependency conflicts early
+RUN pip uninstall jwt -y || true && \
+    pip uninstall PyJWT -y || true && \
+    pip install PyJWT==2.9.0 --no-cache-dir
+
+# Copy only necessary files for build
+COPY pyproject.toml README.md schema.prisma poetry.lock ./
+COPY litellm/ ./litellm/
+COPY enterprise/ ./enterprise/
+COPY docker/ ./docker/
+
+# Build Admin UI once
+RUN chmod +x docker/build_admin_ui.sh && ./docker/build_admin_ui.sh
+
+# Build the package
+RUN rm -rf dist/* && python -m build
+
+# Install the built package
+RUN pip install dist/*.whl
+
+# Runtime stage
+FROM $LITELLM_RUNTIME_IMAGE AS runtime
+
+# Ensure runtime stage runs as root
+USER root
+
+# Install only runtime dependencies
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    libssl3 \
+    && rm -rf /var/lib/apt/lists/*
+
+WORKDIR /app
+
+# Copy only necessary runtime files
+COPY docker/entrypoint.sh docker/prod_entrypoint.sh ./docker/
+COPY litellm/ ./litellm/
+COPY pyproject.toml README.md schema.prisma poetry.lock ./
+
+# Copy pre-built wheels and install everything at once
+COPY --from=builder /wheels/ /wheels/
+COPY --from=builder /app/dist/*.whl .
+
+# Install all dependencies in one step with no-cache for smaller image
+RUN pip install --no-cache-dir *.whl /wheels/* --no-index --find-links=/wheels/ && \
+    rm -f *.whl && \
+    rm -rf /wheels
+
+# Generate prisma client and set permissions
+RUN prisma generate && \
+    chmod +x docker/entrypoint.sh docker/prod_entrypoint.sh
+
+EXPOSE 4000/tcp
+
+ENTRYPOINT ["docker/prod_entrypoint.sh"]
+
+# Append "--detailed_debug" to the end of CMD to view detailed debug logs 
+CMD ["--port", "4000"]

docker/build_from_pip/Dockerfile.build_from_pip

@@ -13,10 +13,16 @@ RUN apk update && \
 RUN python -m venv ${HOME}/venv
 RUN ${HOME}/venv/bin/pip install --no-cache-dir --upgrade pip
 
-COPY requirements.txt .
+COPY docker/build_from_pip/requirements.txt .
 RUN --mount=type=cache,target=${HOME}/.cache/pip \
     ${HOME}/venv/bin/pip install -r requirements.txt
 
+# Copy Prisma schema file
+COPY schema.prisma .
+
+# Generate prisma client
+RUN prisma generate
+
 EXPOSE 4000/tcp
 
 ENTRYPOINT ["litellm"]

docker/build_from_pip/requirements.txt

@@ -1,4 +1,4 @@
-litellm[proxy] # Specify the litellm version you want to use
+litellm[proxy]==1.67.4.dev1 # Specify the litellm version you want to use
 prometheus_client
 langfuse
 prisma

docs/my-website/.gitignore

@@ -10,6 +10,7 @@
 
 # Misc
 .DS_Store
+.env
 .env.local
 .env.development.local
 .env.test.local
@@ -19,3 +20,4 @@ npm-debug.log*
 yarn-debug.log*
 yarn-error.log*
 yarn.lock
+pnpm-lock.yaml

docs/my-website/docs/aiohttp_benchmarks.md

@@ -0,0 +1,38 @@
+# LiteLLM v1.71.1 Benchmarks
+
+## Overview
+
+This document presents performance benchmarks comparing LiteLLM's v1.71.1 to prior litellm versions.
+
+**Related PR:** [#11097](https://github.com/BerriAI/litellm/pull/11097)
+
+## Testing Methodology
+
+The load testing was conducted using the following parameters:
+- **Request Rate:** 200 RPS (Requests Per Second)
+- **User Ramp Up:** 200 concurrent users
+- **Transport Comparison:** httpx (existing) vs aiohttp (new implementation)
+- **Number of pods/instance of litellm:** 1
+- **Machine Specs:** 2 vCPUs, 4GB RAM
+- **LiteLLM Settings:**
+    - Tested against a [fake openai endpoint](https://exampleopenaiendpoint-production.up.railway.app/)
+    - Set `USE_AIOHTTP_TRANSPORT="True"` in the environment variables. This feature flag enables the aiohttp transport.
+
+
+## Benchmark Results
+
+| Metric | httpx (Existing) | aiohttp (LiteLLM v1.71.1) | Improvement | Calculation |
+|--------|------------------|-------------------|-------------|-------------|
+| **RPS** | 50.2 | 224 | **+346%** ✅ | (224 - 50.2) / 50.2 × 100 = 346% |
+| **Median Latency** | 2,500ms | 74ms | **-97%** ✅ | (74 - 2500) / 2500 × 100 = -97% |
+| **95th Percentile** | 5,600ms | 250ms | **-96%** ✅ | (250 - 5600) / 5600 × 100 = -96% |
+| **99th Percentile** | 6,200ms | 330ms | **-95%** ✅ | (330 - 6200) / 6200 × 100 = -95% |
+
+## Key Improvements
+
+- **4.5x increase** in requests per second (from 50.2 to 224 RPS)
+- **97% reduction** in median response time (from 2.5 seconds to 74ms)
+- **96% reduction** in 95th percentile latency (from 5.6 seconds to 250ms)
+- **95% reduction** in 99th percentile latency (from 6.2 seconds to 330ms)
+
+

docs/my-website/docs/anthropic_unified.md

@@ -1,7 +1,7 @@
 import Tabs from '@theme/Tabs';
 import TabItem from '@theme/TabItem';
 
-# /v1/messages [BETA] 
+# /v1/messages
 
 Use LiteLLM to call all your LLM APIs in the Anthropic `v1/messages` format. 
 
@@ -14,20 +14,20 @@ Use LiteLLM to call all your LLM APIs in the Anthropic `v1/messages` format.
 | Logging | ✅ | works across all integrations |
 | End-user Tracking | ✅ | |
 | Streaming | ✅ | |
-| Fallbacks | ✅ | between anthropic models |
-| Loadbalancing | ✅ | between anthropic models |
-
-Planned improvement:
-- Vertex AI Anthropic support
-- Bedrock Anthropic support
+| Fallbacks | ✅ | between supported models |
+| Loadbalancing | ✅ | between supported models |
+| Support llm providers | **All LiteLLM supported providers** | `openai`, `anthropic`, `bedrock`, `vertex_ai`, `gemini`, `azure`, `azure_ai`, etc. |
 
 ## Usage 
 ---
 
 ### LiteLLM Python SDK 
 
+<Tabs>
+<TabItem value="anthropic" label="Anthropic">
+
 #### Non-streaming example
-```python showLineNumbers title="Example using LiteLLM Python SDK"
+```python showLineNumbers title="Anthropic Example using LiteLLM Python SDK"
 import litellm
 response = await litellm.anthropic.messages.acreate(
     messages=[{"role": "user", "content": "Hello, can you tell me a short joke?"}],
@@ -37,6 +37,179 @@ response = await litellm.anthropic.messages.acreate(
 )
 ```
 
+#### Streaming example
+```python showLineNumbers title="Anthropic Streaming Example using LiteLLM Python SDK"
+import litellm
+response = await litellm.anthropic.messages.acreate(
+    messages=[{"role": "user", "content": "Hello, can you tell me a short joke?"}],
+    api_key=api_key,
+    model="anthropic/claude-3-haiku-20240307",
+    max_tokens=100,
+    stream=True,
+)
+async for chunk in response:
+    print(chunk)
+```
+
+</TabItem>
+
+<TabItem value="openai" label="OpenAI">
+
+#### Non-streaming example
+```python showLineNumbers title="OpenAI Example using LiteLLM Python SDK"
+import litellm
+import os
+
+# Set API key
+os.environ["OPENAI_API_KEY"] = "your-openai-api-key"
+
+response = await litellm.anthropic.messages.acreate(
+    messages=[{"role": "user", "content": "Hello, can you tell me a short joke?"}],
+    model="openai/gpt-4",
+    max_tokens=100,
+)
+```
+
+#### Streaming example
+```python showLineNumbers title="OpenAI Streaming Example using LiteLLM Python SDK"
+import litellm
+import os
+
+# Set API key
+os.environ["OPENAI_API_KEY"] = "your-openai-api-key"
+
+response = await litellm.anthropic.messages.acreate(
+    messages=[{"role": "user", "content": "Hello, can you tell me a short joke?"}],
+    model="openai/gpt-4",
+    max_tokens=100,
+    stream=True,
+)
+async for chunk in response:
+    print(chunk)
+```
+
+</TabItem>
+
+<TabItem value="gemini" label="Google AI Studio">
+
+#### Non-streaming example
+```python showLineNumbers title="Google Gemini Example using LiteLLM Python SDK"
+import litellm
+import os
+
+# Set API key
+os.environ["GEMINI_API_KEY"] = "your-gemini-api-key"
+
+response = await litellm.anthropic.messages.acreate(
+    messages=[{"role": "user", "content": "Hello, can you tell me a short joke?"}],
+    model="gemini/gemini-2.0-flash-exp",
+    max_tokens=100,
+)
+```
+
+#### Streaming example
+```python showLineNumbers title="Google Gemini Streaming Example using LiteLLM Python SDK"
+import litellm
+import os
+
+# Set API key
+os.environ["GEMINI_API_KEY"] = "your-gemini-api-key"
+
+response = await litellm.anthropic.messages.acreate(
+    messages=[{"role": "user", "content": "Hello, can you tell me a short joke?"}],
+    model="gemini/gemini-2.0-flash-exp",
+    max_tokens=100,
+    stream=True,
+)
+async for chunk in response:
+    print(chunk)
+```
+
+</TabItem>
+
+<TabItem value="vertex" label="Vertex AI">
+
+#### Non-streaming example
+```python showLineNumbers title="Vertex AI Example using LiteLLM Python SDK"
+import litellm
+import os
+
+# Set credentials - Vertex AI uses application default credentials
+# Run 'gcloud auth application-default login' to authenticate
+os.environ["VERTEXAI_PROJECT"] = "your-gcp-project-id"
+os.environ["VERTEXAI_LOCATION"] = "us-central1"
+
+response = await litellm.anthropic.messages.acreate(
+    messages=[{"role": "user", "content": "Hello, can you tell me a short joke?"}],
+    model="vertex_ai/gemini-2.0-flash-exp",
+    max_tokens=100,
+)
+```
+
+#### Streaming example
+```python showLineNumbers title="Vertex AI Streaming Example using LiteLLM Python SDK"
+import litellm
+import os
+
+# Set credentials - Vertex AI uses application default credentials
+# Run 'gcloud auth application-default login' to authenticate
+os.environ["VERTEXAI_PROJECT"] = "your-gcp-project-id"
+os.environ["VERTEXAI_LOCATION"] = "us-central1"
+
+response = await litellm.anthropic.messages.acreate(
+    messages=[{"role": "user", "content": "Hello, can you tell me a short joke?"}],
+    model="vertex_ai/gemini-2.0-flash-exp",
+    max_tokens=100,
+    stream=True,
+)
+async for chunk in response:
+    print(chunk)
+```
+
+</TabItem>
+
+<TabItem value="bedrock" label="AWS Bedrock">
+
+#### Non-streaming example
+```python showLineNumbers title="AWS Bedrock Example using LiteLLM Python SDK"
+import litellm
+import os
+
+# Set AWS credentials
+os.environ["AWS_ACCESS_KEY_ID"] = "your-access-key-id"
+os.environ["AWS_SECRET_ACCESS_KEY"] = "your-secret-access-key"
+os.environ["AWS_REGION_NAME"] = "us-west-2"  # or your AWS region
+
+response = await litellm.anthropic.messages.acreate(
+    messages=[{"role": "user", "content": "Hello, can you tell me a short joke?"}],
+    model="bedrock/anthropic.claude-3-sonnet-20240229-v1:0",
+    max_tokens=100,
+)
+```
+
+#### Streaming example
+```python showLineNumbers title="AWS Bedrock Streaming Example using LiteLLM Python SDK"
+import litellm
+import os
+
+# Set AWS credentials
+os.environ["AWS_ACCESS_KEY_ID"] = "your-access-key-id"
+os.environ["AWS_SECRET_ACCESS_KEY"] = "your-secret-access-key"
+os.environ["AWS_REGION_NAME"] = "us-west-2"  # or your AWS region
+
+response = await litellm.anthropic.messages.acreate(
+    messages=[{"role": "user", "content": "Hello, can you tell me a short joke?"}],
+    model="bedrock/anthropic.claude-3-sonnet-20240229-v1:0",
+    max_tokens=100,
+    stream=True,
+)
+async for chunk in response:
+    print(chunk)
+```
+
+</TabItem>
+</Tabs>
+
 Example response:
 ```json
 {
@@ -61,22 +234,10 @@ Example response:
 }
 ```
 
-#### Streaming example
-```python showLineNumbers title="Example using LiteLLM Python SDK"
-import litellm
-response = await litellm.anthropic.messages.acreate(
-    messages=[{"role": "user", "content": "Hello, can you tell me a short joke?"}],
-    api_key=api_key,
-    model="anthropic/claude-3-haiku-20240307",
-    max_tokens=100,
-    stream=True,
-)
-async for chunk in response:
-    print(chunk)
-```
-
 ### LiteLLM Proxy Server 
 
+<Tabs>
+<TabItem value="anthropic-proxy" label="Anthropic">
 
 1. Setup config.yaml
 
@@ -85,6 +246,7 @@ model_list:
     - model_name: anthropic-claude
       litellm_params:
         model: claude-3-7-sonnet-latest
+        api_key: os.environ/ANTHROPIC_API_KEY
 ```
 
 2. Start proxy 
@@ -95,10 +257,7 @@ litellm --config /path/to/config.yaml
 
 3. Test it! 
 
-<Tabs>
-<TabItem label="Anthropic Python SDK" value="python">
-
-```python showLineNumbers title="Example using LiteLLM Proxy Server"
+```python showLineNumbers title="Anthropic Example using LiteLLM Proxy Server"
 import anthropic
 
 # point anthropic sdk to litellm proxy 
@@ -113,8 +272,165 @@ response = client.messages.create(
     max_tokens=100,
 )
 ```
+
 </TabItem>
-<TabItem label="curl" value="curl">
+
+<TabItem value="openai-proxy" label="OpenAI">
+
+1. Setup config.yaml
+
+```yaml
+model_list:
+    - model_name: openai-gpt4
+      litellm_params:
+        model: openai/gpt-4
+        api_key: os.environ/OPENAI_API_KEY
+```
+
+2. Start proxy 
+
+```bash
+litellm --config /path/to/config.yaml
+```
+
+3. Test it! 
+
+```python showLineNumbers title="OpenAI Example using LiteLLM Proxy Server"
+import anthropic
+
+# point anthropic sdk to litellm proxy 
+client = anthropic.Anthropic(
+    base_url="http://0.0.0.0:4000",
+    api_key="sk-1234",
+)
+
+response = client.messages.create(
+    messages=[{"role": "user", "content": "Hello, can you tell me a short joke?"}],
+    model="openai-gpt4",
+    max_tokens=100,
+)
+```
+
+</TabItem>
+
+<TabItem value="gemini-proxy" label="Google AI Studio">
+
+1. Setup config.yaml
+
+```yaml
+model_list:
+    - model_name: gemini-2-flash
+      litellm_params:
+        model: gemini/gemini-2.0-flash-exp
+        api_key: os.environ/GEMINI_API_KEY
+```
+
+2. Start proxy 
+
+```bash
+litellm --config /path/to/config.yaml
+```
+
+3. Test it! 
+
+```python showLineNumbers title="Google Gemini Example using LiteLLM Proxy Server"
+import anthropic
+
+# point anthropic sdk to litellm proxy 
+client = anthropic.Anthropic(
+    base_url="http://0.0.0.0:4000",
+    api_key="sk-1234",
+)
+
+response = client.messages.create(
+    messages=[{"role": "user", "content": "Hello, can you tell me a short joke?"}],
+    model="gemini-2-flash",
+    max_tokens=100,
+)
+```
+
+</TabItem>
+
+<TabItem value="vertex-proxy" label="Vertex AI">
+
+1. Setup config.yaml
+
+```yaml
+model_list:
+    - model_name: vertex-gemini
+      litellm_params:
+        model: vertex_ai/gemini-2.0-flash-exp
+        vertex_project: your-gcp-project-id
+        vertex_location: us-central1
+```
+
+2. Start proxy 
+
+```bash
+litellm --config /path/to/config.yaml
+```
+
+3. Test it! 
+
+```python showLineNumbers title="Vertex AI Example using LiteLLM Proxy Server"
+import anthropic
+
+# point anthropic sdk to litellm proxy 
+client = anthropic.Anthropic(
+    base_url="http://0.0.0.0:4000",
+    api_key="sk-1234",
+)
+
+response = client.messages.create(
+    messages=[{"role": "user", "content": "Hello, can you tell me a short joke?"}],
+    model="vertex-gemini",
+    max_tokens=100,
+)
+```
+
+</TabItem>
+
+<TabItem value="bedrock-proxy" label="AWS Bedrock">
+
+1. Setup config.yaml
+
+```yaml
+model_list:
+    - model_name: bedrock-claude
+      litellm_params:
+        model: bedrock/anthropic.claude-3-sonnet-20240229-v1:0
+        aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID
+        aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY
+        aws_region_name: us-west-2
+```
+
+2. Start proxy 
+
+```bash
+litellm --config /path/to/config.yaml
+```
+
+3. Test it! 
+
+```python showLineNumbers title="AWS Bedrock Example using LiteLLM Proxy Server"
+import anthropic
+
+# point anthropic sdk to litellm proxy 
+client = anthropic.Anthropic(
+    base_url="http://0.0.0.0:4000",
+    api_key="sk-1234",
+)
+
+response = client.messages.create(
+    messages=[{"role": "user", "content": "Hello, can you tell me a short joke?"}],
+    model="bedrock-claude",
+    max_tokens=100,
+)
+```
+
+</TabItem>
+
+<TabItem value="curl-proxy" label="curl">
 
 ```bash showLineNumbers title="Example using LiteLLM Proxy Server"
 curl -L -X POST 'http://0.0.0.0:4000/v1/messages' \
@@ -136,7 +452,6 @@ curl -L -X POST 'http://0.0.0.0:4000/v1/messages' \
 </TabItem>
 </Tabs>
 
-
 ## Request Format
 ---
 
@@ -189,7 +504,7 @@ Request body will be in the Anthropic messages API format. **litellm follows the
 - **system** (string or array):  
   A system prompt providing context or specific instructions to the model.
 - **temperature** (number):  
-  Controls randomness in the model’s responses. Valid range: `0 < temperature < 1`.
+  Controls randomness in the model's responses. Valid range: `0 < temperature < 1`.
 - **thinking** (object):  
   Configuration for enabling extended thinking. If enabled, it includes:
   - **budget_tokens** (integer):  
@@ -201,7 +516,7 @@ Request body will be in the Anthropic messages API format. **litellm follows the
 - **tools** (array of objects):  
   Definitions for tools available to the model. Each tool includes:
   - **name** (string):  
-    The tool’s name.
+    The tool's name.
   - **description** (string):  
     A detailed description of the tool.
   - **input_schema** (object):  

docs/my-website/docs/apply_guardrail.md

@@ -0,0 +1,70 @@
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+
+# /guardrails/apply_guardrail
+
+Use this endpoint to directly call a guardrail configured on your LiteLLM instance. This is useful when you have services that need to directly call a guardrail. 
+
+
+## Usage
+---
+
+In this example `mask_pii` is the guardrail name configured on LiteLLM.
+
+```bash showLineNumbers title="Example calling the endpoint"
+curl -X POST 'http://localhost:4000/guardrails/apply_guardrail' \
+-H 'Content-Type: application/json' \
+-H 'Authorization: Bearer your-api-key' \
+-d '{
+    "guardrail_name": "mask_pii",
+    "text": "My name is John Doe and my email is john@example.com",
+    "language": "en",
+    "entities": ["NAME", "EMAIL"]
+}'
+```
+
+
+## Request Format
+---
+
+The request body should follow the ApplyGuardrailRequest format.
+
+#### Example Request Body
+
+```json
+{
+    "guardrail_name": "mask_pii",
+    "text": "My name is John Doe and my email is john@example.com",
+    "language": "en",
+    "entities": ["NAME", "EMAIL"]
+}
+```
+
+#### Required Fields
+- **guardrail_name** (string):  
+  The identifier for the guardrail to apply (e.g., "mask_pii").
+- **text** (string):  
+  The input text to process through the guardrail.
+
+#### Optional Fields
+- **language** (string):  
+  The language of the input text (e.g., "en" for English).
+- **entities** (array of strings):  
+  Specific entities to process or filter (e.g., ["NAME", "EMAIL"]).
+
+## Response Format
+---
+
+The response will contain the processed text after applying the guardrail.
+
+#### Example Response
+
+```json
+{
+    "response_text": "My name is [REDACTED] and my email is [REDACTED]"
+}
+```
+
+#### Response Fields
+- **response_text** (string):  
+  The text after applying the guardrail.

docs/my-website/docs/audio_transcription.md

@@ -1,13 +1,24 @@
 import Tabs from '@theme/Tabs';
 import TabItem from '@theme/TabItem';
 
-# Speech to Text
+# /audio/transcriptions
 
-Use this to loadbalance across Azure + OpenAI. 
+## Overview 
+
+| Feature | Supported | Notes | 
+|-------|-------|-------|
+| Cost Tracking | ✅ |  |
+| Logging | ✅ | works across all integrations |
+| End-user Tracking | ✅ | |
+| Fallbacks | ✅ | between supported models |
+| Loadbalancing | ✅ | between supported models |
+| Support llm providers | `openai`, `azure`, `vertex_ai`, `gemini`, `deepgram`, `groq`, `fireworks_ai` | |
 
 ## Quick Start
 
-```python
+### LiteLLM Python SDK
+
+```python showLineNumbers title="Python SDK Example"
 from litellm import transcription
 import os 
 
@@ -20,7 +31,7 @@ response = transcription(model="whisper", file=audio_file)
 print(f"response: {response}")
 ```
 
-## Proxy Usage
+### LiteLLM Proxy
 
 ### Add model to config 
 
@@ -28,7 +39,7 @@ print(f"response: {response}")
 <Tabs>
 <TabItem value="openai" label="OpenAI">
 
-```yaml
+```yaml showLineNumbers title="OpenAI Configuration"
 model_list:
 - model_name: whisper
   litellm_params:
@@ -43,7 +54,7 @@ general_settings:
 </TabItem>
 <TabItem value="openai+azure" label="OpenAI + Azure">
 
-```yaml
+```yaml showLineNumbers title="OpenAI + Azure Configuration"
 model_list:
 - model_name: whisper
   litellm_params:
@@ -69,7 +80,7 @@ general_settings:
 
 ### Start proxy 
 
-```bash
+```bash showLineNumbers title="Start Proxy Server"
 litellm --config /path/to/config.yaml 
 
 # RUNNING on http://0.0.0.0:8000
@@ -80,7 +91,7 @@ litellm --config /path/to/config.yaml
 <Tabs>
 <TabItem value="curl" label="Curl">
 
-```bash
+```bash showLineNumbers title="Test with cURL"
 curl --location 'http://0.0.0.0:8000/v1/audio/transcriptions' \
 --header 'Authorization: Bearer sk-1234' \
 --form 'file=@"/Users/krrishdholakia/Downloads/gettysburg.wav"' \
@@ -88,9 +99,9 @@ curl --location 'http://0.0.0.0:8000/v1/audio/transcriptions' \
 ```
 
 </TabItem>
-<TabItem value="openai" label="OpenAI">
+<TabItem value="openai" label="OpenAI Python SDK">
 
-```python
+```python showLineNumbers title="Test with OpenAI Python SDK"
 from openai import OpenAI
 client = openai.OpenAI(
     api_key="sk-1234",
@@ -113,4 +124,82 @@ transcript = client.audio.transcriptions.create(
 - Azure
 - [Fireworks AI](./providers/fireworks_ai.md#audio-transcription)
 - [Groq](./providers/groq.md#speech-to-text---whisper)
-- [Deepgram](./providers/deepgram.md)
+- [Deepgram](./providers/deepgram.md)
+
+---
+
+## Fallbacks
+
+You can configure fallbacks for audio transcription to automatically retry with different models if the primary model fails.
+
+<Tabs>
+<TabItem value="curl" label="Curl">
+
+```bash showLineNumbers title="Test with cURL and Fallbacks"
+curl --location 'http://0.0.0.0:4000/v1/audio/transcriptions' \
+--header 'Authorization: Bearer sk-1234' \
+--form 'file=@"gettysburg.wav"' \
+--form 'model="groq/whisper-large-v3"' \
+--form 'fallbacks[]="openai/whisper-1"'
+```
+
+</TabItem>
+<TabItem value="openai" label="OpenAI Python SDK">
+
+```python showLineNumbers title="Test with OpenAI Python SDK and Fallbacks"
+from openai import OpenAI
+client = OpenAI(
+    api_key="sk-1234",
+    base_url="http://0.0.0.0:4000"
+)
+
+audio_file = open("gettysburg.wav", "rb")
+transcript = client.audio.transcriptions.create(
+    model="groq/whisper-large-v3",
+    file=audio_file,
+    extra_body={
+        "fallbacks": ["openai/whisper-1"]
+    }
+)
+```
+</TabItem>
+</Tabs>
+
+### Testing Fallbacks
+
+You can test your fallback configuration using `mock_testing_fallbacks=true` to simulate failures:
+
+<Tabs>
+<TabItem value="curl" label="Curl">
+
+```bash showLineNumbers title="Test Fallbacks with Mock Testing"
+curl --location 'http://0.0.0.0:4000/v1/audio/transcriptions' \
+--header 'Authorization: Bearer sk-1234' \
+--form 'file=@"gettysburg.wav"' \
+--form 'model="groq/whisper-large-v3"' \
+--form 'fallbacks[]="openai/whisper-1"' \
+--form 'mock_testing_fallbacks=true'
+```
+
+</TabItem>
+<TabItem value="openai" label="OpenAI Python SDK">
+
+```python showLineNumbers title="Test Fallbacks with Mock Testing"
+from openai import OpenAI
+client = OpenAI(
+    api_key="sk-1234",
+    base_url="http://0.0.0.0:4000"
+)
+
+audio_file = open("gettysburg.wav", "rb")
+transcript = client.audio.transcriptions.create(
+    model="groq/whisper-large-v3",
+    file=audio_file,
+    extra_body={
+        "fallbacks": ["openai/whisper-1"],
+        "mock_testing_fallbacks": True
+    }
+)
+```
+</TabItem>
+</Tabs>

docs/my-website/docs/batches.md

@@ -78,8 +78,9 @@ curl http://localhost:4000/v1/batches \
 **Create File for Batch Completion**
 
 ```python
-from litellm
+import litellm
 import os 
+import asyncio
 
 os.environ["OPENAI_API_KEY"] = "sk-.."
 
@@ -97,8 +98,9 @@ print("Response from creating file=", file_obj)
 **Create Batch Request**
 
 ```python
-from litellm
+import litellm
 import os 
+import asyncio
 
 create_batch_response = await litellm.acreate_batch(
     completion_window="24h",
@@ -114,10 +116,38 @@ print("response from litellm.create_batch=", create_batch_response)
 **Retrieve the Specific Batch and File Content**
 
 ```python
+    # Maximum wait time before we give up
+    MAX_WAIT_TIME = 300  
+
+    # Time to wait between each status check
+    POLL_INTERVAL = 5
+    
+    #Time waited till now 
+    waited = 0
+
+    # Wait for the batch to finish processing before trying to retrieve output
+    # This loop checks the batch status every few seconds (polling)
+
+    while True:
+        retrieved_batch = await litellm.aretrieve_batch(
+            batch_id=create_batch_response.id,
+            custom_llm_provider="openai"
+        )
+        
+        status = retrieved_batch.status
+        print(f"⏳ Batch status: {status}")
+        
+        if status == "completed" and retrieved_batch.output_file_id:
+            print("✅ Batch complete. Output file ID:", retrieved_batch.output_file_id)
+            break
+        elif status in ["failed", "cancelled", "expired"]:
+            raise RuntimeError(f"❌ Batch failed with status: {status}")
+        
+        await asyncio.sleep(POLL_INTERVAL)
+        waited += POLL_INTERVAL
+        if waited > MAX_WAIT_TIME:
+            raise TimeoutError("❌ Timed out waiting for batch to complete.")
 
-retrieved_batch = await litellm.aretrieve_batch(
-    batch_id=create_batch_response.id, custom_llm_provider="openai"
-)
 print("retrieved batch=", retrieved_batch)
 # just assert that we retrieved a non None batch
 

docs/my-website/docs/benchmarks.md

@@ -7,13 +7,11 @@ Benchmarks for LiteLLM Gateway (Proxy Server) tested against a fake OpenAI endpo
 
 Use this config for testing:
 
-**Note:**  we're currently migrating to aiohttp which has 10x higher throughput. We recommend using the `aiohttp_openai/` provider for load testing.
-
 ```yaml
 model_list:
   - model_name: "fake-openai-endpoint"
     litellm_params:
-      model: aiohttp_openai/any
+      model: openai/any
       api_base: https://your-fake-openai-endpoint.com/chat/completions
       api_key: "test"
 ```

docs/my-website/docs/caching/all_caches.md

@@ -236,10 +236,10 @@ response2 = completion(
 
 ### Quick Start
 
-Install diskcache:
+Install the disk caching extra:
 
 ```shell
-pip install diskcache
+pip install "litellm[caching]"
 ```
 
 Then you can use the disk cache as follows.

docs/my-website/docs/completion/document_understanding.md

@@ -9,6 +9,7 @@ Works for:
 - Vertex AI models (Gemini + Anthropic)
 - Bedrock Models
 - Anthropic API Models
+- OpenAI API Models
 
 ## Quick Start
 
@@ -187,6 +188,97 @@ curl -X POST 'http://0.0.0.0:4000/chat/completions' \
 </TabItem>
 </Tabs>
 
+## Specifying format 
+
+To specify the format of the document, you can use the `format` parameter. 
+
+
+<Tabs>
+<TabItem value="sdk" label="SDK">
+
+```python
+from litellm.utils import supports_pdf_input, completion
+
+# set aws credentials
+os.environ["AWS_ACCESS_KEY_ID"] = ""
+os.environ["AWS_SECRET_ACCESS_KEY"] = ""
+os.environ["AWS_REGION_NAME"] = ""
+
+
+# pdf url
+file_url = "https://www.w3.org/WAI/ER/tests/xhtml/testfiles/resources/pdf/dummy.pdf"
+
+# model
+model = "bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0"
+
+file_content = [
+    {"type": "text", "text": "What's this file about?"},
+    {
+        "type": "file",
+        "file": {
+            "file_id": file_url,
+            "format": "application/pdf",
+        }
+    },
+]
+
+
+if not supports_pdf_input(model, None):
+    print("Model does not support image input")
+
+response = completion(
+    model=model,
+    messages=[{"role": "user", "content": file_content}],
+)
+assert response is not None
+```
+</TabItem>
+<TabItem value="proxy" label="PROXY">
+
+1. Setup config.yaml
+
+```yaml
+model_list:
+  - model_name: bedrock-model
+    litellm_params:
+      model: bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0
+      aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID
+      aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY
+      aws_region_name: os.environ/AWS_REGION_NAME
+```
+
+2. Start the proxy 
+
+```bash
+litellm --config /path/to/config.yaml
+```
+
+3. Test it! 
+
+```bash
+curl -X POST 'http://0.0.0.0:4000/chat/completions' \
+-H 'Content-Type: application/json' \
+-H 'Authorization: Bearer sk-1234' \
+-d '{
+    "model": "bedrock-model",
+    "messages": [
+        {"role": "user", "content": [
+            {"type": "text", "text": "What's this file about?"},
+            {
+                "type": "file",
+                "file": {
+                    "file_id": "https://www.w3.org/WAI/ER/tests/xhtml/testfiles/resources/pdf/dummy.pdf",
+                    "format": "application/pdf",
+                }
+            }
+        ]},
+    ]
+}'
+```
+</TabItem>
+</Tabs>
+
+
 ## Checking if a model supports pdf input
 
 <Tabs>

docs/my-website/docs/completion/input.md

@@ -39,29 +39,32 @@ This is a list of openai params we translate across providers.
 
 Use `litellm.get_supported_openai_params()` for an updated list of params for each model + provider 
 
-| Provider | temperature | max_completion_tokens | max_tokens | top_p | stream | stream_options | stop | n | presence_penalty | frequency_penalty | functions | function_call | logit_bias | user | response_format | seed | tools | tool_choice | logprobs | top_logprobs | extra_headers |
-|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
-|Anthropic| ✅ | ✅ | ✅ |✅ | ✅ | ✅ | ✅ |  |  |   |  |  |  |✅ | ✅ | | ✅ | ✅ |  |  | ✅ |
-|OpenAI| ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |✅ | ✅ | ✅ | ✅ |✅ | ✅ | ✅ | ✅ | ✅ |
-|Azure OpenAI| ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |✅ | ✅ | ✅ | ✅ |✅ | ✅ |  |  | ✅ |
-|xAI| ✅ |  | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |  |  | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |  |
-|Replicate | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | |  |   |  |   |
-|Anyscale | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |
-|Cohere| ✅ | ✅ | ✅ | ✅ |  ✅ | ✅ | ✅ | ✅ | ✅ |   |   |
-|Huggingface| ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |  |   |    |
-|Openrouter| ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | | | | | ✅ |✅ | | | |
-|AI21| ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |  |  |
-|VertexAI| ✅ | ✅ | ✅ | | ✅ | ✅ |  |  |  |  |   | | | | ✅ | ✅ | | |
-|Bedrock| ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |  |  |   |  |   | | | | | ✅ (model dependent) | |
-|Sagemaker| ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |  |  |   |
-|TogetherAI| ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |  |  |  |  |  | ✅ |  |  | ✅ |  | ✅ | ✅ |  |  |  |
-|AlephAlpha| ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |   |  |   |
-|NLP Cloud| ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | | |  |  |   |
-|Petals| ✅ | ✅ |  | ✅ | ✅ | |  |   |  |   |
-|Ollama| ✅ | ✅ | ✅ |✅ | ✅ | ✅ |  |   | ✅ |  |   | | | ✅ |  | |✅| | | | | | |
-|Databricks| ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |  |   | |  |   | | | | | |
-|ClarifAI| ✅ | ✅ | ✅ | |✅ | ✅ |  |   | |  |   | | | | | |
-|Github| ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | | | | | ✅ |✅ (model dependent)|✅ (model dependent)| | |
+| Provider | temperature | max_completion_tokens | max_tokens | top_p | stream | stream_options | stop | n | presence_penalty | frequency_penalty | functions | function_call | logit_bias | user | response_format | seed| tools | tool_choice | logprobs | top_logprobs | extra_headers |
+|--------------|-------------|------------------------|------------|-------|--------|----------------|------|-----|------------------|-------------------|-----------|----------------|-------------|------|------------------|-------------------|--------|--------------|----------|---------------|----------------------|
+| Anthropic| ✅| ✅ | ✅ | ✅| ✅ | ✅ | ✅ | || | || | ✅ | ✅ | | ✅ | ✅ || | ✅|
+| OpenAI | ✅| ✅ | ✅ | ✅| ✅ | ✅ | ✅ | ✅| ✅ | ✅| ✅| ✅ | ✅| ✅ | ✅ | ✅| ✅ | ✅ | ✅ | ✅| ✅|
+| Azure OpenAI | ✅| ✅ | ✅ | ✅| ✅ | ✅ | ✅ | ✅| ✅ | ✅| ✅| ✅ | ✅| ✅ | ✅ | ✅| ✅ | ✅ | ✅ | ✅| ✅|
+| xAI| ✅|| ✅ | ✅| ✅ | ✅ | ✅ | ✅| ✅ | ✅| || ✅| ✅ | ✅ | ✅| ✅ | ✅ | ✅ | ✅||
+| Replicate| ✅| ✅ | ✅ | ✅| ✅ | ✅ || || | || ||| |||| ||
+| Anyscale | ✅| ✅ | ✅ | ✅| ✅ | ✅ || || | || ||| |||| ||
+| Cohere | ✅| ✅ | ✅ | ✅| ✅ | ✅ | ✅ | ✅|| | || ||| |||| ||
+| Huggingface| ✅| ✅ | ✅ | ✅| ✅ | ✅ | ✅ | || | || ||| |||| ||
+| Openrouter | ✅| ✅ | ✅ | ✅| ✅ | ✅ | ✅ | ✅| ✅ | ✅| ✅|| ||| ✅| ✅ ||| ||
+| AI21 | ✅| ✅ | ✅ | ✅| ✅ | ✅ | ✅ | ✅|| | || ||| |||| ||
+| VertexAI | ✅| ✅ | ✅ | | ✅ | ✅ || || | || || ✅ | ✅|||| ||
+| Bedrock| ✅| ✅ | ✅ | ✅| ✅ | ✅ || || | || || ✅ (model dependent) | |||| ||
+| Sagemaker| ✅| ✅ | ✅ | ✅| ✅ | ✅ | ✅ | || | || ||| |||| ||
+| TogetherAI | ✅| ✅ | ✅ | ✅| ✅ | ✅ || || | ✅|| || ✅ | | ✅ | ✅ || ||
+| Sambanova| ✅| ✅ | ✅ | ✅| ✅ | ✅ | ✅ | || | || || ✅ | | ✅ | ✅ || ||
+| AlephAlpha | ✅| ✅ | ✅ | ✅| ✅ | ✅ | ✅ | || | || ||| |||| ||
+| NLP Cloud| ✅| ✅ | ✅ | ✅| ✅ | ✅ || || | || ||| |||| ||
+| Petals | ✅| ✅ || ✅| ✅ ||| || | || ||| |||| ||
+| Ollama | ✅| ✅ | ✅ | ✅| ✅ | ✅ || ✅|| | || ✅||| | ✅ ||| ||
+| Databricks | ✅| ✅ | ✅ | ✅| ✅ | ✅ || || | || ||| |||| ||
+| ClarifAI | ✅| ✅ | ✅ | | ✅ | ✅ || || | || ||| |||| ||
+| Github | ✅| ✅ | ✅ | ✅| ✅ | ✅ | ✅ | ✅| ✅ | ✅| ✅|| || ✅ | ✅ (model dependent) | ✅ (model dependent) || ||
+| Novita AI| ✅| ✅ || ✅| ✅ | ✅ | ✅ | ✅| ✅ | ✅| || ✅||| |||| ||
+
 :::note
 
 By default, LiteLLM raises an exception if the openai param being passed in isn't supported. 

docs/my-website/docs/completion/knowledgebase.md

@@ -0,0 +1,358 @@
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+import Image from '@theme/IdealImage';
+
+# Using Vector Stores (Knowledge Bases)
+
+<Image 
+  img={require('../../img/kb.png')}
+  style={{width: '100%', display: 'block', margin: '2rem auto'}}
+/>
+<p style={{textAlign: 'left', color: '#666'}}>
+  Use Vector Stores with any LiteLLM supported model
+</p>
+
+
+LiteLLM integrates with vector stores, allowing your models to access your organization's data for more accurate and contextually relevant responses.
+
+## Supported Vector Stores
+- [Bedrock Knowledge Bases](https://aws.amazon.com/bedrock/knowledge-bases/)
+- [OpenAI Vector Stores](https://platform.openai.com/docs/api-reference/vector-stores/search)
+- [Azure Vector Stores](https://learn.microsoft.com/en-us/azure/ai-services/openai/how-to/file-search?tabs=python#vector-stores
+
+## Quick Start
+
+In order to use a vector store with LiteLLM, you need to 
+
+- Initialize litellm.vector_store_registry
+- Pass tools with vector_store_ids to the completion request. Where `vector_store_ids` is a list of vector store ids you initialized in litellm.vector_store_registry
+
+### LiteLLM Python SDK
+
+LiteLLM's allows you to use vector stores in the [OpenAI API spec](https://platform.openai.com/docs/api-reference/chat/create) by passing a tool with vector_store_ids you want to use
+
+```python showLineNumbers title="Basic Bedrock Knowledge Base Usage"
+import os
+import litellm
+
+from litellm.vector_stores.vector_store_registry import VectorStoreRegistry, LiteLLM_ManagedVectorStore
+
+# Init vector store registry
+litellm.vector_store_registry = VectorStoreRegistry(
+    vector_stores=[
+        LiteLLM_ManagedVectorStore(
+            vector_store_id="T37J8R4WTM",
+            custom_llm_provider="bedrock"
+        )
+    ]
+)
+
+
+# Make a completion request with vector_store_ids parameter
+response = await litellm.acompletion(
+    model="anthropic/claude-3-5-sonnet", 
+    messages=[{"role": "user", "content": "What is litellm?"}],
+    tools=[
+        {
+            "type": "file_search",
+            "vector_store_ids": ["T37J8R4WTM"]
+        }
+    ],
+)
+
+print(response.choices[0].message.content)
+```
+
+### LiteLLM Proxy
+
+#### 1. Configure your vector_store_registry
+
+In order to use a vector store with LiteLLM, you need to configure your vector_store_registry. This tells litellm which vector stores to use and api provider to use for the vector store.
+
+<Tabs>
+<TabItem value="config-yaml" label="config.yaml">
+
+```yaml showLineNumbers title="config.yaml"
+model_list:
+  - model_name: claude-3-5-sonnet
+    litellm_params:
+      model: anthropic/claude-3-5-sonnet
+      api_key: os.environ/ANTHROPIC_API_KEY
+
+vector_store_registry:
+  - vector_store_name: "bedrock-litellm-website-knowledgebase"
+    litellm_params:
+      vector_store_id: "T37J8R4WTM"
+      custom_llm_provider: "bedrock"
+      vector_store_description: "Bedrock vector store for the Litellm website knowledgebase"
+      vector_store_metadata:
+        source: "https://www.litellm.com/docs"
+
+```
+
+</TabItem>
+
+<TabItem value="litellm-ui" label="LiteLLM UI">
+
+On the LiteLLM UI, Navigate to Experimental > Vector Stores > Create Vector Store. On this page you can create a vector store with a name, vector store id and credentials.
+<Image 
+  img={require('../../img/kb_2.png')}
+  style={{width: '50%'}}
+/>
+
+
+
+
+</TabItem>
+
+</Tabs>
+
+#### 2. Make a request with vector_store_ids parameter
+
+<Tabs>
+<TabItem value="curl" label="Curl">
+
+```bash showLineNumbers title="Curl Request to LiteLLM Proxy"
+curl http://localhost:4000/v1/chat/completions \
+  -H "Content-Type: application/json" \
+  -H "Authorization: Bearer $LITELLM_API_KEY" \
+  -d '{
+    "model": "claude-3-5-sonnet",
+    "messages": [{"role": "user", "content": "What is litellm?"}],
+    "tools": [
+        {
+            "type": "file_search",
+            "vector_store_ids": ["T37J8R4WTM"]
+        }
+    ]
+  }'
+```
+
+</TabItem>
+
+<TabItem value="openai-sdk" label="OpenAI Python SDK">
+
+```python showLineNumbers title="OpenAI Python SDK Request"
+from openai import OpenAI
+
+# Initialize client with your LiteLLM proxy URL
+client = OpenAI(
+    base_url="http://localhost:4000",
+    api_key="your-litellm-api-key"
+)
+
+# Make a completion request with vector_store_ids parameter
+response = client.chat.completions.create(
+    model="claude-3-5-sonnet",
+    messages=[{"role": "user", "content": "What is litellm?"}],
+    tools=[
+        {
+            "type": "file_search",
+            "vector_store_ids": ["T37J8R4WTM"]
+        }
+    ]
+)
+
+print(response.choices[0].message.content)
+```
+
+</TabItem>
+</Tabs>
+
+
+
+
+## Advanced
+
+### Logging Vector Store Usage
+
+LiteLLM allows you to view your vector store usage in the LiteLLM UI on the `Logs` page.
+
+After completing a request with a vector store, navigate to the `Logs` page on LiteLLM. Here you should be able to see the query sent to the vector store and corresponding response with scores.
+
+<Image 
+  img={require('../../img/kb_4.png')}
+  style={{width: '80%'}}
+/>
+<p style={{textAlign: 'left', color: '#666'}}>
+  LiteLLM Logs Page: Vector Store Usage
+</p>
+
+
+### Listing available vector stores
+
+You can list all available vector stores using the /vector_store/list endpoint
+
+**Request:**
+```bash showLineNumbers title="List all available vector stores"
+curl -X GET "http://localhost:4000/vector_store/list" \
+  -H "Authorization: Bearer $LITELLM_API_KEY"
+```
+
+**Response:**
+
+The response will be a list of all vector stores that are available to use with LiteLLM.
+
+```json
+{
+  "object": "list",
+  "data": [
+    {
+      "vector_store_id": "T37J8R4WTM",
+      "custom_llm_provider": "bedrock",
+      "vector_store_name": "bedrock-litellm-website-knowledgebase",
+      "vector_store_description": "Bedrock vector store for the Litellm website knowledgebase",
+      "vector_store_metadata": {
+        "source": "https://www.litellm.com/docs"
+      },
+      "created_at": "2023-05-03T18:21:36.462Z",
+      "updated_at": "2023-05-03T18:21:36.462Z",
+      "litellm_credential_name": "bedrock_credentials"
+    }
+  ],
+  "total_count": 1,
+  "current_page": 1,
+  "total_pages": 1
+}
+```
+
+
+### Always on for a model
+
+**Use this if you want vector stores to be used by default for a specific model.**
+
+In this config, we add `vector_store_ids` to the claude-3-5-sonnet-with-vector-store model. This means that any request to the claude-3-5-sonnet-with-vector-store model will always use the vector store with the id `T37J8R4WTM` defined in the `vector_store_registry`.
+
+```yaml showLineNumbers title="Always on for a model"
+model_list:
+  - model_name: claude-3-5-sonnet-with-vector-store
+    litellm_params:
+      model: anthropic/claude-3-5-sonnet
+      vector_store_ids: ["T37J8R4WTM"]
+
+vector_store_registry:
+  - vector_store_name: "bedrock-litellm-website-knowledgebase"
+    litellm_params:
+      vector_store_id: "T37J8R4WTM"
+      custom_llm_provider: "bedrock"
+      vector_store_description: "Bedrock vector store for the Litellm website knowledgebase"
+      vector_store_metadata:
+        source: "https://www.litellm.com/docs"
+```
+
+## How It Works
+
+If your request includes a `vector_store_ids` parameter where any of the vector store ids are found in the `vector_store_registry`, LiteLLM will automatically use the vector store for the request.
+
+1. You make a completion request with the `vector_store_ids` parameter and any of the vector store ids are found in the `litellm.vector_store_registry`
+2. LiteLLM automatically:
+   - Uses your last message as the query to retrieve relevant information from the Knowledge Base
+   - Adds the retrieved context to your conversation
+   - Sends the augmented messages to the model
+
+#### Example Transformation
+
+When you pass `vector_store_ids=["YOUR_KNOWLEDGE_BASE_ID"]`, your request flows through these steps:
+
+**1. Original Request to LiteLLM:**
+```json
+{
+    "model": "anthropic/claude-3-5-sonnet",
+    "messages": [
+        {"role": "user", "content": "What is litellm?"}
+    ],
+    "vector_store_ids": ["YOUR_KNOWLEDGE_BASE_ID"]
+}
+```
+
+**2. Request to AWS Bedrock Knowledge Base:**
+```json
+{
+    "retrievalQuery": {
+        "text": "What is litellm?"
+    }
+}
+```
+This is sent to: `https://bedrock-agent-runtime.{aws_region}.amazonaws.com/knowledgebases/YOUR_KNOWLEDGE_BASE_ID/retrieve`
+
+**3. Final Request to LiteLLM:**
+```json
+{
+    "model": "anthropic/claude-3-5-sonnet",
+    "messages": [
+        {"role": "user", "content": "What is litellm?"},
+        {"role": "user", "content": "Context: \n\nLiteLLM is an open-source SDK to simplify LLM API calls across providers (OpenAI, Claude, etc). It provides a standardized interface with robust error handling, streaming, and observability tools."}
+    ]
+}
+```
+
+This process happens automatically whenever you include the `vector_store_ids` parameter in your request.
+
+## API Reference
+
+### LiteLLM Completion Knowledge Base Parameters
+
+When using the Knowledge Base integration with LiteLLM, you can include the following parameters:
+
+| Parameter | Type | Description |
+|-----------|------|-------------|
+| `vector_store_ids` | List[str] | List of Knowledge Base IDs to query |
+
+### VectorStoreRegistry
+
+The `VectorStoreRegistry` is a central component for managing vector stores in LiteLLM. It acts as a registry where you can configure and access your vector stores.
+
+#### What is VectorStoreRegistry?
+
+`VectorStoreRegistry` is a class that:
+- Maintains a collection of vector stores that LiteLLM can use
+- Allows you to register vector stores with their credentials and metadata
+- Makes vector stores accessible via their IDs in your completion requests
+
+#### Using VectorStoreRegistry in Python
+
+```python
+from litellm.vector_stores.vector_store_registry import VectorStoreRegistry, LiteLLM_ManagedVectorStore
+
+# Initialize the vector store registry with one or more vector stores
+litellm.vector_store_registry = VectorStoreRegistry(
+    vector_stores=[
+        LiteLLM_ManagedVectorStore(
+            vector_store_id="YOUR_VECTOR_STORE_ID",  # Required: Unique ID for referencing this store
+            custom_llm_provider="bedrock"            # Required: Provider (e.g., "bedrock")
+        )
+    ]
+)
+```
+
+#### LiteLLM_ManagedVectorStore Parameters
+
+Each vector store in the registry is configured using a `LiteLLM_ManagedVectorStore` object with these parameters:
+
+| Parameter | Type | Required | Description |
+|-----------|------|----------|-------------|
+| `vector_store_id` | str | Yes | Unique identifier for the vector store |
+| `custom_llm_provider` | str | Yes | The provider of the vector store (e.g., "bedrock") |
+| `vector_store_name` | str | No | A friendly name for the vector store |
+| `vector_store_description` | str | No | Description of what the vector store contains |
+| `vector_store_metadata` | dict or str | No | Additional metadata about the vector store |
+| `litellm_credential_name` | str | No | Name of the credentials to use for this vector store |
+
+#### Configuring VectorStoreRegistry in config.yaml
+
+For the LiteLLM Proxy, you can configure the same registry in your `config.yaml` file:
+
+```yaml showLineNumbers title="Vector store configuration in config.yaml"
+vector_store_registry:
+  - vector_store_name: "bedrock-litellm-website-knowledgebase"  # Optional friendly name
+    litellm_params:
+      vector_store_id: "T37J8R4WTM"                            # Required: Unique ID  
+      custom_llm_provider: "bedrock"                           # Required: Provider
+      vector_store_description: "Bedrock vector store for the Litellm website knowledgebase"
+      vector_store_metadata:
+        source: "https://www.litellm.com/docs"
+```
+
+The `litellm_params` section accepts all the same parameters as the `LiteLLM_ManagedVectorStore` constructor in the Python SDK.
+
+

docs/my-website/docs/completion/web_search.md

@@ -8,9 +8,9 @@ Use web search with litellm
 | Feature | Details |
 |---------|---------|
 | Supported Endpoints | - `/chat/completions` <br/> - `/responses` |
-| Supported Providers | `openai` |
+| Supported Providers | `openai`, `xai`, `vertex_ai`, `gemini`, `perplexity` |
 | LiteLLM Cost Tracking | ✅ Supported |
-| LiteLLM Version | `v1.63.15-nightly` or higher |
+| LiteLLM Version | `v1.71.0+` |
 
 
 ## `/chat/completions` (litellm.completion)
@@ -31,8 +31,12 @@ response = completion(
             "content": "What was a positive news story from today?",
         }
     ],
+    web_search_options={
+        "search_context_size": "medium"  # Options: "low", "medium", "high"
+    }
 )
 ```
+
 </TabItem>
 <TabItem value="proxy" label="PROXY">
 
@@ -40,10 +44,30 @@ response = completion(
 
 ```yaml
 model_list:
+  # OpenAI
   - model_name: gpt-4o-search-preview
     litellm_params:
       model: openai/gpt-4o-search-preview
       api_key: os.environ/OPENAI_API_KEY
+  
+  # xAI
+  - model_name: grok-3
+    litellm_params:
+      model: xai/grok-3
+      api_key: os.environ/XAI_API_KEY
+  
+  # VertexAI
+  - model_name: gemini-2-flash
+    litellm_params:
+      model: gemini-2.0-flash
+      vertex_project: your-project-id
+      vertex_location: us-central1
+  
+  # Google AI Studio
+  - model_name: gemini-2-flash-studio
+    litellm_params:
+      model: gemini/gemini-2.0-flash
+      api_key: os.environ/GOOGLE_API_KEY
 ```
 
 2. Start the proxy 
@@ -64,7 +88,7 @@ client = OpenAI(
 )
 
 response = client.chat.completions.create(
-    model="gpt-4o-search-preview",
+    model="grok-3",  # or any other web search enabled model
     messages=[
         {
             "role": "user",
@@ -81,6 +105,7 @@ response = client.chat.completions.create(
 <Tabs>
 <TabItem value="sdk" label="SDK">
 
+**OpenAI (using web_search_options)**
 ```python showLineNumbers
 from litellm import completion
 
@@ -98,6 +123,44 @@ response = completion(
     }
 )
 ```
+
+**xAI (using web_search_options)**
+```python showLineNumbers
+from litellm import completion
+
+# Customize search context size for xAI
+response = completion(
+    model="xai/grok-3",
+    messages=[
+        {
+            "role": "user",
+            "content": "What was a positive news story from today?",
+        }
+    ],
+    web_search_options={
+        "search_context_size": "high"  # Options: "low", "medium" (default), "high"
+    }
+)
+```
+
+**VertexAI/Gemini (using web_search_options)**
+```python showLineNumbers
+from litellm import completion
+
+# Customize search context size for Gemini
+response = completion(
+    model="gemini-2.0-flash",
+    messages=[
+        {
+            "role": "user",
+            "content": "What was a positive news story from today?",
+        }
+    ],
+    web_search_options={
+        "search_context_size": "low"  # Options: "low", "medium" (default), "high"
+    }
+)
+```
 </TabItem>
 <TabItem value="proxy" label="PROXY">
 
@@ -112,7 +175,7 @@ client = OpenAI(
 
 # Customize search context size
 response = client.chat.completions.create(
-    model="gpt-4o-search-preview",
+    model="grok-3",  # works with any web search enabled model
     messages=[
         {
             "role": "user",
@@ -127,6 +190,8 @@ response = client.chat.completions.create(
 </TabItem>
 </Tabs>
 
+
+
 ## `/responses` (litellm.responses)
 
 ### Quick Start
@@ -243,35 +308,119 @@ print(response.output_text)
 </TabItem>
 </Tabs>
 
+## Configuring Web Search in config.yaml
 
+You can set default web search options directly in your proxy config file:
 
+<Tabs>
+<TabItem value="default" label="Default Web Search">
 
+```yaml
+model_list:
+  # Enable web search by default for all requests to this model
+  - model_name: grok-3
+    litellm_params:
+      model: xai/grok-3
+      api_key: os.environ/XAI_API_KEY
+      web_search_options: {}  # Enables web search with default settings
+```
 
+</TabItem>
+<TabItem value="custom" label="Custom Search Context">
+
+```yaml
+model_list:
+  # Set custom web search context size
+  - model_name: grok-3
+    litellm_params:
+      model: xai/grok-3
+      api_key: os.environ/XAI_API_KEY
+      web_search_options:
+        search_context_size: "high"  # Options: "low", "medium", "high"
+  
+  # Different context size for different models
+  - model_name: gpt-4o-search-preview
+    litellm_params:
+      model: openai/gpt-4o-search-preview
+      api_key: os.environ/OPENAI_API_KEY
+      web_search_options:
+        search_context_size: "low"
+  
+  # Gemini with medium context (default)
+  - model_name: gemini-2-flash
+    litellm_params:
+      model: gemini-2.0-flash
+      vertex_project: your-project-id
+      vertex_location: us-central1
+      web_search_options:
+        search_context_size: "medium"
+```
+
+</TabItem>
+</Tabs>
+
+**Note:** When `web_search_options` is set in the config, it applies to all requests to that model. Users can still override these settings by passing `web_search_options` in their API requests.
 
 ## Checking if a model supports web search
 
 <Tabs>
 <TabItem label="SDK" value="sdk">
 
-Use `litellm.supports_web_search(model="openai/gpt-4o-search-preview")` -> returns `True` if model can perform web searches
+Use `litellm.supports_web_search(model="model_name")` -> returns `True` if model can perform web searches
 
 ```python showLineNumbers
+# Check OpenAI models
 assert litellm.supports_web_search(model="openai/gpt-4o-search-preview") == True
+
+# Check xAI models
+assert litellm.supports_web_search(model="xai/grok-3") == True
+
+# Check VertexAI models
+assert litellm.supports_web_search(model="gemini-2.0-flash") == True
+
+# Check Google AI Studio models
+assert litellm.supports_web_search(model="gemini/gemini-2.0-flash") == True
 ```
 </TabItem>
 
 <TabItem label="PROXY" value="proxy">
 
-1. Define OpenAI models in config.yaml
+1. Define models in config.yaml
 
 ```yaml
 model_list:
+  # OpenAI
   - model_name: gpt-4o-search-preview
     litellm_params:
       model: openai/gpt-4o-search-preview
       api_key: os.environ/OPENAI_API_KEY
     model_info:
       supports_web_search: True
+  
+  # xAI
+  - model_name: grok-3
+    litellm_params:
+      model: xai/grok-3
+      api_key: os.environ/XAI_API_KEY
+    model_info:
+      supports_web_search: True
+  
+  # VertexAI
+  - model_name: gemini-2-flash
+    litellm_params:
+      model: gemini-2.0-flash
+      vertex_project: your-project-id
+      vertex_location: us-central1
+    model_info:
+      supports_web_search: True
+  
+  # Google AI Studio
+  - model_name: gemini-2-flash-studio
+    litellm_params:
+      model: gemini/gemini-2.0-flash
+      api_key: os.environ/GOOGLE_API_KEY
+    model_info:
+      supports_web_search: True
 ```
 
 2. Run proxy server
@@ -298,7 +447,19 @@ Expected Response
       "model_group": "gpt-4o-search-preview",
       "providers": ["openai"],
       "max_tokens": 128000,
-      "supports_web_search": true, # 👈 supports_web_search is true
+      "supports_web_search": true
+    },
+    {
+      "model_group": "grok-3",
+      "providers": ["xai"],
+      "max_tokens": 131072,
+      "supports_web_search": true
+    },
+    {
+      "model_group": "gemini-2-flash",
+      "providers": ["vertex_ai"],
+      "max_tokens": 8192,
+      "supports_web_search": true
     }
   ]
 }

docs/my-website/docs/contact.md

@@ -2,5 +2,6 @@
 
 [![](https://dcbadge.vercel.app/api/server/wuPM9dRgDw)](https://discord.gg/wuPM9dRgDw)
 
+* [Community Slack 💭](https://join.slack.com/share/enQtOTE0ODczMzk2Nzk4NC01YjUxNjY2YjBlYTFmNDRiZTM3NDFiYTM3MzVkODFiMDVjOGRjMmNmZTZkZTMzOWQzZGQyZWIwYjQ0MWExYmE3)
 * [Meet with us 👋](https://calendly.com/d/4mp-gd3-k5k/berriai-1-1-onboarding-litellm-hosted-version)
 * Contact us at ishaan@berri.ai / krrish@berri.ai

docs/my-website/docs/contributing.md

@@ -33,11 +33,11 @@ cd litellm/ui/litellm-dashboard
 
 npm run dev
 
-# starts on http://0.0.0.0:3000/ui
+# starts on http://0.0.0.0:3000
 ```
 
 ## 3. Go to local UI 
 
-```
-http://0.0.0.0:3000/ui
+```bash
+http://0.0.0.0:3000
 ```

docs/my-website/docs/data_security.md

@@ -45,7 +45,7 @@ For security inquiries, please contact us at support@berri.ai
 | **Certification** | **Status**                                                                                      |
 |-------------------|-------------------------------------------------------------------------------------------------|
 | SOC 2 Type I      | Certified. Report available upon request on Enterprise plan.                                                           |
-| SOC 2 Type II     | In progress. Certificate available by April 15th, 2025                   |
+| SOC 2 Type II     | Certified. Report available upon request on Enterprise plan.                   |
 | ISO 27001          | Certified. Report available upon request on Enterprise                              |
 
 

docs/my-website/docs/embedding/supported_embedding.md

@@ -225,36 +225,6 @@ response = embedding(
 | text-embedding-3-large | `embedding('text-embedding-3-large', input)` | `os.environ['OPENAI_API_KEY']`       |
 | text-embedding-ada-002 | `embedding('text-embedding-ada-002', input)` | `os.environ['OPENAI_API_KEY']`       |
 
-## Azure OpenAI Embedding Models
-
-### API keys
-This can be set as env variables or passed as **params to litellm.embedding()**
-```python
-import os
-os.environ['AZURE_API_KEY'] = 
-os.environ['AZURE_API_BASE'] = 
-os.environ['AZURE_API_VERSION'] = 
-```
-
-### Usage
-```python
-from litellm import embedding
-response = embedding(
-    model="azure/<your deployment name>",
-    input=["good morning from litellm"],
-    api_key=api_key,
-    api_base=api_base,
-    api_version=api_version,
-)
-print(response)
-```
-
-| Model Name           | Function Call                               |
-|----------------------|---------------------------------------------|
-| text-embedding-ada-002 | `embedding(model="azure/<your deployment name>", input=input)` |
-
-h/t to [Mikko](https://www.linkedin.com/in/mikkolehtimaki/) for this integration
-
 ## OpenAI Compatible Embedding Models
 Use this for calling `/embedding` endpoints on OpenAI Compatible Servers, example https://github.com/xorbitsai/inference
 
@@ -340,9 +310,25 @@ import os
 os.environ['NVIDIA_NIM_API_KEY'] = ""
 response = embedding(
     model='nvidia_nim/<model_name>', 
-    input=["good morning from litellm"]
+    input=["good morning from litellm"],
+    input_type="query"
 )
 ```
+## `input_type` Parameter for Embedding Models
+
+Certain embedding models, such as `nvidia/embed-qa-4` and the E5 family, operate in **dual modes**—one for **indexing documents (passages)** and another for **querying**. To maintain high retrieval accuracy, it's essential to specify how the input text is being used by setting the `input_type` parameter correctly.
+
+### Usage
+
+Set the `input_type` parameter to one of the following values:
+
+- `"passage"` – for embedding content during **indexing** (e.g., documents).
+- `"query"` – for embedding content during **retrieval** (e.g., user queries).
+
+> **Warning:** Incorrect usage of `input_type` can lead to a significant drop in retrieval performance.
+
+
+
 All models listed [here](https://build.nvidia.com/explore/retrieval) are supported:
 
 | Model Name         | Function Call                                         |
@@ -357,6 +343,7 @@ All models listed [here](https://build.nvidia.com/explore/retrieval) are support
 | snowflake/arctic-embed-l | `embedding(model="nvidia_nim/snowflake/arctic-embed-l", input)` |
 | baai/bge-m3 | `embedding(model="nvidia_nim/baai/bge-m3", input)` |
 
+
 ## HuggingFace Embedding Models
 LiteLLM supports all Feature-Extraction + Sentence Similarity Embedding models: https://huggingface.co/models?pipeline_tag=feature-extraction
 
@@ -499,7 +486,7 @@ response = embedding(
 print(response)
 ```
 
-## Supported Models
+### Supported Models
 All models listed here https://docs.voyageai.com/embeddings/#models-and-specifics are supported
 
 | Model Name               | Function Call                                                                                                                                                      |
@@ -508,7 +495,7 @@ All models listed here https://docs.voyageai.com/embeddings/#models-and-specific
 | voyage-lite-01 | `embedding(model="voyage/voyage-lite-01", input)` | 
 | voyage-lite-01-instruct | `embedding(model="voyage/voyage-lite-01-instruct", input)` | 
 
-## Provider-specific Params
+### Provider-specific Params
 
 
 :::info
@@ -570,3 +557,28 @@ curl -X POST 'http://0.0.0.0:4000/v1/embeddings' \
 ```
 </TabItem>
 </Tabs>
+
+## Nebius AI Studio Embedding Models
+
+### Usage - Embedding
+```python
+from litellm import embedding
+import os
+
+os.environ['NEBIUS_API_KEY'] = ""
+response = embedding(
+    model="nebius/BAAI/bge-en-icl",
+    input=["Good morning from litellm!"],
+)
+print(response)
+```
+
+### Supported Models
+All supported models can be found here: https://studio.nebius.ai/models/embedding
+
+| Model Name               | Function Call                                                   |
+|--------------------------|-----------------------------------------------------------------|
+| BAAI/bge-en-icl | `embedding(model="nebius/BAAI/bge-en-icl", input)`              | 
+| BAAI/bge-multilingual-gemma2 | `embedding(model="nebius/BAAI/bge-multilingual-gemma2", input)` | 
+| intfloat/e5-mistral-7b-instruct | `embedding(model="nebius/intfloat/e5-mistral-7b-instruct", input)`      | 
+

docs/my-website/docs/enterprise.md

@@ -7,6 +7,8 @@ For companies that need SSO, user management and professional support for LiteLL
 Get free 7-day trial key [here](https://www.litellm.ai/#trial)
 :::
 
+## Enterprise Features
+
 Includes all enterprise features.
 
 <Image img={require('../img/enterprise_vs_oss.png')} />
@@ -18,32 +20,13 @@ This covers:
 - [**Enterprise Features**](./proxy/enterprise)
 - ✅ **Feature Prioritization**
 - ✅ **Custom Integrations**
-- ✅ **Professional Support - Dedicated discord + slack**
+- ✅ **Professional Support - Dedicated Slack/Teams channel**
 
 
-Deployment Options:
+## Self-Hosted
 
-**Self-Hosted**
-1. Manage Yourself - you can deploy our Docker Image or build a custom image from our pip package, and manage your own infrastructure. In this case, we would give you a license key + provide support via a dedicated support channel. 
+Manage Yourself - you can deploy our Docker Image or build a custom image from our pip package, and manage your own infrastructure. In this case, we would give you a license key + provide support via a dedicated support channel. 
 
-2. We Manage - you give us subscription access on your AWS/Azure/GCP account, and we manage the deployment.
-
-**Managed**
-
-You can use our cloud product where we setup a dedicated instance for you. 
-
-## Frequently Asked Questions
-
-### SLA's + Professional Support
-
-Professional Support can assist with LLM/Provider integrations, deployment, upgrade management, and LLM Provider troubleshooting.  We can’t solve your own infrastructure-related issues but we will guide you to fix them.
-
-- 1 hour for Sev0 issues - 100% production traffic is failing
-- 6 hours for Sev1 - <100% production traffic is failing
-- 24h for Sev2-Sev3 between 7am – 7pm PT (Monday through Saturday) - setup issues e.g. Redis working on our end, but not on your infrastructure.
-- 72h SLA for patching vulnerabilities in the software. 
-
-**We can offer custom SLAs** based on your needs and the severity of the issue
 
 ### What’s the cost of the Self-Managed Enterprise edition?
 
@@ -58,8 +41,72 @@ You just deploy [our docker image](https://docs.litellm.ai/docs/proxy/deploy) an
 LITELLM_LICENSE="eyJ..."
 ```
 
-No data leaves your environment. 
+**No data leaves your environment.** 
+
+
+## Hosted LiteLLM Proxy
+
+LiteLLM maintains the proxy, so you can focus on your core products. 
+
+We provide a dedicated proxy for your team, and manage the infrastructure. 
+
+### **Status**: GA 
+
+Our proxy is already used in production by customers. 
+
+See our status page for [**live reliability**](https://status.litellm.ai/)
+
+### **Benefits**
+- **No Maintenance, No Infra**: We'll maintain the proxy, and spin up any additional infrastructure (e.g.: separate server for spend logs) to make sure you can load balance + track spend across multiple LLM projects. 
+- **Reliable**: Our hosted proxy is tested on 1k requests per second, making it reliable for high load.
+- **Secure**: LiteLLM is SOC-2 Type 2 and ISO 27001 certified, to make sure your data is as secure as possible.
+
+### Supported data regions for LiteLLM Cloud
+
+You can find [supported data regions litellm here](../docs/data_security#supported-data-regions-for-litellm-cloud)
+
+
+## Frequently Asked Questions
+
+### SLA's + Professional Support
+
+Professional Support can assist with LLM/Provider integrations, deployment, upgrade management, and LLM Provider troubleshooting.  We can’t solve your own infrastructure-related issues but we will guide you to fix them.
+
+- 1 hour for Sev0 issues - 100% production traffic is failing
+- 6 hours for Sev1 - < 100% production traffic is failing
+- 24h for Sev2-Sev3 between 7am – 7pm PT (Monday through Saturday) - setup issues e.g. Redis working on our end, but not on your infrastructure.
+- 72h SLA for patching vulnerabilities in the software. 
+
+**We can offer custom SLAs** based on your needs and the severity of the issue
 
 ## Data Security / Legal / Compliance FAQs
 
-[Data Security / Legal / Compliance FAQs](./data_security.md)
+[Data Security / Legal / Compliance FAQs](./data_security.md)
+
+
+### Pricing
+
+Pricing is based on usage. We can figure out a price that works for your team, on the call. 
+
+[**Contact Us to learn more**](https://calendly.com/d/4mp-gd3-k5k/litellm-1-1-onboarding-chat)
+
+
+
+## **Screenshots**
+
+### 1. Create keys
+
+<Image img={require('../img/litellm_hosted_ui_create_key.png')} />
+
+### 2. Add Models
+
+<Image img={require('../img/litellm_hosted_ui_add_models.png')}/>
+
+### 3. Track spend 
+
+<Image img={require('../img/litellm_hosted_usage_dashboard.png')} />
+
+
+### 4. Configure load balancing 
+
+<Image img={require('../img/litellm_hosted_ui_router.png')} />

docs/my-website/docs/extras/contributing.md

@@ -1,3 +1,6 @@
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+
 # Contributing to Documentation
 
 This website is built using [Docusaurus 2](https://docusaurus.io/), a modern static website generator.
@@ -9,31 +12,51 @@ git clone https://github.com/BerriAI/litellm.git
 
 ### Local setup for locally running docs
 
-#### Installation
-```
-npm install --global yarn
-```
-
-
-### Local Development
-
 ```
 cd docs/my-website
 ```
 
-Let's Install requirement
 
+<Tabs>
+
+<TabItem value="yarn" label="Yarn">
+
+Installation
+```
+npm install --global yarn
+```
+Install requirement
 ```
 yarn
 ```
 Run website
-
 ```
 yarn start
 ```
-Open docs here: [http://localhost:3000/](http://localhost:3000/)
 
+</TabItem>
+
+<TabItem value="pnpm" label="pnpm">
+
+Installation
 ```
+npm install --global pnpm
+```
+Install requirement
+```
+pnpm install
+```
+Run website
+```
+pnpm start
+```
+
+</TabItem>
+
+</Tabs>
+
+
+Open docs here: [http://localhost:3000/](http://localhost:3000/)
 
 This command builds your Markdown files into HTML and starts a development server to browse your documentation. Open up [http://127.0.0.1:8000/](http://127.0.0.1:8000/) in your web browser to see your documentation. You can make changes to your Markdown files and your docs will automatically rebuild.
 
@@ -42,8 +65,4 @@ This command builds your Markdown files into HTML and starts a development serve
 ### Making changes to Docs
 - All the docs are placed under the `docs` directory
 - If you are adding a new `.md` file or editing the hierarchy edit `mkdocs.yml` in the root of the project
-- After testing your changes, make a change to the `main` branch of [github.com/BerriAI/litellm](https://github.com/BerriAI/litellm)
-
-
-
-
+- After testing your changes, make a change/pull request to the `main` branch of [github.com/BerriAI/litellm](https://github.com/BerriAI/litellm)

docs/my-website/docs/extras/contributing_code.md

@@ -4,20 +4,23 @@
 
 Here are the core requirements for any PR submitted to LiteLLM
 
-
+- [ ] Sign the Contributor License Agreement (CLA) - [see details](#contributor-license-agreement-cla)
 - [ ] Add testing, **Adding at least 1 test is a hard requirement** - [see details](#2-adding-testing-to-your-pr)
 - [ ] Ensure your PR passes the following tests:
-    - [ ] [Unit Tests](#3-running-unit-tests)
-    - [ ] [Formatting / Linting Tests](#35-running-linting-tests)
+  - [ ] [Unit Tests](#3-running-unit-tests)
+  - [ ] [Formatting / Linting Tests](#35-running-linting-tests)
 - [ ] Keep scope as isolated as possible. As a general rule, your changes should address 1 specific problem at a time
 
+## **Contributor License Agreement (CLA)**
 
+Before contributing code to LiteLLM, you must sign our [Contributor License Agreement (CLA)](https://cla-assistant.io/BerriAI/litellm). This is a legal requirement for all contributions to be merged into the main repository. The CLA helps protect both you and the project by clearly defining the terms under which your contributions are made.
+
+**Important:** We strongly recommend reviewing and signing the CLA before starting work on your contribution to avoid any delays in the PR process. You can find the CLA [here](https://cla-assistant.io/BerriAI/litellm) and sign it through our CLA management system when you submit your first PR.
 
 ## Quick start
 
 ## 1. Setup your local dev environment
 
-
 Here's how to modify the repo locally:
 
 Step 1: Clone the repo
@@ -36,14 +39,14 @@ That's it, your local dev environment is ready!
 
 ## 2. Adding Testing to your PR
 
-- Add your test to the [`tests/litellm/` directory](https://github.com/BerriAI/litellm/tree/main/tests/litellm)
+- Add your test to the [`tests/test_litellm/` directory](https://github.com/BerriAI/litellm/tree/main/tests/litellm)
 
 - This directory 1:1 maps the the `litellm/` directory, and can only contain mocked tests.
 - Do not add real llm api calls to this directory.
 
-### 2.1 File Naming Convention for `tests/litellm/`
+### 2.1 File Naming Convention for `tests/test_litellm/`
 
-The `tests/litellm/` directory follows the same directory structure as `litellm/`.
+The `tests/test_litellm/` directory follows the same directory structure as `litellm/`.
 
 - `litellm/proxy/test_caching_routes.py` maps to `litellm/proxy/caching_routes.py`
 - `test_{filename}.py` maps to `litellm/{filename}.py`
@@ -71,9 +74,9 @@ LiteLLM uses mypy for linting. On ci/cd we also run `black` for formatting.
 - push your fork to your GitHub repo
 - submit a PR from there
 
-
 ## Advanced
-### Building LiteLLM Docker Image 
+
+### Building LiteLLM Docker Image
 
 Some people might want to build the LiteLLM docker image themselves. Follow these instructions if you want to build / run the LiteLLM Docker Image yourself.
 

docs/my-website/docs/guides/security_settings.md

@@ -1,14 +1,14 @@
 import Tabs from '@theme/Tabs';
 import TabItem from '@theme/TabItem';
 
-# SSL Security Settings
+# SSL, HTTP Proxy Security Settings
 
 If you're in an environment using an older TTS bundle, with an older encryption, follow this guide. 
 
 
 LiteLLM uses HTTPX for network requests, unless otherwise specified. 
 
-1. Disable SSL verification
+## 1. Disable SSL verification
 
 
 <Tabs>
@@ -35,7 +35,7 @@ export SSL_VERIFY="False"
 </TabItem>
 </Tabs>
 
-2. Lower security settings
+## 2. Lower security settings
 
 <Tabs>
 <TabItem value="sdk" label="SDK">
@@ -63,4 +63,29 @@ export SSL_CERTIFICATE="/path/to/certificate.pem"
 </TabItem>
 </Tabs>
 
+## 3. Use HTTP_PROXY environment variable
+
+Both httpx and aiohttp libraries use `urllib.request.getproxies` from environment variables. Before client initialization, you may set proxy (and optional SSL_CERT_FILE) by setting the environment variables:
+
+<Tabs>
+<TabItem value="sdk" label="SDK">
+
+```python
+import litellm
+litellm.aiohttp_trust_env = True
+```
+
+```bash
+export HTTPS_PROXY='http://username:password@proxy_uri:port'
+```
+</TabItem>
+
+<TabItem value="proxy" label="PROXY">
+
+```bash
+export HTTPS_PROXY='http://username:password@proxy_uri:port'
+export AIOHTTP_TRUST_ENV='True'
+```
+</TabItem>
+</Tabs>
 

docs/my-website/docs/hosted.md

@@ -1,66 +0,0 @@
-import Image from '@theme/IdealImage';
-
-# Hosted LiteLLM Proxy
-
-LiteLLM maintains the proxy, so you can focus on your core products. 
-
-## [**Get Onboarded**](https://calendly.com/d/4mp-gd3-k5k/litellm-1-1-onboarding-chat)
-
-This is in alpha. Schedule a call with us, and we'll give you a hosted proxy within 30 minutes. 
-
-[**🚨 Schedule Call**](https://calendly.com/d/4mp-gd3-k5k/litellm-1-1-onboarding-chat)
-
-### **Status**: Alpha 
-
-Our proxy is already used in production by customers. 
-
-See our status page for [**live reliability**](https://status.litellm.ai/)
-
-### **Benefits**
-- **No Maintenance, No Infra**: We'll maintain the proxy, and spin up any additional infrastructure (e.g.: separate server for spend logs) to make sure you can load balance + track spend across multiple LLM projects. 
-- **Reliable**: Our hosted proxy is tested on 1k requests per second, making it reliable for high load.
-- **Secure**: LiteLLM is currently undergoing SOC-2 compliance, to make sure your data is as secure as possible.
-
-## Data Privacy & Security
-
-You can find our [data privacy & security policy for cloud litellm here](../docs/data_security#litellm-cloud)
-
-## Supported data regions for LiteLLM Cloud
-
-You can find [supported data regions litellm here](../docs/data_security#supported-data-regions-for-litellm-cloud)
-
-### Pricing
-
-Pricing is based on usage. We can figure out a price that works for your team, on the call. 
-
-[**🚨 Schedule Call**](https://calendly.com/d/4mp-gd3-k5k/litellm-1-1-onboarding-chat)
-
-## **Screenshots**
-
-### 1. Create keys
-
-<Image img={require('../img/litellm_hosted_ui_create_key.png')} />
-
-### 2. Add Models
-
-<Image img={require('../img/litellm_hosted_ui_add_models.png')}/>
-
-### 3. Track spend 
-
-<Image img={require('../img/litellm_hosted_usage_dashboard.png')} />
-
-
-### 4. Configure load balancing 
-
-<Image img={require('../img/litellm_hosted_ui_router.png')} />
-
-#### [**🚨 Schedule Call**](https://calendly.com/d/4mp-gd3-k5k/litellm-1-1-onboarding-chat)
-
-## Feature List 
-
-- Easy way to add/remove models
-- 100% uptime even when models are added/removed
-- custom callback webhooks
-- your domain name with HTTPS
-- Ability to create/delete User API keys
-- Reasonable set monthly cost

docs/my-website/docs/image_edits.md

@@ -0,0 +1,211 @@
+import Image from '@theme/IdealImage';
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+
+# /images/edits
+
+LiteLLM provides image editing functionality that maps to OpenAI's `/images/edits` API endpoint.
+
+| Feature | Supported | Notes |
+|---------|-----------|--------|
+| Cost Tracking | ✅ | Works with all supported models |
+| Logging | ✅ | Works across all integrations |
+| End-user Tracking | ✅ | |
+| Fallbacks | ✅ | Works between supported models |
+| Loadbalancing | ✅ | Works between supported models |
+| Supported operations | Create image edits | |
+| Supported LiteLLM SDK Versions | 1.63.8+ | |
+| Supported LiteLLM Proxy Versions | 1.71.1+ | |
+| Supported LLM providers | **OpenAI** | Currently only `openai` is supported |
+
+## Usage
+
+### LiteLLM Python SDK
+
+<Tabs>
+<TabItem value="openai" label="OpenAI">
+
+#### Basic Image Edit
+```python showLineNumbers title="OpenAI Image Edit"
+import litellm
+
+# Edit an image with a prompt
+response = litellm.image_edit(
+    model="gpt-image-1",
+    image=open("original_image.png", "rb"),
+    prompt="Add a red hat to the person in the image",
+    n=1,
+    size="1024x1024"
+)
+
+print(response)
+```
+
+#### Image Edit with Mask
+```python showLineNumbers title="OpenAI Image Edit with Mask"
+import litellm
+
+# Edit an image with a mask to specify the area to edit
+response = litellm.image_edit(
+    model="gpt-image-1",
+    image=open("original_image.png", "rb"),
+    mask=open("mask_image.png", "rb"),  # Transparent areas will be edited
+    prompt="Replace the background with a beach scene",
+    n=2,
+    size="512x512",
+    response_format="url"
+)
+
+print(response)
+```
+
+#### Async Image Edit
+```python showLineNumbers title="Async OpenAI Image Edit"
+import litellm
+import asyncio
+
+async def edit_image():
+    response = await litellm.aimage_edit(
+        model="gpt-image-1",
+        image=open("original_image.png", "rb"),
+        prompt="Make the image look like a painting",
+        n=1,
+        size="1024x1024",
+        response_format="b64_json"
+    )
+    return response
+
+# Run the async function
+response = asyncio.run(edit_image())
+print(response)
+```
+
+#### Image Edit with Custom Parameters
+```python showLineNumbers title="OpenAI Image Edit with Custom Parameters"
+import litellm
+
+# Edit image with additional parameters
+response = litellm.image_edit(
+    model="gpt-image-1",
+    image=open("portrait.png", "rb"),
+    prompt="Add sunglasses and a smile",
+    n=3,
+    size="1024x1024",
+    response_format="url",
+    user="user-123",
+    timeout=60,
+    extra_headers={"Custom-Header": "value"}
+)
+
+print(f"Generated {len(response.data)} image variations")
+for i, image_data in enumerate(response.data):
+    print(f"Image {i+1}: {image_data.url}")
+```
+
+</TabItem>
+</Tabs>
+
+### LiteLLM Proxy with OpenAI SDK
+
+
+<Tabs>
+<TabItem value="openai" label="OpenAI">
+
+First, add this to your litellm proxy config.yaml:
+```yaml showLineNumbers title="OpenAI Proxy Configuration"
+model_list:
+  - model_name: gpt-image-1
+    litellm_params:
+      model: gpt-image-1
+      api_key: os.environ/OPENAI_API_KEY
+```
+
+Start the LiteLLM proxy server:
+
+```bash showLineNumbers title="Start LiteLLM Proxy Server"
+litellm --config /path/to/config.yaml
+
+# RUNNING on http://0.0.0.0:4000
+```
+
+#### Basic Image Edit via Proxy
+```python showLineNumbers title="OpenAI Proxy Image Edit"
+from openai import OpenAI
+
+# Initialize client with your proxy URL
+client = OpenAI(
+    base_url="http://localhost:4000",  # Your proxy URL
+    api_key="your-api-key"             # Your proxy API key
+)
+
+# Edit an image
+response = client.images.edit(
+    model="gpt-image-1",
+    image=open("original_image.png", "rb"),
+    prompt="Add a red hat to the person in the image",
+    n=1,
+    size="1024x1024"
+)
+
+print(response)
+```
+
+#### cURL Example
+```bash showLineNumbers title="cURL Image Edit Request"
+curl -X POST "http://localhost:4000/v1/images/edits" \
+  -H "Authorization: Bearer your-api-key" \
+  -F "model=gpt-image-1" \
+  -F "image=@original_image.png" \
+  -F "mask=@mask_image.png" \
+  -F "prompt=Add a beautiful sunset in the background" \
+  -F "n=1" \
+  -F "size=1024x1024" \
+  -F "response_format=url"
+```
+
+</TabItem>
+</Tabs>
+
+## Supported Image Edit Parameters
+
+| Parameter | Type | Description | Required |
+|-----------|------|-------------|----------|
+| `image` | `FileTypes` | The image to edit. Must be a valid PNG file, less than 4MB, and square. | ✅ |
+| `prompt` | `str` | A text description of the desired image edit. | ✅ |
+| `model` | `str` | The model to use for image editing | Optional (defaults to `dall-e-2`) |
+| `mask` | `str` | An additional image whose fully transparent areas indicate where the original image should be edited. Must be a valid PNG file, less than 4MB, and have the same dimensions as `image`. | Optional |
+| `n` | `int` | The number of images to generate. Must be between 1 and 10. | Optional (defaults to 1) |
+| `size` | `str` | The size of the generated images. Must be one of `256x256`, `512x512`, or `1024x1024`. | Optional (defaults to `1024x1024`) |
+| `response_format` | `str` | The format in which the generated images are returned. Must be one of `url` or `b64_json`. | Optional (defaults to `url`) |
+| `user` | `str` | A unique identifier representing your end-user. | Optional |
+
+
+## Response Format
+
+The response follows the OpenAI Images API format:
+
+```python showLineNumbers title="Image Edit Response Structure"
+{
+    "created": 1677649800,
+    "data": [
+        {
+            "url": "https://example.com/edited_image_1.png"
+        },
+        {
+            "url": "https://example.com/edited_image_2.png"
+        }
+    ]
+}
+```
+
+For `b64_json` format:
+```python showLineNumbers title="Base64 Response Structure"
+{
+    "created": 1677649800,
+    "data": [
+        {
+            "b64_json": "iVBORw0KGgoAAAANSUhEUgAA..."
+        }
+    ]
+}
+```

docs/my-website/docs/image_generation.md

@@ -1,8 +1,15 @@
-# Images
+
+import Image from '@theme/IdealImage';
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+
+# Image Generations
 
 ## Quick Start
 
-```python
+### LiteLLM Python SDK
+
+```python showLineNumbers
 from litellm import image_generation
 import os 
 
@@ -14,24 +21,23 @@ response = image_generation(prompt="A cute baby sea otter", model="dall-e-3")
 print(f"response: {response}")
 ```
 
-## Proxy Usage
+### LiteLLM Proxy
 
 ### Setup config.yaml 
 
-```yaml
+```yaml showLineNumbers
 model_list:
   - model_name: gpt-image-1 ### RECEIVED MODEL NAME ###
     litellm_params: # all params accepted by litellm.image_generation()
       model: azure/gpt-image-1 ### MODEL NAME sent to `litellm.image_generation()` ###
       api_base: https://my-endpoint-europe-berri-992.openai.azure.com/
       api_key: "os.environ/AZURE_API_KEY_EU" # does os.getenv("AZURE_API_KEY_EU")
-      rpm: 6      # [OPTIONAL] Rate limit for this deployment: in requests per minute (rpm)
 
 ```
 
 ### Start proxy 
 
-```bash
+```bash showLineNumbers
 litellm --config /path/to/config.yaml 
 
 # RUNNING on http://0.0.0.0:4000
@@ -57,7 +63,7 @@ curl -X POST 'http://0.0.0.0:4000/v1/images/generations' \
 </TabItem>
 <TabItem value="openai" label="OpenAI">
 
-```python
+```python showLineNumbers
 from openai import OpenAI
 client = openai.OpenAI(
     api_key="sk-1234",
@@ -108,11 +114,15 @@ Any non-openai params, will be treated as provider-specific params, and sent in
 
 - `n`: *int (optional)* The number of images to generate. Must be between 1 and 10. For dall-e-3, only n=1 is supported.
 
-- `quality`: *string (optional)* The quality of the image that will be generated. hd creates images with finer details and greater consistency across the image. This param is only supported for dall-e-3.
-
+- `quality`: *string (optional)* The quality of the image that will be generated.
+  *   `auto` (default value) will automatically select the best quality for the given model.
+  *   `high`, `medium` and `low` are supported for `gpt-image-1`.
+  *   `hd` and `standard` are supported for `dall-e-3`.
+  *   `standard` is the only option for `dall-e-2`.
+  
 - `response_format`: *string (optional)* The format in which the generated images are returned. Must be one of url or b64_json.
 
-- `size`: *string (optional)* The size of the generated images. Must be one of 256x256, 512x512, or 1024x1024 for gpt-image-1. Must be one of 1024x1024, 1792x1024, or 1024x1792 for dall-e-3 models.
+- `size`: *string (optional)* The size of the generated images. Must be one of `1024x1024`, `1536x1024` (landscape), `1024x1536` (portrait), or `auto` (default value) for `gpt-image-1`, one of `256x256`, `512x512`, or `1024x1024` for `dall-e-2`, and one of `1024x1024`, `1792x1024`, or `1024x1792` for `dall-e-3`.
 
 - `timeout`: *integer* - The maximum time, in seconds, to wait for the API to respond. Defaults to 600 seconds (10 minutes).
 

docs/my-website/docs/index.md

@@ -208,6 +208,22 @@ response = completion(
 )
 ```
 
+</TabItem>
+<TabItem value="novita" label="Novita AI">
+
+```python
+from litellm import completion
+import os
+
+## set ENV variables. Visit https://novita.ai/settings/key-management to get your API key
+os.environ["NOVITA_API_KEY"] = "novita-api-key"
+
+response = completion(
+  model="novita/deepseek/deepseek-r1",
+  messages=[{ "content": "Hello, how are you?","role": "user"}]
+)
+```
+
 </TabItem>
 
 </Tabs>
@@ -411,6 +427,23 @@ response = completion(
 )
 ```
 
+</TabItem>
+<TabItem value="novita" label="Novita AI">
+
+```python
+from litellm import completion
+import os
+
+## set ENV variables. Visit https://novita.ai/settings/key-management to get your API key
+os.environ["NOVITA_API_KEY"] = "novita_api_key"
+
+response = completion(
+  model="novita/deepseek/deepseek-r1",
+  messages = [{ "content": "Hello, how are you?","role": "user"}],
+  stream=True,
+)
+```
+
 </TabItem>
 
 </Tabs>

docs/my-website/docs/integrations/index.md

@@ -0,0 +1,5 @@
+# Integrations
+
+This section covers integrations with various tools and services that can be used with LiteLLM (either Proxy or SDK).
+
+Click into each section to learn more about the integrations.

docs/my-website/docs/mcp.md

@@ -4,9 +4,7 @@ import Image from '@theme/IdealImage';
 
 # /mcp [BETA] - Model Context Protocol
 
-## Expose MCP tools on LiteLLM Proxy Server
-
-This allows you to define tools that can be called by any MCP compatible client. Define your `mcp_servers` with LiteLLM and all your clients can list and call available tools.
+LiteLLM Proxy provides an MCP Gateway that allows you to use a fixed endpoint for all MCP tools and control MCP access by Key, Team. 
 
 <Image 
   img={require('../img/mcp_2.png')}
@@ -16,28 +14,36 @@ This allows you to define tools that can be called by any MCP compatible client.
   LiteLLM MCP Architecture: Use MCP tools with all LiteLLM supported models
 </p>
 
-#### How it works
+## Overview
+| Feature | Description |
+|---------|-------------|
+| MCP Operations | • List Tools<br/>• Call Tools |
+| Supported MCP Transports | • Streamable HTTP<br/>• SSE |
+| LiteLLM Permission Management | ✨ Enterprise Only<br/>• By Key<br/>• By Team<br/>• By Organization |
 
-LiteLLM exposes the following MCP endpoints:
+## Adding your MCP
 
-- `/mcp/tools/list` - List all available tools
-- `/mcp/tools/call` - Call a specific tool with the provided arguments
+<Tabs>
+<TabItem value="ui" label="LiteLLM UI">
 
-When MCP clients connect to LiteLLM they can follow this workflow:
+On the LiteLLM UI, Navigate to "MCP Servers" and click "Add New MCP Server".
 
-1. Connect to the LiteLLM MCP server
-2. List all available tools on LiteLLM
-3. Client makes LLM API request with tool call(s)
-4. LLM API returns which tools to call and with what arguments
-5. MCP client makes MCP tool calls to LiteLLM
-6. LiteLLM makes the tool calls to the appropriate MCP server
-7. LiteLLM returns the tool call results to the MCP client
+On this form, you should enter your MCP Server URL and the transport you want to use.
 
-#### Usage
+LiteLLM supports the following MCP transports:
+- Streamable HTTP
+- SSE (Server-Sent Events)
 
-#### 1. Define your tools on under `mcp_servers` in your config.yaml file.
+<Image 
+  img={require('../img/add_mcp.png')}
+  style={{width: '80%', display: 'block', margin: '0'}}
+/>
 
-LiteLLM allows you to define your tools on the `mcp_servers` section in your config.yaml file. All tools listed here will be available to MCP clients (when they connect to LiteLLM and call `list_tools`).
+</TabItem>
+
+<TabItem value="config" label="config.yaml">
+
+Add your MCP servers directly in your `config.yaml` file:
 
 ```yaml title="config.yaml" showLineNumbers
 model_list:
@@ -47,131 +53,416 @@ model_list:
       api_key: sk-xxxxxxx
 
 mcp_servers:
-  {
-    "zapier_mcp": {
-      "url": "https://actions.zapier.com/mcp/sk-akxxxxx/sse"
-    },
-    "fetch": {
-      "url": "http://localhost:8000/sse"
-    }
-  }
+  # HTTP Streamable Server
+  deepwiki_mcp:
+    url: "https://mcp.deepwiki.com/mcp"
+  # SSE Server
+  zapier_mcp:
+    url: "https://actions.zapier.com/mcp/sk-akxxxxx/sse"
+  
+  # Full configuration with all optional fields
+  my_http_server:
+    url: "https://my-mcp-server.com/mcp"
+    transport: "http"
+    description: "My custom MCP server"
+    auth_type: "api_key"
+    spec_version: "2025-03-26"
 ```
 
+**Configuration Options:**
+- **Server Name**: Use any descriptive name for your MCP server (e.g., `zapier_mcp`, `deepwiki_mcp`)
+- **URL**: The endpoint URL for your MCP server (required)
+- **Transport**: Optional transport type (defaults to `sse`)
+  - `sse` - SSE (Server-Sent Events) transport
+  - `http` - Streamable HTTP transport
+- **Description**: Optional description for the server
+- **Auth Type**: Optional authentication type
+- **Spec Version**: Optional MCP specification version (defaults to `2025-03-26`)
 
-#### 2. Start LiteLLM Gateway
+</TabItem>
+</Tabs>
+
+
+
+## Using your MCP
 
 <Tabs>
-<TabItem value="docker" label="Docker Run">
+<TabItem value="openai" label="OpenAI API">
 
-```shell title="Docker Run" showLineNumbers
-docker run -d \
-  -p 4000:4000 \
-  -e OPENAI_API_KEY=$OPENAI_API_KEY \
-  --name my-app \
-  -v $(pwd)/my_config.yaml:/app/config.yaml \
-  my-app:latest \
-  --config /app/config.yaml \
-  --port 4000 \
-  --detailed_debug \
+#### Connect via OpenAI Responses API
+
+Use the OpenAI Responses API to connect to your LiteLLM MCP server:
+
+```bash title="cURL Example" showLineNumbers
+curl --location 'https://api.openai.com/v1/responses' \
+--header 'Content-Type: application/json' \
+--header "Authorization: Bearer $OPENAI_API_KEY" \
+--data '{
+    "model": "gpt-4o",
+    "tools": [
+        {
+            "type": "mcp",
+            "server_label": "litellm",
+            "server_url": "<your-litellm-proxy-base-url>/mcp",
+            "require_approval": "never",
+            "headers": {
+                "x-litellm-api-key": "Bearer YOUR_LITELLM_API_KEY"
+            }
+        }
+    ],
+    "input": "Run available tools",
+    "tool_choice": "required"
+}'
 ```
 
 </TabItem>
 
-<TabItem value="py" label="litellm pip">
+<TabItem value="litellm" label="LiteLLM Proxy">
 
-```shell title="litellm pip" showLineNumbers
-litellm --config config.yaml --detailed_debug
+#### Connect via LiteLLM Proxy Responses API
+
+Use this when calling LiteLLM Proxy for LLM API requests to `/v1/responses` endpoint.
+
+```bash title="cURL Example" showLineNumbers
+curl --location '<your-litellm-proxy-base-url>/v1/responses' \
+--header 'Content-Type: application/json' \
+--header "Authorization: Bearer $LITELLM_API_KEY" \
+--data '{
+    "model": "gpt-4o",
+    "tools": [
+        {
+            "type": "mcp",
+            "server_label": "litellm",
+            "server_url": "<your-litellm-proxy-base-url>/mcp",
+            "require_approval": "never",
+            "headers": {
+                "x-litellm-api-key": "Bearer YOUR_LITELLM_API_KEY"
+            }
+        }
+    ],
+    "input": "Run available tools",
+    "tool_choice": "required"
+}'
+```
+
+</TabItem>
+
+<TabItem value="cursor" label="Cursor IDE">
+
+#### Connect via Cursor IDE
+
+Use tools directly from Cursor IDE with LiteLLM MCP:
+
+**Setup Instructions:**
+
+1. **Open Cursor Settings**: Use `⇧+⌘+J` (Mac) or `Ctrl+Shift+J` (Windows/Linux)
+2. **Navigate to MCP Tools**: Go to the "MCP Tools" tab and click "New MCP Server"
+3. **Add Configuration**: Copy and paste the JSON configuration below, then save with `Cmd+S` or `Ctrl+S`
+
+```json title="Cursor MCP Configuration" showLineNumbers
+{
+  "mcpServers": {
+    "LiteLLM": {
+      "url": "<your-litellm-proxy-base-url>/mcp",
+      "headers": {
+        "x-litellm-api-key": "Bearer $LITELLM_API_KEY"
+      }
+    }
+  }
+}
+```
+
+</TabItem>
+
+<TabItem value="http" label="Streamable HTTP">
+
+#### Connect via Streamable HTTP Transport
+
+Connect to LiteLLM MCP using HTTP transport. Compatible with any MCP client that supports HTTP streaming:
+
+**Server URL:**
+```text showLineNumbers
+<your-litellm-proxy-base-url>/mcp
+```
+
+**Headers:**
+```text showLineNumbers
+x-litellm-api-key: Bearer YOUR_LITELLM_API_KEY
+```
+
+This URL can be used with any MCP client that supports HTTP transport. Refer to your client documentation to determine the appropriate transport method.
+
+</TabItem>
+
+<TabItem value="fastmcp" label="Python FastMCP">
+
+#### Connect via Python FastMCP Client
+
+Use the Python FastMCP client to connect to your LiteLLM MCP server:
+
+**Installation:**
+
+```bash title="Install FastMCP" showLineNumbers
+pip install fastmcp
+```
+
+or with uv:
+
+```bash title="Install with uv" showLineNumbers
+uv pip install fastmcp
+```
+
+**Usage:**
+
+```python title="Python FastMCP Example" showLineNumbers
+import asyncio
+import json
+
+from fastmcp import Client
+from fastmcp.client.transports import StreamableHttpTransport
+
+# Create the transport with your LiteLLM MCP server URL
+server_url = "<your-litellm-proxy-base-url>/mcp"
+transport = StreamableHttpTransport(
+    server_url,
+    headers={
+        "x-litellm-api-key": "Bearer YOUR_LITELLM_API_KEY"
+    }
+)
+
+# Initialize the client with the transport
+client = Client(transport=transport)
+
+
+async def main():
+    # Connection is established here
+    print("Connecting to LiteLLM MCP server...")
+    async with client:
+        print(f"Client connected: {client.is_connected()}")
+
+        # Make MCP calls within the context
+        print("Fetching available tools...")
+        tools = await client.list_tools()
+
+        print(f"Available tools: {json.dumps([t.name for t in tools], indent=2)}")
+        
+        # Example: Call a tool (replace 'tool_name' with an actual tool name)
+        if tools:
+            tool_name = tools[0].name
+            print(f"Calling tool: {tool_name}")
+            
+            # Call the tool with appropriate arguments
+            result = await client.call_tool(tool_name, arguments={})
+            print(f"Tool result: {result}")
+
+
+# Run the example
+if __name__ == "__main__":
+    asyncio.run(main())
 ```
 
 </TabItem>
 </Tabs>
 
 
-#### 3. Make an LLM API request 
+## Using your MCP with client side credentials
 
-In this example we will do the following:
+Use this if you want to pass a client side authentication token to LiteLLM to then pass to your MCP to auth to your MCP.
 
-1. Use MCP client to list MCP tools on LiteLLM Proxy
-2. Use `transform_mcp_tool_to_openai_tool` to convert MCP tools to OpenAI tools
-3. Provide the MCP tools to `gpt-4o`
-4. Handle tool call from `gpt-4o`
-5. Convert OpenAI tool call to MCP tool call
-6. Execute tool call on MCP server
+You can specify your MCP auth token using the header `x-mcp-auth`. LiteLLM will forward this token to your MCP server for authentication.
 
-```python title="MCP Client List Tools" showLineNumbers
+<Tabs>
+<TabItem value="openai" label="OpenAI API">
+
+#### Connect via OpenAI Responses API with MCP Auth
+
+Use the OpenAI Responses API and include the `x-mcp-auth` header for your MCP server authentication:
+
+```bash title="cURL Example with MCP Auth" showLineNumbers
+curl --location 'https://api.openai.com/v1/responses' \
+--header 'Content-Type: application/json' \
+--header "Authorization: Bearer $OPENAI_API_KEY" \
+--data '{
+    "model": "gpt-4o",
+    "tools": [
+        {
+            "type": "mcp",
+            "server_label": "litellm",
+            "server_url": "<your-litellm-proxy-base-url>/mcp",
+            "require_approval": "never",
+            "headers": {
+                "x-litellm-api-key": "Bearer YOUR_LITELLM_API_KEY",
+                "x-mcp-auth": YOUR_MCP_AUTH_TOKEN
+            }
+        }
+    ],
+    "input": "Run available tools",
+    "tool_choice": "required"
+}'
+```
+
+</TabItem>
+
+<TabItem value="litellm" label="LiteLLM Proxy">
+
+#### Connect via LiteLLM Proxy Responses API with MCP Auth
+
+Use this when calling LiteLLM Proxy for LLM API requests to `/v1/responses` endpoint with MCP authentication:
+
+```bash title="cURL Example with MCP Auth" showLineNumbers
+curl --location '<your-litellm-proxy-base-url>/v1/responses' \
+--header 'Content-Type: application/json' \
+--header "Authorization: Bearer $LITELLM_API_KEY" \
+--data '{
+    "model": "gpt-4o",
+    "tools": [
+        {
+            "type": "mcp",
+            "server_label": "litellm",
+            "server_url": "<your-litellm-proxy-base-url>/mcp",
+            "require_approval": "never",
+            "headers": {
+                "x-litellm-api-key": "Bearer YOUR_LITELLM_API_KEY",
+                "x-mcp-auth": "YOUR_MCP_AUTH_TOKEN"
+            }
+        }
+    ],
+    "input": "Run available tools",
+    "tool_choice": "required"
+}'
+```
+
+</TabItem>
+
+<TabItem value="cursor" label="Cursor IDE">
+
+#### Connect via Cursor IDE with MCP Auth
+
+Use tools directly from Cursor IDE with LiteLLM MCP and include your MCP authentication token:
+
+**Setup Instructions:**
+
+1. **Open Cursor Settings**: Use `⇧+⌘+J` (Mac) or `Ctrl+Shift+J` (Windows/Linux)
+2. **Navigate to MCP Tools**: Go to the "MCP Tools" tab and click "New MCP Server"
+3. **Add Configuration**: Copy and paste the JSON configuration below, then save with `Cmd+S` or `Ctrl+S`
+
+```json title="Cursor MCP Configuration with Auth" showLineNumbers
+{
+  "mcpServers": {
+    "LiteLLM": {
+      "url": "<your-litellm-proxy-base-url>/mcp",
+      "headers": {
+        "x-litellm-api-key": "Bearer $LITELLM_API_KEY",
+        "x-mcp-auth": "$MCP_AUTH_TOKEN"
+      }
+    }
+  }
+}
+```
+
+</TabItem>
+
+<TabItem value="http" label="Streamable HTTP">
+
+#### Connect via Streamable HTTP Transport with MCP Auth
+
+Connect to LiteLLM MCP using HTTP transport with MCP authentication:
+
+**Server URL:**
+```text showLineNumbers
+<your-litellm-proxy-base-url>/mcp
+```
+
+**Headers:**
+```text showLineNumbers
+x-litellm-api-key: Bearer YOUR_LITELLM_API_KEY
+x-mcp-auth: Bearer YOUR_MCP_AUTH_TOKEN
+```
+
+This URL can be used with any MCP client that supports HTTP transport. The `x-mcp-auth` header will be forwarded to your MCP server for authentication.
+
+</TabItem>
+
+<TabItem value="fastmcp" label="Python FastMCP">
+
+#### Connect via Python FastMCP Client with MCP Auth
+
+Use the Python FastMCP client to connect to your LiteLLM MCP server with MCP authentication:
+
+```python title="Python FastMCP Example with MCP Auth" showLineNumbers
 import asyncio
-from openai import AsyncOpenAI
-from openai.types.chat import ChatCompletionUserMessageParam
-from mcp import ClientSession
-from mcp.client.sse import sse_client
-from litellm.experimental_mcp_client.tools import (
-    transform_mcp_tool_to_openai_tool,
-    transform_openai_tool_call_request_to_mcp_tool_call_request,
+import json
+
+from fastmcp import Client
+from fastmcp.client.transports import StreamableHttpTransport
+
+# Create the transport with your LiteLLM MCP server URL and auth headers
+server_url = "<your-litellm-proxy-base-url>/mcp"
+transport = StreamableHttpTransport(
+    server_url,
+    headers={
+        "x-litellm-api-key": "Bearer YOUR_LITELLM_API_KEY",
+        "x-mcp-auth": "Bearer YOUR_MCP_AUTH_TOKEN"
+    }
 )
 
+# Initialize the client with the transport
+client = Client(transport=transport)
+
 
 async def main():
-    # Initialize clients
-    
-    # point OpenAI client to LiteLLM Proxy
-    client = AsyncOpenAI(api_key="sk-1234", base_url="http://localhost:4000")
+    # Connection is established here
+    print("Connecting to LiteLLM MCP server with authentication...")
+    async with client:
+        print(f"Client connected: {client.is_connected()}")
 
-    # Point MCP client to LiteLLM Proxy
-    async with sse_client("http://localhost:4000/mcp/") as (read, write):
-        async with ClientSession(read, write) as session:
-            await session.initialize()
+        # Make MCP calls within the context
+        print("Fetching available tools...")
+        tools = await client.list_tools()
 
-            # 1. List MCP tools on LiteLLM Proxy
-            mcp_tools = await session.list_tools()
-            print("List of MCP tools for MCP server:", mcp_tools.tools)
-
-            # Create message
-            messages = [
-                ChatCompletionUserMessageParam(
-                    content="Send an email about LiteLLM supporting MCP", role="user"
-                )
-            ]
-
-            # 2. Use `transform_mcp_tool_to_openai_tool` to convert MCP tools to OpenAI tools
-            # Since OpenAI only supports tools in the OpenAI format, we need to convert the MCP tools to the OpenAI format.
-            openai_tools = [
-                transform_mcp_tool_to_openai_tool(tool) for tool in mcp_tools.tools
-            ]
-
-            # 3. Provide the MCP tools to `gpt-4o`
-            response = await client.chat.completions.create(
-                model="gpt-4o",
-                messages=messages,
-                tools=openai_tools,
-                tool_choice="auto",
-            )
-
-            # 4. Handle tool call from `gpt-4o`
-            if response.choices[0].message.tool_calls:
-                tool_call = response.choices[0].message.tool_calls[0]
-                if tool_call:
-
-                    # 5. Convert OpenAI tool call to MCP tool call
-                    # Since MCP servers expect tools in the MCP format, we need to convert the OpenAI tool call to the MCP format.
-                    # This is done using litellm.experimental_mcp_client.tools.transform_openai_tool_call_request_to_mcp_tool_call_request
-                    mcp_call = (
-                        transform_openai_tool_call_request_to_mcp_tool_call_request(
-                            openai_tool=tool_call.model_dump()
-                        )
-                    )
-
-                    # 6. Execute tool call on MCP server
-                    result = await session.call_tool(
-                        name=mcp_call.name, arguments=mcp_call.arguments
-                    )
-
-                    print("Result:", result)
+        print(f"Available tools: {json.dumps([t.name for t in tools], indent=2)}")
+        
+        # Example: Call a tool (replace 'tool_name' with an actual tool name)
+        if tools:
+            tool_name = tools[0].name
+            print(f"Calling tool: {tool_name}")
+            
+            # Call the tool with appropriate arguments
+            result = await client.call_tool(tool_name, arguments={})
+            print(f"Tool result: {result}")
 
 
-# Run it
-asyncio.run(main())
+# Run the example
+if __name__ == "__main__":
+    asyncio.run(main())
 ```
 
+</TabItem>
+</Tabs>
+
+
+## ✨ MCP Permission Management
+
+LiteLLM supports managing permissions for MCP Servers by Keys, Teams, Organizations (entities) on LiteLLM. When a MCP client attempts to list tools, LiteLLM will only return the tools the entity has permissions to access.
+
+When Creating a Key, Team, or Organization, you can select the allowed MCP Servers that the entity has access to.
+
+<Image 
+  img={require('../img/mcp_key.png')}
+  style={{width: '80%', display: 'block', margin: '0'}}
+/>
+
+
+## LiteLLM Proxy - Walk through MCP Gateway
+LiteLLM exposes an MCP Gateway for admins to add all their MCP servers to LiteLLM. The key benefits of using LiteLLM Proxy with MCP are:
+
+1. Use a fixed endpoint for all MCP tools
+2. MCP Permission management by Key, Team, or User
+
+This video demonstrates how you can onboard an MCP server to LiteLLM Proxy, use it and set access controls.
+
+<iframe width="840" height="500" src="https://www.loom.com/embed/f7aa8d217879430987f3e64291757bfc" frameborder="0" webkitallowfullscreen mozallowfullscreen allowfullscreen></iframe>
+
 ## LiteLLM Python SDK MCP Bridge
 
 LiteLLM Python SDK acts as a MCP bridge to utilize MCP tools with all LiteLLM supported models. LiteLLM offers the following features for using MCP
@@ -424,4 +715,4 @@ async with stdio_client(server_params) as (read, write):
 ```
 
 </TabItem>
-</Tabs>
+</Tabs>

docs/my-website/docs/observability/argilla.md

@@ -50,7 +50,7 @@ For further configuration, please refer to the [Argilla documentation](https://d
 ## Usage
 
 <Tabs>
-<Tab value="sdk" label="SDK">
+<TabItem value="sdk" label="SDK">
 
 ```python
 import os
@@ -78,9 +78,9 @@ response = completion(
 )
 ```
 
-</Tab>
+</TabItem>
 
-<Tab value="proxy" label="PROXY">
+<TabItem value="proxy" label="PROXY">
 
 ```yaml
 litellm_settings:
@@ -90,7 +90,7 @@ litellm_settings:
     llm_output: "response"
 ```
 
-</Tab>
+</TabItem>
 </Tabs>
 
 ## Example Output

docs/my-website/docs/observability/braintrust.md

@@ -2,25 +2,24 @@ import Image from '@theme/IdealImage';
 import Tabs from '@theme/Tabs';
 import TabItem from '@theme/TabItem';
 
-# Braintrust - Evals + Logging 
+# Braintrust - Evals + Logging
 
 [Braintrust](https://www.braintrust.dev/) manages evaluations, logging, prompt playground, to data management for AI products.
 
-
 ## Quick Start
 
 ```python
-# pip install langfuse 
+# pip install braintrust
 import litellm
 import os
 
-# set env 
-os.environ["BRAINTRUST_API_KEY"] = "" 
+# set env
+os.environ["BRAINTRUST_API_KEY"] = ""
 os.environ['OPENAI_API_KEY']=""
 
 # set braintrust as a callback, litellm will send the data to braintrust
-litellm.callbacks = ["braintrust"] 
- 
+litellm.callbacks = ["braintrust"]
+
 # openai call
 response = litellm.completion(
   model="gpt-3.5-turbo",
@@ -30,16 +29,16 @@ response = litellm.completion(
 )
 ```
 
-
-
 ## OpenAI Proxy Usage
 
-1. Add keys to env 
+1. Add keys to env
+
 ```env
-BRAINTRUST_API_KEY="" 
+BRAINTRUST_API_KEY=""
 ```
 
-2. Add braintrust to callbacks 
+2. Add braintrust to callbacks
+
 ```yaml
 model_list:
   - model_name: gpt-3.5-turbo
@@ -47,12 +46,11 @@ model_list:
       model: gpt-3.5-turbo
       api_key: os.environ/OPENAI_API_KEY
 
-
 litellm_settings:
   callbacks: ["braintrust"]
 ```
 
-3. Test it! 
+3. Test it!
 
 ```bash
 curl -X POST 'http://0.0.0.0:4000/chat/completions' \
@@ -69,6 +67,8 @@ curl -X POST 'http://0.0.0.0:4000/chat/completions' \
 
 ## Advanced - pass Project ID or name
 
+It is recommended that you include the `project_id` or `project_name` to ensure your traces are being written out to the correct Braintrust project.
+
 <Tabs>
 <TabItem value="sdk" label="SDK">
 
@@ -77,12 +77,28 @@ response = litellm.completion(
   model="gpt-3.5-turbo",
   messages=[
     {"role": "user", "content": "Hi 👋 - i'm openai"}
-  ], 
+  ],
   metadata={
     "project_id": "1234",
     # passing project_name will try to find a project with that name, or create one if it doesn't exist
     # if both project_id and project_name are passed, project_id will be used
-    # "project_name": "my-special-project" 
+    # "project_name": "my-special-project"
+  }
+)
+```
+
+Note: Other `metadata` can be included here as well when using the SDK.
+
+```python
+response = litellm.completion(
+  model="gpt-3.5-turbo",
+  messages=[
+    {"role": "user", "content": "Hi 👋 - i'm openai"}
+  ],
+  metadata={
+    "project_id": "1234",
+    "item1": "an item",
+    "item2": "another item"
   }
 )
 ```
@@ -127,7 +143,7 @@ response = client.chat.completions.create(
         }
     ],
     extra_body={ # pass in any provider-specific param, if not supported by openai, https://docs.litellm.ai/docs/completion/input#provider-specific-params
-        "metadata": { # 👈 use for logging additional params (e.g. to langfuse)
+        "metadata": { # 👈 use for logging additional params (e.g. to braintrust)
             "project_id": "my-special-project"
         }
     }
@@ -141,10 +157,10 @@ For more examples, [**Click Here**](../proxy/user_keys.md#chatcompletions)
 </TabItem>
 </Tabs>
 
-## Full API Spec 
+## Full API Spec
 
-Here's everything you can pass in metadata for a braintrust request 
+Here's everything you can pass in metadata for a braintrust request
 
-`braintrust_*` - any metadata field starting with `braintrust_` will be passed as metadata to the logging request 
+`braintrust_*` - If you are adding metadata from _proxy request headers_, any metadata field starting with `braintrust_` will be passed as metadata to the logging request. If you are using the SDK, just pass your metadata like normal (e.g., `metadata={"project_name": "my-test-project", "item1": "an item", "item2": "another item"}`)
 
-`project_id`  - set the project id for a braintrust call. Default is `litellm`. 
+`project_id` - Set the project id for a braintrust call. Default is `litellm`.

docs/my-website/docs/observability/deepeval_integration.md

@@ -0,0 +1,55 @@
+import Image from '@theme/IdealImage';
+
+# 🔭 DeepEval - Open-Source Evals with Tracing
+
+### What is DeepEval?
+[DeepEval](https://deepeval.com) is an open-source evaluation framework for LLMs ([Github](https://github.com/confident-ai/deepeval)). 
+
+### What is Confident AI?
+
+[Confident AI](https://documentation.confident-ai.com) (the ***deepeval*** platfrom) offers an Observatory for teams to trace and monitor LLM applications. Think Datadog for LLM apps. The observatory allows you to:
+
+- Detect and debug issues in your LLM applications in real-time
+- Search and analyze historical generation data with powerful filters
+- Collect human feedback on model responses
+- Run evaluations to measure and improve performance
+- Track costs and latency to optimize resource usage
+
+<Image img={require('../../img/deepeval_dashboard.png')} />
+
+### Quickstart
+
+```python
+import os
+import time
+import litellm
+
+
+os.environ['OPENAI_API_KEY']='<your-openai-api-key>'
+os.environ['CONFIDENT_API_KEY']='<your-confident-api-key>'
+
+litellm.success_callback = ["deepeval"]
+litellm.failure_callback = ["deepeval"]
+
+try:
+    response = litellm.completion(
+        model="gpt-3.5-turbo",
+        messages=[
+            {"role": "user", "content": "What's the weather like in San Francisco?"}
+        ],
+    )
+except Exception as e:
+    print(e)
+
+print(response)
+```
+
+:::info
+You can obtain your `CONFIDENT_API_KEY` by logging into [Confident AI](https://app.confident-ai.com/project) platform. 
+:::
+
+## Support & Talk with Deepeval team
+- [Confident AI Docs 📝](https://documentation.confident-ai.com)
+- [Platform 🚀](https://confident-ai.com)
+- [Community Discord 💭](https://discord.gg/wuPM9dRgDw)
+- Support ✉️ support@confident-ai.com

docs/my-website/docs/observability/helicone_integration.md

@@ -52,6 +52,7 @@ from litellm import completion
 ## Set env variables
 os.environ["HELICONE_API_KEY"] = "your-helicone-key"
 os.environ["OPENAI_API_KEY"] = "your-openai-key"
+# os.environ["HELICONE_API_BASE"] = "" # [OPTIONAL] defaults to `https://api.helicone.ai`
 
 # Set callbacks
 litellm.success_callback = ["helicone"]

docs/my-website/docs/observability/langfuse_integration.md

@@ -21,7 +21,7 @@ Example trace in Langfuse using multiple models via LiteLLM:
 ### Pre-Requisites
 Ensure you have run `pip install langfuse` for this integration
 ```shell
-pip install langfuse>=2.0.0 litellm
+pip install langfuse==2.45.0 litellm
 ```
 
 ### Quick Start

docs/my-website/docs/observability/langfuse_otel_integration.md

@@ -0,0 +1,181 @@
+# Langfuse OpenTelemetry Integration
+
+The Langfuse OpenTelemetry integration allows you to send LiteLLM traces and observability data to Langfuse using the OpenTelemetry protocol. This provides a standardized way to collect and analyze your LLM usage data.
+
+## Features
+
+- Automatic trace collection for all LiteLLM requests
+- Support for Langfuse Cloud (EU and US regions)
+- Support for self-hosted Langfuse instances
+- Custom endpoint configuration
+- Secure authentication using Basic Auth
+- Consistent attribute mapping with other OTEL integrations
+
+## Prerequisites
+
+1. **Langfuse Account**: Sign up at [Langfuse Cloud](https://cloud.langfuse.com) or set up a self-hosted instance
+2. **API Keys**: Get your public and secret keys from your Langfuse project settings
+3. **Dependencies**: Install required packages:
+   ```bash
+   pip install litellm opentelemetry-api opentelemetry-sdk
+   ```
+
+## Configuration
+
+### Environment Variables
+
+| Variable | Required | Description | Example |
+|----------|----------|-------------|---------|
+| `LANGFUSE_PUBLIC_KEY` | Yes | Your Langfuse public key | `pk-lf-...` |
+| `LANGFUSE_SECRET_KEY` | Yes | Your Langfuse secret key | `sk-lf-...` |
+| `LANGFUSE_HOST` | No | Langfuse host URL | `https://us.cloud.langfuse.com` (default) |
+
+### Endpoint Resolution
+
+The integration automatically constructs the OTEL endpoint from the `LANGFUSE_HOST`:
+- **Default (US)**: `https://us.cloud.langfuse.com/api/public/otel`
+- **EU Region**: `https://cloud.langfuse.com/api/public/otel`
+- **Self-hosted**: `{LANGFUSE_HOST}/api/public/otel`
+
+## Usage
+
+### Basic Setup
+
+```python
+import os
+import litellm
+
+# Set your Langfuse credentials
+os.environ["LANGFUSE_PUBLIC_KEY"] = "pk-lf-..."
+os.environ["LANGFUSE_SECRET_KEY"] = "sk-lf-..."
+
+# Enable Langfuse OTEL integration
+litellm.callbacks = ["langfuse_otel"]
+
+# Make LLM requests as usual
+response = litellm.completion(
+    model="gpt-3.5-turbo",
+    messages=[{"role": "user", "content": "Hello!"}]
+)
+```
+
+### Advanced Configuration
+
+```python
+import os
+import litellm
+
+# Set your Langfuse credentials
+os.environ["LANGFUSE_PUBLIC_KEY"] = "pk-lf-..."
+os.environ["LANGFUSE_SECRET_KEY"] = "sk-lf-..."
+
+# Use EU region
+os.environ["LANGFUSE_HOST"] = "https://cloud.langfuse.com"  # EU region
+# os.environ["LANGFUSE_HOST"] = "https://us.cloud.langfuse.com"  # US region (default)
+
+# Or use self-hosted instance
+# os.environ["LANGFUSE_HOST"] = "https://my-langfuse.company.com"
+
+litellm.callbacks = ["langfuse_otel"]
+```
+
+### Manual OTEL Configuration
+
+If you need direct control over the OpenTelemetry configuration:
+
+```python
+import os
+import base64
+import litellm
+
+# Get keys for your project from the project settings page: https://cloud.langfuse.com
+os.environ["LANGFUSE_PUBLIC_KEY"] = "pk-lf-..." 
+os.environ["LANGFUSE_SECRET_KEY"] = "sk-lf-..." 
+os.environ["LANGFUSE_HOST"] = "https://cloud.langfuse.com" # EU region
+# os.environ["LANGFUSE_HOST"] = "https://us.cloud.langfuse.com" # US region
+
+LANGFUSE_AUTH = base64.b64encode(
+    f"{os.environ.get('LANGFUSE_PUBLIC_KEY')}:{os.environ.get('LANGFUSE_SECRET_KEY')}".encode()
+).decode()
+
+os.environ["OTEL_EXPORTER_OTLP_ENDPOINT"] = os.environ.get("LANGFUSE_HOST") + "/api/public/otel"
+os.environ["OTEL_EXPORTER_OTLP_HEADERS"] = f"Authorization=Basic {LANGFUSE_AUTH}"
+
+litellm.callbacks = ["langfuse_otel"]
+```
+
+### With LiteLLM Proxy
+
+Add the integration to your proxy configuration:
+
+```yaml
+# config.yaml
+litellm_settings:
+  callbacks: ["langfuse_otel"]
+
+environment_variables:
+  LANGFUSE_PUBLIC_KEY: "pk-lf-..."
+  LANGFUSE_SECRET_KEY: "sk-lf-..."
+  LANGFUSE_HOST: "https://us.cloud.langfuse.com"  # Default US region
+```
+
+## Data Collected
+
+The integration automatically collects the following data:
+
+- **Request Details**: Model, messages, parameters (temperature, max_tokens, etc.)
+- **Response Details**: Generated content, token usage, finish reason
+- **Timing Information**: Request duration, time to first token
+- **Metadata**: User ID, session ID, custom tags (if provided)
+- **Error Information**: Exception details and stack traces (if errors occur)
+
+## Authentication
+
+The integration uses HTTP Basic Authentication with your Langfuse public and secret keys:
+
+```
+Authorization: Basic <base64(public_key:secret_key)>
+```
+
+This is automatically handled by the integration - you just need to provide the keys via environment variables.
+
+## Troubleshooting
+
+### Common Issues
+
+1. **Missing Credentials Error**
+   ```
+   ValueError: LANGFUSE_PUBLIC_KEY and LANGFUSE_SECRET_KEY must be set
+   ```
+   **Solution**: Ensure both environment variables are set with valid keys.
+
+2. **Connection Issues**
+   - Check your internet connection
+   - Verify the endpoint URL is correct
+   - For self-hosted instances, ensure the `/api/public/otel` endpoint is accessible
+
+3. **Authentication Errors**
+   - Verify your public and secret keys are correct
+   - Check that the keys belong to the same Langfuse project
+   - Ensure the keys have the necessary permissions
+
+### Debug Mode
+
+Enable verbose logging to see detailed information:
+
+```python
+import litellm
+litellm.set_verbose = True
+```
+
+This will show:
+- Endpoint resolution logic
+- Authentication header creation
+- OTEL trace submission details
+
+## Related Links
+
+- [Langfuse Documentation](https://langfuse.com/docs)
+- [Langfuse OpenTelemetry Guide](https://langfuse.com/docs/integrations/opentelemetry)
+- [OpenTelemetry Python SDK](https://opentelemetry.io/docs/languages/python/)
+- [LiteLLM Observability](https://docs.litellm.ai/docs/observability/) 

docs/my-website/docs/observability/langsmith_integration.md

@@ -1,4 +1,6 @@
 import Image from '@theme/IdealImage';
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
 
 # Langsmith - Logging LLM Input/Output
 
@@ -22,10 +24,13 @@ pip install litellm
 ## Quick Start
 Use just 2 lines of code, to instantly log your responses **across all providers** with Langsmith
 
+<Tabs>
+<TabItem value="python" label="SDK">
 
 ```python
-litellm.success_callback = ["langsmith"]
+litellm.callbacks = ["langsmith"]
 ```
+
 ```python
 import litellm
 import os
@@ -37,7 +42,7 @@ os.environ["LANGSMITH_DEFAULT_RUN_NAME"] = "" # defaults to LLMRun
 os.environ['OPENAI_API_KEY']=""
 
 # set langsmith as a callback, litellm will send the data to langsmith
-litellm.success_callback = ["langsmith"] 
+litellm.callbacks = ["langsmith"] 
  
 # openai call
 response = litellm.completion(
@@ -47,8 +52,124 @@ response = litellm.completion(
   ]
 )
 ```
+</TabItem>
+<TabItem value="proxy" label="LiteLLM Proxy">
+
+1. Setup config.yaml
+```yaml
+model_list:
+  - model_name: gpt-3.5-turbo
+    litellm_params:
+      model: openai/gpt-3.5-turbo
+      api_key: os.environ/OPENAI_API_KEY
+
+litellm_settings:
+  callbacks: ["langsmith"]
+```
+
+2. Start LiteLLM Proxy
+```bash
+litellm --config /path/to/config.yaml
+```
+
+3. Test it!
+```bash
+curl -L -X POST 'http://0.0.0.0:4000/v1/chat/completions' \
+-H 'Content-Type: application/json' \
+-H 'Authorization: Bearer sk-eWkpOhYaHiuIZV-29JDeTQ' \
+-d '{
+  "model": "gpt-3.5-turbo",
+  "messages": [
+    {
+      "role": "user",
+      "content": "Hey, how are you?"
+    }
+  ],
+  "max_completion_tokens": 250
+}'
+```
+</TabItem>
+</Tabs>
+
+
 
 ## Advanced
+
+### Local Testing - Control Batch Size
+
+Set the size of the batch that Langsmith will process at a time, default is 512. 
+
+Set `langsmith_batch_size=1` when testing locally, to see logs land quickly.
+
+<Tabs>
+<TabItem value="python" label="SDK">
+
+```python
+import litellm
+import os
+
+os.environ["LANGSMITH_API_KEY"] = ""
+# LLM API Keys
+os.environ['OPENAI_API_KEY']=""
+
+# set langsmith as a callback, litellm will send the data to langsmith
+litellm.callbacks = ["langsmith"] 
+litellm.langsmith_batch_size = 1 # 👈 KEY CHANGE
+ 
+response = litellm.completion(
+    model="gpt-3.5-turbo",
+     messages=[
+        {"role": "user", "content": "Hi 👋 - i'm openai"}
+    ]
+)
+print(response)
+```
+</TabItem>
+<TabItem value="proxy" label="LiteLLM Proxy">
+
+1. Setup config.yaml
+```yaml
+model_list:
+  - model_name: gpt-3.5-turbo
+    litellm_params:
+      model: openai/gpt-3.5-turbo
+      api_key: os.environ/OPENAI_API_KEY
+
+litellm_settings:
+  langsmith_batch_size: 1
+  callbacks: ["langsmith"]
+```
+
+2. Start LiteLLM Proxy
+```bash
+litellm --config /path/to/config.yaml
+```
+
+3. Test it!
+```bash
+curl -L -X POST 'http://0.0.0.0:4000/v1/chat/completions' \
+-H 'Content-Type: application/json' \
+-H 'Authorization: Bearer sk-eWkpOhYaHiuIZV-29JDeTQ' \
+-d '{
+  "model": "gpt-3.5-turbo",
+  "messages": [
+    {
+      "role": "user",
+      "content": "Hey, how are you?"
+    }
+  ],
+  "max_completion_tokens": 250
+}'
+```
+
+
+
+</TabItem>
+</Tabs>
+
+
+
+
 ### Set Langsmith fields
 
 ```python

docs/my-website/docs/observability/opentelemetry_integration.md

@@ -34,8 +34,9 @@ OTEL_HEADERS="Authorization=Bearer%20<your-api-key>"
 <TabItem value="otel-col" label="Log to OTEL HTTP Collector">
 
 ```shell
-OTEL_EXPORTER="otlp_http"
-OTEL_ENDPOINT="http://0.0.0.0:4318"
+OTEL_EXPORTER_OTLP_ENDPOINT="http://0.0.0.0:4318"
+OTEL_EXPORTER_OTLP_PROTOCOL=http/json
+OTEL_EXPORTER_OTLP_HEADERS="api-key=key,other-config-value=value"
 ```
 
 </TabItem>
@@ -43,8 +44,9 @@ OTEL_ENDPOINT="http://0.0.0.0:4318"
 <TabItem value="otel-col-grpc" label="Log to OTEL GRPC Collector">
 
 ```shell
-OTEL_EXPORTER="otlp_grpc"
-OTEL_ENDPOINT="http://0.0.0.0:4317"
+OTEL_EXPORTER_OTLP_ENDPOINT="http://0.0.0.0:4318"
+OTEL_EXPORTER_OTLP_PROTOCOL=grpc
+OTEL_EXPORTER_OTLP_HEADERS="api-key=key,other-config-value=value"
 ```
 
 </TabItem>
@@ -98,7 +100,7 @@ LiteLLM emits the user_api_key_metadata
 - user_id
 - team_id
 
-for successful + failed requests 
+for successful + failed requests
 
 click under `litellm_request` in the trace
 

docs/my-website/docs/observability/phoenix_integration.md

@@ -1,6 +1,6 @@
 import Image from '@theme/IdealImage';
 
-# Phoenix OSS
+# Arize Phoenix OSS
 
 Open source tracing and evaluation platform
 
@@ -21,6 +21,9 @@ Use just 2 lines of code, to instantly log your responses **across all providers
 
 You can also use the instrumentor option instead of the callback, which you can find [here](https://docs.arize.com/phoenix/tracing/integrations-tracing/litellm).
 
+```bash
+pip install opentelemetry-api opentelemetry-sdk opentelemetry-exporter-otlp litellm[proxy]
+```
 ```python
 litellm.callbacks = ["arize_phoenix"]
 ```
@@ -29,14 +32,14 @@ import litellm
 import os
 
 os.environ["PHOENIX_API_KEY"] = "" # Necessary only using Phoenix Cloud
-os.environ["PHOENIX_COLLECTOR_HTTP_ENDPOINT"] = "" # The URL of your Phoenix OSS instance
+os.environ["PHOENIX_COLLECTOR_HTTP_ENDPOINT"] = "" # The URL of your Phoenix OSS instance e.g. http://localhost:6006/v1/traces
 # This defaults to https://app.phoenix.arize.com/v1/traces for Phoenix Cloud
 
 # LLM API Keys
 os.environ['OPENAI_API_KEY']=""
 
 # set arize as a callback, litellm will send the data to arize
-litellm.callbacks = ["phoenix"]
+litellm.callbacks = ["arize_phoenix"]
  
 # openai call
 response = litellm.completion(

docs/my-website/docs/observability/sentry.md

@@ -49,6 +49,18 @@ response = completion(model="gpt-3.5-turbo", messages=[{"role": "user", "content
 print(response)
 ```
 
+#### Sample Rate Options
+
+- **SENTRY_API_SAMPLE_RATE**: Controls what percentage of errors are sent to Sentry
+  - Value between 0 and 1 (default is 1.0 or 100% of errors)
+  - Example: 0.5 sends 50% of errors, 0.1 sends 10% of errors
+
+- **SENTRY_API_TRACE_RATE**: Controls what percentage of transactions are sampled for performance monitoring
+  - Value between 0 and 1 (default is 1.0 or 100% of transactions)
+  - Example: 0.5 traces 50% of transactions, 0.1 traces 10% of transactions
+
+These options are useful for high-volume applications where sampling a subset of errors and transactions provides sufficient visibility while managing costs.
+
 ## Redacting Messages, Response Content from Sentry Logging 
 
 Set `litellm.turn_off_message_logging=True` This will prevent the messages and responses from being logged to sentry, but request metadata will still be logged.

docs/my-website/docs/oidc.md

@@ -19,6 +19,7 @@ LiteLLM supports the following OIDC identity providers:
 | CircleCI v2              | `circleci_v2`| No               |
 | GitHub Actions           | `github`     | Yes              |
 | Azure Kubernetes Service | `azure`      | No               |
+| Azure AD                 | `azure`      | Yes              |
 | File                     | `file`       | No               |
 | Environment Variable     | `env`        | No               |
 | Environment Path         | `env_path`   | No               |
@@ -261,3 +262,15 @@ The custom role below is the recommended minimum permissions for the Azure appli
 _Note: Your UUIDs will be different._
 
 Please contact us for paid enterprise support if you need help setting up Azure AD applications.
+
+### Azure AD -> Amazon Bedrock
+```yaml
+model list:
+  - model_name: aws/claude-3-5-sonnet
+    litellm_params:
+      model: bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0
+      aws_region_name: "eu-central-1"
+      aws_role_name: "arn:aws:iam::12345678:role/bedrock-role"
+      aws_web_identity_token: "oidc/azure/api://123-456-789-9d04"
+      aws_session_name: "litellm-session"
+```

docs/my-website/docs/pass_through/bedrock.md

@@ -257,7 +257,7 @@ proxy_endpoint = "http://0.0.0.0:4000/bedrock" # 👈 your proxy base url
 # # Create a Config object with the proxy
 # Custom headers
 custom_headers = {
-    'litellm_user_api_key': 'sk-1234', # 👈 your proxy api key
+    'litellm_user_api_key': 'Bearer sk-1234', # 👈 your proxy api key
 }
 
 
@@ -274,9 +274,7 @@ runtime_client = boto3.client(
 
 # Custom header injection
 def inject_custom_headers(request, **kwargs):
-    request.headers.update({
-        'litellm_user_api_key': 'sk-1234',
-    })
+    request.headers.update(custom_headers)
 
 # Attach the event to inject custom headers before the request is sent
 runtime_client.meta.events.register('before-send.*.*', inject_custom_headers)

docs/my-website/docs/pass_through/vertex_ai.md

@@ -116,7 +116,7 @@ curl \
 <TabItem value="curl" label="curl">
 
 ```bash
-curl http://localhost:4000/vertex_ai/vertex_ai/v1/projects/${PROJECT_ID}/locations/us-central1/publishers/google/models/${MODEL_ID}:generateContent \
+curl http://localhost:4000/vertex_ai/v1/projects/${PROJECT_ID}/locations/us-central1/publishers/google/models/${MODEL_ID}:generateContent \
   -H "Content-Type: application/json" \
   -H "x-litellm-api-key: Bearer sk-1234" \
   -d '{

docs/my-website/docs/pass_through/vllm.md

@@ -23,12 +23,22 @@ Supports **ALL** VLLM Endpoints (including streaming).
 
 ## Quick Start
 
-Let's call the VLLM [`/metrics` endpoint](https://vllm.readthedocs.io/en/latest/api_reference/api_reference.html)
+Let's call the VLLM [`/score` endpoint](https://vllm.readthedocs.io/en/latest/api_reference/api_reference.html)
 
-1. Add HOSTED VLLM API BASE to your environment 
+1. Add a VLLM hosted model to your LiteLLM Proxy 
 
-```bash
-export HOSTED_VLLM_API_BASE="https://my-vllm-server.com"
+:::info
+
+Works with LiteLLM v1.72.0+. 
+
+:::
+
+```yaml
+model_list:
+  - model_name: "my-vllm-model"
+    litellm_params:
+      model: hosted_vllm/vllm-1.72
+      api_base: https://my-vllm-server.com
 ```
 
 2. Start LiteLLM Proxy 
@@ -41,12 +51,19 @@ litellm
 
 3. Test it! 
 
-Let's call the VLLM `/metrics` endpoint
+Let's call the VLLM `/score` endpoint
 
 ```bash
-curl -L -X GET 'http://0.0.0.0:4000/vllm/metrics' \
--H 'Content-Type: application/json' \
--H 'Authorization: Bearer sk-1234' \
+curl -X 'POST' \
+  'http://0.0.0.0:4000/vllm/score' \
+  -H 'accept: application/json' \
+  -H 'Content-Type: application/json' \
+  -d '{
+  "model": "my-vllm-model",
+  "encoding_format": "float",
+  "text_1": "What is the capital of France?",
+  "text_2": "The capital of France is Paris."
+}'
 ```
 
 

docs/my-website/docs/projects/GPTLocalhost.md

@@ -0,0 +1,3 @@
+# GPTLocalhost
+
+[GPTLocalhost](https://gptlocalhost.com/demo#LiteLLM) - LiteLLM is supported by GPTLocalhost, a local Word Add-in for you to use models in LiteLLM within Microsoft Word. 100% Private.

docs/my-website/docs/providers/anthropic.md

@@ -4,6 +4,8 @@ import TabItem from '@theme/TabItem';
 # Anthropic
 LiteLLM supports all anthropic models.
 
+- `claude-4` (`claude-opus-4-20250514`, `claude-sonnet-4-20250514`)
+- `claude-3.7` (`claude-3-7-sonnet-20250219`)
 - `claude-3.5` (`claude-3-5-sonnet-20240620`)
 - `claude-3` (`claude-3-haiku-20240307`, `claude-3-opus-20240229`, `claude-3-sonnet-20240229`)
 - `claude-2`
@@ -64,7 +66,7 @@ from litellm import completion
 os.environ["ANTHROPIC_API_KEY"] = "your-api-key"
 
 messages = [{"role": "user", "content": "Hey! how's it going?"}]
-response = completion(model="claude-3-opus-20240229", messages=messages)
+response = completion(model="claude-opus-4-20250514", messages=messages)
 print(response)
 ```
 
@@ -80,7 +82,7 @@ from litellm import completion
 os.environ["ANTHROPIC_API_KEY"] = "your-api-key"
 
 messages = [{"role": "user", "content": "Hey! how's it going?"}]
-response = completion(model="claude-3-opus-20240229", messages=messages, stream=True)
+response = completion(model="claude-opus-4-20250514", messages=messages, stream=True)
 for chunk in response:
     print(chunk["choices"][0]["delta"]["content"])  # same as openai format
 ```
@@ -102,9 +104,9 @@ export ANTHROPIC_API_KEY="your-api-key"
 
 ```yaml
 model_list:
-  - model_name: claude-3 ### RECEIVED MODEL NAME ###
+  - model_name: claude-4 ### RECEIVED MODEL NAME ###
     litellm_params: # all params accepted by litellm.completion() - https://docs.litellm.ai/docs/completion/input
-      model: claude-3-opus-20240229 ### MODEL NAME sent to `litellm.completion()` ###
+      model: claude-opus-4-20250514 ### MODEL NAME sent to `litellm.completion()` ###
       api_key: "os.environ/ANTHROPIC_API_KEY" # does os.getenv("AZURE_API_KEY_EU")
 ```
 
@@ -156,7 +158,7 @@ curl --location 'http://0.0.0.0:4000/chat/completions' \
 <TabItem value="cli" label="cli">
 
 ```bash
-$ litellm --model claude-3-opus-20240229
+$ litellm --model claude-opus-4-20250514
 
 # Server running on http://0.0.0.0:4000
 ```
@@ -244,6 +246,9 @@ print(response)
 
 | Model Name       | Function Call                              |
 |------------------|--------------------------------------------|
+| claude-opus-4  | `completion('claude-opus-4-20250514', messages)` | `os.environ['ANTHROPIC_API_KEY']`       |
+| claude-sonnet-4  | `completion('claude-sonnet-4-20250514', messages)` | `os.environ['ANTHROPIC_API_KEY']`       |
+| claude-3.7  | `completion('claude-3-7-sonnet-20250219', messages)` | `os.environ['ANTHROPIC_API_KEY']`       |
 | claude-3-5-sonnet  | `completion('claude-3-5-sonnet-20240620', messages)` | `os.environ['ANTHROPIC_API_KEY']`       |
 | claude-3-haiku  | `completion('claude-3-haiku-20240307', messages)` | `os.environ['ANTHROPIC_API_KEY']`       |
 | claude-3-opus  | `completion('claude-3-opus-20240229', messages)` | `os.environ['ANTHROPIC_API_KEY']`       |
@@ -601,11 +606,6 @@ response = await client.chat.completions.create(
 
 ## **Function/Tool Calling**
 
-:::info 
-
-LiteLLM now uses Anthropic's 'tool' param 🎉 (v1.34.29+)
-:::
-
 ```python
 from litellm import completion
 
@@ -664,6 +664,185 @@ response = completion(
 )
 ```
 
+### Disable Tool Calling
+
+You can disable tool calling by setting the `tool_choice` to `"none"`.
+
+<Tabs>
+<TabItem value="sdk" label="SDK">
+
+```python
+from litellm import completion
+
+response = completion(
+    model="anthropic/claude-3-opus-20240229",
+    messages=messages,
+    tools=tools,
+    tool_choice="none",
+)
+
+```
+</TabItem>
+<TabItem value="proxy" label="Proxy">
+
+1. Setup config.yaml
+
+```yaml
+model_list:
+  - model_name: anthropic-claude-model
+    litellm_params:
+        model: anthropic/claude-3-opus-20240229
+        api_key: os.environ/ANTHROPIC_API_KEY
+```
+
+2. Start proxy
+
+```bash
+litellm --config /path/to/config.yaml
+```
+
+3. Test it! 
+
+Replace `anything` with your LiteLLM Proxy Virtual Key, if [setup](../proxy/virtual_keys).
+
+```bash
+curl http://0.0.0.0:4000/v1/chat/completions \
+  -H "Content-Type: application/json" \
+  -H "Authorization: Bearer anything" \
+  -d '{
+    "model": "anthropic-claude-model",
+    "messages": [{"role": "user", "content": "Who won the World Cup in 2022?"}],
+    "tools": [{"type": "mcp", "server_label": "deepwiki", "server_url": "https://mcp.deepwiki.com/mcp", "require_approval": "never"}],
+    "tool_choice": "none"
+  }'
+```
+</TabItem>
+</Tabs>
+
+
+
+### MCP Tool Calling 
+
+Here's how to use MCP tool calling with Anthropic:
+
+<Tabs>
+<TabItem value="sdk" label="LiteLLM SDK">
+
+LiteLLM supports MCP tool calling with Anthropic in the OpenAI Responses API format.
+
+<Tabs>
+<TabItem value="openai_format" label="OpenAI Format">
+
+
+```python
+import os 
+from litellm import completion
+
+os.environ["ANTHROPIC_API_KEY"] = "sk-ant-..."
+
+tools=[
+    {
+        "type": "mcp",
+        "server_label": "deepwiki",
+        "server_url": "https://mcp.deepwiki.com/mcp",
+        "require_approval": "never",
+    },
+]
+
+response = completion(
+    model="anthropic/claude-sonnet-4-20250514",
+    messages=[{"role": "user", "content": "Who won the World Cup in 2022?"}],
+    tools=tools
+)
+```
+
+</TabItem>
+<TabItem value="anthropic_format" label="Anthropic Format">
+
+```python
+import os 
+from litellm import completion
+
+os.environ["ANTHROPIC_API_KEY"] = "sk-ant-..."
+
+tools = [
+    {
+        "type": "url",
+        "url": "https://mcp.deepwiki.com/mcp",
+        "name": "deepwiki-mcp",
+    }
+]
+response = completion(
+    model="anthropic/claude-sonnet-4-20250514",
+    messages=[{"role": "user", "content": "Who won the World Cup in 2022?"}],
+    tools=tools
+)
+
+print(response)
+```
+</TabItem>
+
+</Tabs>
+
+</TabItem>
+<TabItem value="proxy" label="LiteLLM Proxy">
+
+1. Setup config.yaml
+
+```yaml
+model_list:
+  - model_name: claude-4-sonnet
+    litellm_params:
+        model: anthropic/claude-sonnet-4-20250514
+        api_key: os.environ/ANTHROPIC_API_KEY
+```
+
+2. Start proxy
+
+```bash
+litellm --config /path/to/config.yaml
+```
+
+3. Test it! 
+
+<Tabs>
+<TabItem value="openai" label="OpenAI Format">
+
+```bash
+curl http://0.0.0.0:4000/v1/chat/completions \
+  -H "Content-Type: application/json" \
+  -H "Authorization: Bearer $LITELLM_KEY" \
+  -d '{
+    "model": "claude-4-sonnet",
+    "messages": [{"role": "user", "content": "Who won the World Cup in 2022?"}],
+    "tools": [{"type": "mcp", "server_label": "deepwiki", "server_url": "https://mcp.deepwiki.com/mcp", "require_approval": "never"}]
+  }'
+```
+
+</TabItem>
+<TabItem value="anthropic" label="Anthropic Format">
+
+```bash
+curl http://0.0.0.0:4000/v1/chat/completions \
+  -H "Content-Type: application/json" \
+  -H "Authorization: Bearer $LITELLM_KEY" \
+  -d '{
+    "model": "claude-4-sonnet",
+    "messages": [{"role": "user", "content": "Who won the World Cup in 2022?"}],
+    "tools": [
+        {
+            "type": "url",
+            "url": "https://mcp.deepwiki.com/mcp",
+            "name": "deepwiki-mcp",
+        }
+    ]
+  }'
+```
+
+</TabItem>
+</Tabs>
+</TabItem>
+</Tabs>
 
 ### Parallel Function Calling 
 
@@ -750,7 +929,11 @@ except Exception as e:
 
 s/o @[Shekhar Patnaik](https://www.linkedin.com/in/patnaikshekhar) for requesting this!
 
-### Computer Tools
+### Anthropic Hosted Tools (Computer, Text Editor, Web Search)
+
+
+<Tabs>
+<TabItem value="computer" label="Computer">
 
 ```python
 from litellm import completion
@@ -781,6 +964,205 @@ resp = completion(
 print(resp)
 ```
 
+</TabItem>
+<TabItem value="text_editor" label="Text Editor">
+
+<Tabs>
+<TabItem value="sdk" label="SDK">
+
+```python
+from litellm import completion
+
+tools = [{
+    "type": "text_editor_20250124",
+    "name": "str_replace_editor"
+}]
+model = "claude-3-5-sonnet-20241022"
+messages = [{"role": "user", "content": "There's a syntax error in my primes.py file. Can you help me fix it?"}]
+
+resp = completion(
+    model=model,
+    messages=messages,
+    tools=tools,
+)
+
+print(resp)
+```
+
+</TabItem>
+<TabItem value="proxy" label="PROXY">
+
+1. Setup config.yaml
+
+```yaml
+- model_name: claude-3-5-sonnet-latest
+  litellm_params:
+    model: anthropic/claude-3-5-sonnet-latest
+    api_key: os.environ/ANTHROPIC_API_KEY
+```
+
+2. Start proxy
+
+```bash
+litellm --config /path/to/config.yaml
+```
+
+3. Test it! 
+
+```bash
+curl http://0.0.0.0:4000/v1/chat/completions \
+  -H "Content-Type: application/json" \
+  -H "Authorization: Bearer $LITELLM_KEY" \
+  -d '{
+    "model": "claude-3-5-sonnet-latest",
+    "messages": [{"role": "user", "content": "There's a syntax error in my primes.py file. Can you help me fix it?"}],
+    "tools": [{"type": "text_editor_20250124", "name": "str_replace_editor"}]
+  }'
+```
+</TabItem>
+</Tabs>
+
+</TabItem>
+<TabItem value="web_search" label="Web Search">
+
+:::info
+Live from v1.70.1+
+:::
+
+LiteLLM maps OpenAI's `search_context_size` param to Anthropic's `max_uses` param.
+
+| OpenAI | Anthropic |
+| --- | --- |
+| Low | 1 | 
+| Medium | 5 | 
+| High | 10 | 
+
+
+<Tabs>
+<TabItem value="sdk" label="SDK">
+
+
+<Tabs>
+<TabItem value="openai" label="OpenAI Format">
+
+```python
+from litellm import completion
+
+model = "claude-3-5-sonnet-20241022"
+messages = [{"role": "user", "content": "What's the weather like today?"}]
+
+resp = completion(
+    model=model,
+    messages=messages,
+    web_search_options={
+        "search_context_size": "medium",
+        "user_location": {
+            "type": "approximate",
+            "approximate": {
+                "city": "San Francisco",
+            },
+        }
+    }
+)
+
+print(resp)
+```
+</TabItem>
+<TabItem value="anthropic" label="Anthropic Format">
+
+```python
+from litellm import completion
+
+tools = [{
+    "type": "web_search_20250305",
+    "name": "web_search",
+    "max_uses": 5
+}]
+model = "claude-3-5-sonnet-20241022"
+messages = [{"role": "user", "content": "There's a syntax error in my primes.py file. Can you help me fix it?"}]
+
+resp = completion(
+    model=model,
+    messages=messages,
+    tools=tools,
+)
+
+print(resp)
+```
+</TabItem>
+
+</Tabs>
+</TabItem>
+
+<TabItem value="proxy" label="PROXY">
+
+1. Setup config.yaml
+
+```yaml
+- model_name: claude-3-5-sonnet-latest
+  litellm_params:
+    model: anthropic/claude-3-5-sonnet-latest
+    api_key: os.environ/ANTHROPIC_API_KEY
+```
+
+2. Start proxy
+
+```bash
+litellm --config /path/to/config.yaml
+```
+
+3. Test it! 
+
+<Tabs>
+<TabItem value="openai" label="OpenAI Format">
+
+
+```bash
+curl http://0.0.0.0:4000/v1/chat/completions \
+  -H "Content-Type: application/json" \
+  -H "Authorization: Bearer $LITELLM_KEY" \
+  -d '{
+    "model": "claude-3-5-sonnet-latest",
+    "messages": [{"role": "user", "content": "What's the weather like today?"}],
+    "web_search_options": {
+        "search_context_size": "medium",
+        "user_location": {
+            "type": "approximate",
+            "approximate": {
+                "city": "San Francisco",
+            },
+        }
+    }
+  }'
+```
+</TabItem>
+<TabItem value="anthropic" label="Anthropic Format">
+
+```bash
+curl http://0.0.0.0:4000/v1/chat/completions \
+  -H "Content-Type: application/json" \
+  -H "Authorization: Bearer $LITELLM_KEY" \
+  -d '{
+    "model": "claude-3-5-sonnet-latest",
+    "messages": [{"role": "user", "content": "What's the weather like today?"}],
+    "tools": [{
+        "type": "web_search_20250305",
+        "name": "web_search",
+        "max_uses": 5
+    }]
+  }'
+```
+
+</TabItem>
+</Tabs>
+</TabItem>
+</Tabs>
+
+</TabItem>
+</Tabs>
+
+
+
 ## Usage - Vision 
 
 ```python

docs/my-website/docs/providers/azure/azure.md

@@ -11,7 +11,7 @@ import TabItem from '@theme/TabItem';
 |-------|-------|
 | Description | Azure OpenAI Service provides REST API access to OpenAI's powerful language models including o1, o1-mini, GPT-4o, GPT-4o mini, GPT-4 Turbo with Vision, GPT-4, GPT-3.5-Turbo, and Embeddings model series |
 | Provider Route on LiteLLM | `azure/`, [`azure/o_series/`](#azure-o-series-models) |
-| Supported Operations | [`/chat/completions`](#azure-openai-chat-completion-models), [`/completions`](#azure-instruct-models), [`/embeddings`](../embedding/supported_embedding#azure-openai-embedding-models), [`/audio/speech`](#azure-text-to-speech-tts), [`/audio/transcriptions`](../audio_transcription), `/fine_tuning`, [`/batches`](#azure-batches-api), `/files`, [`/images`](../image_generation#azure-openai-image-generation-models) |
+| Supported Operations | [`/chat/completions`](#azure-openai-chat-completion-models), [`/responses`](./azure_responses), [`/completions`](#azure-instruct-models), [`/embeddings`](./azure_embedding), [`/audio/speech`](#azure-text-to-speech-tts), [`/audio/transcriptions`](../audio_transcription), `/fine_tuning`, [`/batches`](#azure-batches-api), `/files`, [`/images`](../image_generation#azure-openai-image-generation-models) |
 | Link to Provider Doc | [Azure OpenAI ↗](https://learn.microsoft.com/en-us/azure/ai-services/openai/overview)
 
 ## API Keys, Params
@@ -558,6 +558,7 @@ model_list:
       tenant_id: os.environ/AZURE_TENANT_ID
       client_id: os.environ/AZURE_CLIENT_ID
       client_secret: os.environ/AZURE_CLIENT_SECRET
+      azure_scope: os.environ/AZURE_SCOPE  # defaults to "https://cognitiveservices.azure.com/.default"
 ```
 
 Test it 
@@ -594,6 +595,7 @@ model_list:
       client_id: os.environ/AZURE_CLIENT_ID
       azure_username: os.environ/AZURE_USERNAME
       azure_password: os.environ/AZURE_PASSWORD
+      azure_scope: os.environ/AZURE_SCOPE  # defaults to "https://cognitiveservices.azure.com/.default"
 ```
 
 Test it 
@@ -1001,129 +1003,6 @@ Expected Response:
 {"data":[{"id":"batch_R3V...}
 ```
 
-
-## **Azure Responses API**
-
-| Property | Details |
-|-------|-------|
-| Description | Azure OpenAI Responses API |
-| `custom_llm_provider` on LiteLLM | `azure/` |
-| Supported Operations | `/v1/responses`|
-| Azure OpenAI Responses API | [Azure OpenAI Responses API ↗](https://learn.microsoft.com/en-us/azure/ai-services/openai/how-to/responses?tabs=python-secure) |
-| Cost Tracking, Logging Support | ✅ LiteLLM will log, track cost for Responses API Requests |
-| Supported OpenAI Params | ✅ All OpenAI params are supported, [See here](https://github.com/BerriAI/litellm/blob/0717369ae6969882d149933da48eeb8ab0e691bd/litellm/llms/openai/responses/transformation.py#L23) |
-
-## Usage
-
-## Create a model response
-
-<Tabs>
-<TabItem value="litellm-sdk" label="LiteLLM SDK">
-
-#### Non-streaming
-
-```python showLineNumbers title="Azure Responses API"
-import litellm
-
-# Non-streaming response
-response = litellm.responses(
-    model="azure/o1-pro",
-    input="Tell me a three sentence bedtime story about a unicorn.",
-    max_output_tokens=100,
-    api_key=os.getenv("AZURE_RESPONSES_OPENAI_API_KEY"),
-    api_base="https://litellm8397336933.openai.azure.com/",
-    api_version="2023-03-15-preview",
-)
-
-print(response)
-```
-
-#### Streaming
-```python showLineNumbers title="Azure Responses API"
-import litellm
-
-# Streaming response
-response = litellm.responses(
-    model="azure/o1-pro",
-    input="Tell me a three sentence bedtime story about a unicorn.",
-    stream=True,
-    api_key=os.getenv("AZURE_RESPONSES_OPENAI_API_KEY"),
-    api_base="https://litellm8397336933.openai.azure.com/",
-    api_version="2023-03-15-preview",
-)
-
-for event in response:
-    print(event)
-```
-
-</TabItem>
-<TabItem value="proxy" label="OpenAI SDK with LiteLLM Proxy">
-
-First, add this to your litellm proxy config.yaml:
-```yaml showLineNumbers title="Azure Responses API"
-model_list:
-  - model_name: o1-pro
-    litellm_params:
-      model: azure/o1-pro
-      api_key: os.environ/AZURE_RESPONSES_OPENAI_API_KEY
-      api_base: https://litellm8397336933.openai.azure.com/
-      api_version: 2023-03-15-preview
-```
-
-Start your LiteLLM proxy:
-```bash
-litellm --config /path/to/config.yaml
-
-# RUNNING on http://0.0.0.0:4000
-```
-
-Then use the OpenAI SDK pointed to your proxy:
-
-#### Non-streaming
-```python showLineNumbers
-from openai import OpenAI
-
-# Initialize client with your proxy URL
-client = OpenAI(
-    base_url="http://localhost:4000",  # Your proxy URL
-    api_key="your-api-key"             # Your proxy API key
-)
-
-# Non-streaming response
-response = client.responses.create(
-    model="o1-pro",
-    input="Tell me a three sentence bedtime story about a unicorn."
-)
-
-print(response)
-```
-
-#### Streaming
-```python showLineNumbers
-from openai import OpenAI
-
-# Initialize client with your proxy URL
-client = OpenAI(
-    base_url="http://localhost:4000",  # Your proxy URL
-    api_key="your-api-key"             # Your proxy API key
-)
-
-# Streaming response
-response = client.responses.create(
-    model="o1-pro",
-    input="Tell me a three sentence bedtime story about a unicorn.",
-    stream=True
-)
-
-for event in response:
-    print(event)
-```
-
-</TabItem>
-</Tabs>
-
-
-
 ## Advanced
 ### Azure API Load-Balancing
 

docs/my-website/docs/providers/azure/azure_embedding.md

@@ -0,0 +1,93 @@
+import Image from '@theme/IdealImage';
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+
+# Azure OpenAI Embeddings
+
+### API keys
+This can be set as env variables or passed as **params to litellm.embedding()**
+```python
+import os
+os.environ['AZURE_API_KEY'] = 
+os.environ['AZURE_API_BASE'] = 
+os.environ['AZURE_API_VERSION'] = 
+```
+
+### Usage
+```python
+from litellm import embedding
+response = embedding(
+    model="azure/<your deployment name>",
+    input=["good morning from litellm"],
+    api_key=api_key,
+    api_base=api_base,
+    api_version=api_version,
+)
+print(response)
+```
+
+| Model Name           | Function Call                               |
+|----------------------|---------------------------------------------|
+| text-embedding-ada-002 | `embedding(model="azure/<your deployment name>", input=input)` |
+
+h/t to [Mikko](https://www.linkedin.com/in/mikkolehtimaki/) for this integration
+
+
+## **Usage - LiteLLM Proxy Server**
+
+Here's how to call Azure OpenAI models with the LiteLLM Proxy Server
+
+### 1. Save key in your environment
+
+```bash
+export AZURE_API_KEY=""
+```
+
+### 2. Start the proxy 
+
+```yaml
+model_list:
+  - model_name: text-embedding-ada-002
+    litellm_params:
+      model: azure/my-deployment-name
+      api_base: https://openai-gpt-4-test-v-1.openai.azure.com/
+      api_version: "2023-05-15"
+      api_key: os.environ/AZURE_API_KEY # The `os.environ/` prefix tells litellm to read this from the env.
+```
+
+### 3. Test it
+
+<Tabs>
+<TabItem value="Curl" label="Curl Request">
+
+```shell
+curl --location 'http://0.0.0.0:4000/embeddings' \
+  --header 'Content-Type: application/json' \
+  --data ' {
+  "model": "text-embedding-ada-002",
+  "input": ["write a litellm poem"]
+  }'
+```
+</TabItem>
+<TabItem value="openai" label="OpenAI v1.0.0+">
+
+```python
+import openai
+from openai import OpenAI
+
+# set base_url to your proxy server
+# set api_key to send to proxy server
+client = OpenAI(api_key="<proxy-api-key>", base_url="http://0.0.0.0:4000")
+
+response = client.embeddings.create(
+    input=["hello from litellm"],
+    model="text-embedding-ada-002"
+)
+
+print(response)
+
+```
+</TabItem>
+</Tabs>
+
+

docs/my-website/docs/providers/azure/azure_responses.md

@@ -0,0 +1,295 @@
+import Image from '@theme/IdealImage';
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+
+# Azure Responses API
+
+| Property | Details |
+|-------|-------|
+| Description | Azure OpenAI Responses API |
+| `custom_llm_provider` on LiteLLM | `azure/` |
+| Supported Operations | `/v1/responses`|
+| Azure OpenAI Responses API | [Azure OpenAI Responses API ↗](https://learn.microsoft.com/en-us/azure/ai-services/openai/how-to/responses?tabs=python-secure) |
+| Cost Tracking, Logging Support | ✅ LiteLLM will log, track cost for Responses API Requests |
+| Supported OpenAI Params | ✅ All OpenAI params are supported, [See here](https://github.com/BerriAI/litellm/blob/0717369ae6969882d149933da48eeb8ab0e691bd/litellm/llms/openai/responses/transformation.py#L23) |
+
+## Usage
+
+## Create a model response
+
+<Tabs>
+<TabItem value="litellm-sdk" label="LiteLLM SDK">
+
+#### Non-streaming
+
+```python showLineNumbers title="Azure Responses API"
+import litellm
+
+# Non-streaming response
+response = litellm.responses(
+    model="azure/o1-pro",
+    input="Tell me a three sentence bedtime story about a unicorn.",
+    max_output_tokens=100,
+    api_key=os.getenv("AZURE_RESPONSES_OPENAI_API_KEY"),
+    api_base="https://litellm8397336933.openai.azure.com/",
+    api_version="2023-03-15-preview",
+)
+
+print(response)
+```
+
+#### Streaming
+```python showLineNumbers title="Azure Responses API"
+import litellm
+
+# Streaming response
+response = litellm.responses(
+    model="azure/o1-pro",
+    input="Tell me a three sentence bedtime story about a unicorn.",
+    stream=True,
+    api_key=os.getenv("AZURE_RESPONSES_OPENAI_API_KEY"),
+    api_base="https://litellm8397336933.openai.azure.com/",
+    api_version="2023-03-15-preview",
+)
+
+for event in response:
+    print(event)
+```
+
+</TabItem>
+<TabItem value="proxy" label="OpenAI SDK with LiteLLM Proxy">
+
+First, add this to your litellm proxy config.yaml:
+```yaml showLineNumbers title="Azure Responses API"
+model_list:
+  - model_name: o1-pro
+    litellm_params:
+      model: azure/o1-pro
+      api_key: os.environ/AZURE_RESPONSES_OPENAI_API_KEY
+      api_base: https://litellm8397336933.openai.azure.com/
+      api_version: 2023-03-15-preview
+```
+
+Start your LiteLLM proxy:
+```bash
+litellm --config /path/to/config.yaml
+
+# RUNNING on http://0.0.0.0:4000
+```
+
+Then use the OpenAI SDK pointed to your proxy:
+
+#### Non-streaming
+```python showLineNumbers
+from openai import OpenAI
+
+# Initialize client with your proxy URL
+client = OpenAI(
+    base_url="http://localhost:4000",  # Your proxy URL
+    api_key="your-api-key"             # Your proxy API key
+)
+
+# Non-streaming response
+response = client.responses.create(
+    model="o1-pro",
+    input="Tell me a three sentence bedtime story about a unicorn."
+)
+
+print(response)
+```
+
+#### Streaming
+```python showLineNumbers
+from openai import OpenAI
+
+# Initialize client with your proxy URL
+client = OpenAI(
+    base_url="http://localhost:4000",  # Your proxy URL
+    api_key="your-api-key"             # Your proxy API key
+)
+
+# Streaming response
+response = client.responses.create(
+    model="o1-pro",
+    input="Tell me a three sentence bedtime story about a unicorn.",
+    stream=True
+)
+
+for event in response:
+    print(event)
+```
+
+</TabItem>
+</Tabs>
+
+## Azure Codex Models
+
+Codex models use Azure's new [/v1/preview API](https://learn.microsoft.com/en-us/azure/ai-services/openai/api-version-lifecycle?tabs=key#next-generation-api) which provides ongoing access to the latest features with no need to update `api-version` each month. 
+
+**LiteLLM will send your requests to the `/v1/preview` endpoint when you set `api_version="preview"`.**
+
+<Tabs>
+<TabItem value="litellm-sdk" label="LiteLLM SDK">
+
+#### Non-streaming
+
+```python showLineNumbers title="Azure Codex Models"
+import litellm
+
+# Non-streaming response with Codex models
+response = litellm.responses(
+    model="azure/codex-mini",
+    input="Tell me a three sentence bedtime story about a unicorn.",
+    max_output_tokens=100,
+    api_key=os.getenv("AZURE_RESPONSES_OPENAI_API_KEY"),
+    api_base="https://litellm8397336933.openai.azure.com",
+    api_version="preview", # 👈 key difference
+)
+
+print(response)
+```
+
+#### Streaming
+```python showLineNumbers title="Azure Codex Models"
+import litellm
+
+# Streaming response with Codex models
+response = litellm.responses(
+    model="azure/codex-mini",
+    input="Tell me a three sentence bedtime story about a unicorn.",
+    stream=True,
+    api_key=os.getenv("AZURE_RESPONSES_OPENAI_API_KEY"),
+    api_base="https://litellm8397336933.openai.azure.com",
+    api_version="preview", # 👈 key difference
+)
+
+for event in response:
+    print(event)
+```
+
+</TabItem>
+<TabItem value="proxy" label="OpenAI SDK with LiteLLM Proxy">
+
+First, add this to your litellm proxy config.yaml:
+```yaml showLineNumbers title="Azure Codex Models"
+model_list:
+  - model_name: codex-mini
+    litellm_params:
+      model: azure/codex-mini
+      api_key: os.environ/AZURE_RESPONSES_OPENAI_API_KEY
+      api_base: https://litellm8397336933.openai.azure.com
+      api_version: preview # 👈 key difference
+```
+
+Start your LiteLLM proxy:
+```bash
+litellm --config /path/to/config.yaml
+
+# RUNNING on http://0.0.0.0:4000
+```
+
+Then use the OpenAI SDK pointed to your proxy:
+
+#### Non-streaming
+```python showLineNumbers
+from openai import OpenAI
+
+# Initialize client with your proxy URL
+client = OpenAI(
+    base_url="http://localhost:4000",  # Your proxy URL
+    api_key="your-api-key"             # Your proxy API key
+)
+
+# Non-streaming response
+response = client.responses.create(
+    model="codex-mini",
+    input="Tell me a three sentence bedtime story about a unicorn."
+)
+
+print(response)
+```
+
+#### Streaming
+```python showLineNumbers
+from openai import OpenAI
+
+# Initialize client with your proxy URL
+client = OpenAI(
+    base_url="http://localhost:4000",  # Your proxy URL
+    api_key="your-api-key"             # Your proxy API key
+)
+
+# Streaming response
+response = client.responses.create(
+    model="codex-mini",
+    input="Tell me a three sentence bedtime story about a unicorn.",
+    stream=True
+)
+
+for event in response:
+    print(event)
+```
+
+</TabItem>
+</Tabs>
+
+
+## Calling via `/chat/completions`
+
+You can also call the Azure Responses API via the `/chat/completions` endpoint.
+
+
+<Tabs>
+<TabItem value="litellm-sdk" label="LiteLLM SDK">
+
+```python showLineNumbers
+from litellm import completion
+import os 
+
+os.environ["AZURE_API_BASE"] = "https://my-endpoint-sweden-berri992.openai.azure.com/"
+os.environ["AZURE_API_VERSION"] = "2023-03-15-preview"
+os.environ["AZURE_API_KEY"] = "my-api-key"
+
+response = completion(
+    model="azure/responses/my-custom-o1-pro",
+    messages=[{"role": "user", "content": "Hello world"}],
+)
+
+print(response)
+```
+</TabItem>
+<TabItem value="proxy" label="OpenAI SDK with LiteLLM Proxy">
+
+1. Setup config.yaml
+
+```yaml showLineNumbers
+model_list:
+  - model_name: my-custom-o1-pro
+    litellm_params:
+      model: azure/responses/my-custom-o1-pro
+      api_key: os.environ/AZURE_API_KEY
+      api_base: https://my-endpoint-sweden-berri992.openai.azure.com/
+      api_version: 2023-03-15-preview
+```
+
+2. Start LiteLLM proxy
+```bash
+litellm --config /path/to/config.yaml
+
+# RUNNING on http://0.0.0.0:4000
+```
+
+3. Test it! 
+
+```bash
+curl http://localhost:4000/v1/chat/completions \
+  -X POST \
+  -H "Content-Type: application/json" \
+  -H "Authorization: Bearer $LITELLM_API_KEY" \
+  -d '{
+    "model": "my-custom-o1-pro",
+    "messages": [{"role": "user", "content": "Hello world"}]
+  }'
+```
+</TabItem>
+</Tabs>

docs/my-website/docs/providers/bedrock.md

@@ -60,9 +60,9 @@ Here's how to call Bedrock with the LiteLLM Proxy Server
 
 ```yaml
 model_list:
-  - model_name: bedrock-claude-v1
+  - model_name: bedrock-claude-3-5-sonnet
     litellm_params:
-      model: bedrock/anthropic.claude-instant-v1
+      model: bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0
       aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID
       aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY
       aws_region_name: os.environ/AWS_REGION_NAME

docs/my-website/docs/providers/bedrock_agents.md

@@ -0,0 +1,202 @@
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+
+# Bedrock Agents
+
+Call Bedrock Agents in the OpenAI Request/Response format.
+
+
+| Property | Details |
+|----------|---------|
+| Description | Amazon Bedrock Agents use the reasoning of foundation models (FMs), APIs, and data to break down user requests, gather relevant information, and efficiently complete tasks. |
+| Provider Route on LiteLLM | `bedrock/agent/{AGENT_ID}/{ALIAS_ID}` |
+| Provider Doc | [AWS Bedrock Agents ↗](https://aws.amazon.com/bedrock/agents/) |
+
+## Quick Start
+
+### Model Format to LiteLLM
+
+To call a bedrock agent through LiteLLM, you need to use the following model format to call the agent.
+
+Here the `model=bedrock/agent/` tells LiteLLM to call the bedrock `InvokeAgent` API.
+
+```shell showLineNumbers title="Model Format to LiteLLM"
+bedrock/agent/{AGENT_ID}/{ALIAS_ID}
+```
+
+**Example:**
+- `bedrock/agent/L1RT58GYRW/MFPSBCXYTW`
+- `bedrock/agent/ABCD1234/LIVE`
+
+You can find these IDs in your AWS Bedrock console under Agents.
+
+
+### LiteLLM Python SDK
+
+```python showLineNumbers title="Basic Agent Completion"
+import litellm
+
+# Make a completion request to your Bedrock Agent
+response = litellm.completion(
+    model="bedrock/agent/L1RT58GYRW/MFPSBCXYTW",  # agent/{AGENT_ID}/{ALIAS_ID}
+    messages=[
+        {
+            "role": "user", 
+            "content": "Hi, I need help with analyzing our Q3 sales data and generating a summary report"
+        }
+    ],
+)
+
+print(response.choices[0].message.content)
+print(f"Response cost: ${response._hidden_params['response_cost']}")
+```
+
+```python showLineNumbers title="Streaming Agent Responses"
+import litellm
+
+# Stream responses from your Bedrock Agent
+response = litellm.completion(
+    model="bedrock/agent/L1RT58GYRW/MFPSBCXYTW",
+    messages=[
+        {
+            "role": "user",
+            "content": "Can you help me plan a marketing campaign and provide step-by-step execution details?"
+        }
+    ],
+    stream=True,
+)
+
+for chunk in response:
+    if chunk.choices[0].delta.content:
+        print(chunk.choices[0].delta.content, end="")
+```
+
+
+### LiteLLM Proxy
+
+#### 1. Configure your model in config.yaml
+
+<Tabs>
+<TabItem value="config-yaml" label="config.yaml">
+
+```yaml showLineNumbers title="LiteLLM Proxy Configuration"
+model_list:
+  - model_name: bedrock-agent-1
+    litellm_params:
+      model: bedrock/agent/L1RT58GYRW/MFPSBCXYTW
+      aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID
+      aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY
+      aws_region_name: us-west-2
+
+  - model_name: bedrock-agent-2  
+    litellm_params:
+      model: bedrock/agent/AGENT456/ALIAS789
+      aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID
+      aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY
+      aws_region_name: us-east-1
+```
+
+</TabItem>
+</Tabs>
+
+#### 2. Start the LiteLLM Proxy
+
+```bash showLineNumbers title="Start LiteLLM Proxy"
+litellm --config config.yaml
+```
+
+#### 3. Make requests to your Bedrock Agents
+
+<Tabs>
+<TabItem value="curl" label="Curl">
+
+```bash showLineNumbers title="Basic Agent Request"
+curl http://localhost:4000/v1/chat/completions \
+  -H "Content-Type: application/json" \
+  -H "Authorization: Bearer $LITELLM_API_KEY" \
+  -d '{
+    "model": "bedrock-agent-1",
+    "messages": [
+      {
+        "role": "user", 
+        "content": "Analyze our customer data and suggest retention strategies"
+      }
+    ]
+  }'
+```
+
+```bash showLineNumbers title="Streaming Agent Request"
+curl http://localhost:4000/v1/chat/completions \
+  -H "Content-Type: application/json" \
+  -H "Authorization: Bearer $LITELLM_API_KEY" \
+  -d '{
+    "model": "bedrock-agent-2",
+    "messages": [
+      {
+        "role": "user",
+        "content": "Create a comprehensive social media strategy for our new product"
+      }
+    ],
+    "stream": true
+  }'
+```
+
+</TabItem>
+
+<TabItem value="openai-sdk" label="OpenAI Python SDK">
+
+```python showLineNumbers title="Using OpenAI SDK with LiteLLM Proxy"
+from openai import OpenAI
+
+# Initialize client with your LiteLLM proxy URL
+client = OpenAI(
+    base_url="http://localhost:4000",
+    api_key="your-litellm-api-key"
+)
+
+# Make a completion request to your agent
+response = client.chat.completions.create(
+    model="bedrock-agent-1",
+    messages=[
+      {
+        "role": "user",
+        "content": "Help me prepare for the quarterly business review meeting"
+      }
+    ]
+)
+
+print(response.choices[0].message.content)
+```
+
+```python showLineNumbers title="Streaming with OpenAI SDK"
+from openai import OpenAI
+
+client = OpenAI(
+    base_url="http://localhost:4000", 
+    api_key="your-litellm-api-key"
+)
+
+# Stream agent responses
+stream = client.chat.completions.create(
+    model="bedrock-agent-2",
+    messages=[
+      {
+        "role": "user",
+        "content": "Walk me through launching a new feature beta program"
+      }
+    ],
+    stream=True
+)
+
+for chunk in stream:
+    if chunk.choices[0].delta.content is not None:
+        print(chunk.choices[0].delta.content, end="")
+```
+
+</TabItem>
+</Tabs>
+
+## Further Reading
+
+- [AWS Bedrock Agents Documentation](https://aws.amazon.com/bedrock/agents/)
+- [LiteLLM Authentication to Bedrock](https://docs.litellm.ai/docs/providers/bedrock#boto3---authentication)

docs/my-website/docs/providers/bedrock_vector_store.md

@@ -0,0 +1,144 @@
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+import Image from '@theme/IdealImage';
+
+# Bedrock Knowledge Bases
+
+AWS Bedrock Knowledge Bases allows you to connect your LLM's to your organization's data, letting your models retrieve and reference information specific to your business.
+
+| Property | Details |
+|----------|---------|
+| Description | Bedrock Knowledge Bases connects your data to LLM's, enabling them to retrieve and reference your organization's information in their responses. |
+| Provider Route on LiteLLM | `bedrock` in the litellm vector_store_registry |
+| Provider Doc | [AWS Bedrock Knowledge Bases ↗](https://aws.amazon.com/bedrock/knowledge-bases/) |
+
+## Quick Start
+
+### LiteLLM Python SDK
+
+```python showLineNumbers title="Example using LiteLLM Python SDK"
+import os
+import litellm
+
+from litellm.vector_stores.vector_store_registry import VectorStoreRegistry, LiteLLM_ManagedVectorStore
+
+# Init vector store registry with your Bedrock Knowledge Base
+litellm.vector_store_registry = VectorStoreRegistry(
+    vector_stores=[
+        LiteLLM_ManagedVectorStore(
+            vector_store_id="YOUR_KNOWLEDGE_BASE_ID",  # KB ID from AWS Bedrock
+            custom_llm_provider="bedrock"
+        )
+    ]
+)
+
+# Make a completion request using your Knowledge Base
+response = await litellm.acompletion(
+    model="anthropic/claude-3-5-sonnet", 
+    messages=[{"role": "user", "content": "What does our company policy say about remote work?"}],
+    tools=[
+        {
+            "type": "file_search",
+            "vector_store_ids": ["YOUR_KNOWLEDGE_BASE_ID"]
+        }
+    ],
+)
+
+print(response.choices[0].message.content)
+```
+
+### LiteLLM Proxy
+
+#### 1. Configure your vector_store_registry
+
+<Tabs>
+<TabItem value="config-yaml" label="config.yaml">
+
+```yaml
+model_list:
+  - model_name: claude-3-5-sonnet
+    litellm_params:
+      model: anthropic/claude-3-5-sonnet
+      api_key: os.environ/ANTHROPIC_API_KEY
+
+vector_store_registry:
+  - vector_store_name: "bedrock-company-docs"
+    litellm_params:
+      vector_store_id: "YOUR_KNOWLEDGE_BASE_ID"
+      custom_llm_provider: "bedrock"
+      vector_store_description: "Bedrock Knowledge Base for company documents"
+      vector_store_metadata:
+        source: "Company internal documentation"
+```
+
+</TabItem>
+
+<TabItem value="litellm-ui" label="LiteLLM UI">
+
+On the LiteLLM UI, Navigate to Experimental > Vector Stores > Create Vector Store. On this page you can create a vector store with a name, vector store id and credentials.
+
+<Image 
+  img={require('../../img/kb_2.png')}
+  style={{width: '50%'}}
+/>
+
+</TabItem>
+</Tabs>
+
+#### 2. Make a request with vector_store_ids parameter
+
+<Tabs>
+<TabItem value="curl" label="Curl">
+
+```bash
+curl http://localhost:4000/v1/chat/completions \
+  -H "Content-Type: application/json" \
+  -H "Authorization: Bearer $LITELLM_API_KEY" \
+  -d '{
+    "model": "claude-3-5-sonnet",
+    "messages": [{"role": "user", "content": "What does our company policy say about remote work?"}],
+    "tools": [
+        {
+            "type": "file_search",
+            "vector_store_ids": ["YOUR_KNOWLEDGE_BASE_ID"]
+        }
+    ]
+  }'
+```
+
+</TabItem>
+
+<TabItem value="openai-sdk" label="OpenAI Python SDK">
+
+```python
+from openai import OpenAI
+
+# Initialize client with your LiteLLM proxy URL
+client = OpenAI(
+    base_url="http://localhost:4000",
+    api_key="your-litellm-api-key"
+)
+
+# Make a completion request with vector_store_ids parameter
+response = client.chat.completions.create(
+    model="claude-3-5-sonnet",
+    messages=[{"role": "user", "content": "What does our company policy say about remote work?"}],
+    tools=[
+        {
+            "type": "file_search",
+            "vector_store_ids": ["YOUR_KNOWLEDGE_BASE_ID"]
+        }
+    ]
+)
+
+print(response.choices[0].message.content)
+```
+
+</TabItem>
+</Tabs>
+
+
+Futher Reading Vector Stores:
+- [Always on Vector Stores](https://docs.litellm.ai/docs/completion/knowledgebase#always-on-for-a-model)
+- [Listing available vector stores on litellm proxy](https://docs.litellm.ai/docs/completion/knowledgebase#listing-available-vector-stores)
+- [How LiteLLM Vector Stores Work](https://docs.litellm.ai/docs/completion/knowledgebase#how-it-works)

docs/my-website/docs/providers/cohere.md

@@ -13,7 +13,9 @@ os.environ["COHERE_API_KEY"] = ""
 
 ## Usage
 
-```python
+### LiteLLM Python SDK
+
+```python showLineNumbers
 from litellm import completion
 
 ## set ENV variables
@@ -26,9 +28,9 @@ response = completion(
 )
 ```
 
-## Usage - Streaming
+#### Streaming
 
-```python
+```python showLineNumbers
 from litellm import completion
 
 ## set ENV variables
@@ -46,15 +48,90 @@ for chunk in response:
 ```
 
 
+
+## Usage with LiteLLM Proxy 
+
+Here's how to call Cohere with the LiteLLM Proxy Server
+
+### 1. Save key in your environment
+
+```bash
+export COHERE_API_KEY="your-api-key"
+```
+
+### 2. Start the proxy 
+
+Define the cohere models you want to use in the config.yaml
+
+```yaml showLineNumbers
+model_list:
+  - model_name: command-a-03-2025 
+    litellm_params:
+      model: command-a-03-2025
+      api_key: "os.environ/COHERE_API_KEY"
+```
+
+```bash
+litellm --config /path/to/config.yaml
+```
+
+
+### 3. Test it
+
+
+<Tabs>
+<TabItem value="Curl" label="Curl Request">
+
+```shell showLineNumbers
+curl --location 'http://0.0.0.0:4000/chat/completions' \
+--header 'Content-Type: application/json' \
+--header 'Authorization: Bearer <your-litellm-api-key>' \
+--data ' {
+      "model": "command-a-03-2025",
+      "messages": [
+        {
+          "role": "user",
+          "content": "what llm are you"
+        }
+      ]
+    }
+'
+```
+</TabItem>
+<TabItem value="openai" label="OpenAI v1.0.0+">
+
+```python showLineNumbers
+import openai
+client = openai.OpenAI(
+    api_key="anything",
+    base_url="http://0.0.0.0:4000"
+)
+
+# request sent to model set on litellm proxy
+response = client.chat.completions.create(model="command-a-03-2025", messages = [
+    {
+        "role": "user",
+        "content": "this is a test request, write a short poem"
+    }
+])
+
+print(response)
+
+```
+</TabItem>
+</Tabs>
+
+
 ## Supported Models
 | Model Name | Function Call |
 |------------|----------------|
-| command-r-plus-08-2024 | `completion('command-r-plus-08-2024', messages)` |  
-| command-r-08-2024 | `completion('command-r-08-2024', messages)` |
-| command-r-plus | `completion('command-r-plus', messages)` |  
-| command-r | `completion('command-r', messages)` |
-| command-light | `completion('command-light', messages)` |  
-| command-nightly | `completion('command-nightly', messages)` |
+| command-a-03-2025 | `litellm.completion('command-a-03-2025', messages)` |
+| command-r-plus-08-2024 | `litellm.completion('command-r-plus-08-2024', messages)` |  
+| command-r-08-2024 | `litellm.completion('command-r-08-2024', messages)` |
+| command-r-plus | `litellm.completion('command-r-plus', messages)` |  
+| command-r | `litellm.completion('command-r', messages)` |
+| command-light | `litellm.completion('command-light', messages)` |  
+| command-nightly | `litellm.completion('command-nightly', messages)` |
 
 
 ## Embedding

docs/my-website/docs/providers/custom_llm_server.md

@@ -1,3 +1,7 @@
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+import Image from '@theme/IdealImage';
+
 # Custom API Server (Custom Format)
 
 Call your custom torch-serve / internal LLM APIs via LiteLLM
@@ -8,9 +12,17 @@ Call your custom torch-serve / internal LLM APIs via LiteLLM
 - For modifying incoming/outgoing calls on proxy, [go here](../proxy/call_hooks.md)
 :::
 
+Supported Routes:
+- `/v1/chat/completions` -> `litellm.acompletion`
+- `/v1/completions` -> `litellm.atext_completion`
+- `/v1/embeddings` -> `litellm.aembedding`
+- `/v1/images/generations` -> `litellm.aimage_generation`
+
+- `/v1/messages` -> `litellm.acompletion`
+
 ## Quick Start 
 
-```python
+```python showLineNumbers
 import litellm
 from litellm import CustomLLM, completion, get_llm_provider
 
@@ -251,6 +263,102 @@ Expected Response
 }
 ```
 
+## Anthropic `/v1/messages`
+
+- Write the integration for .acompletion
+- litellm will transform it to /v1/messages
+
+1. Setup your `custom_handler.py` file 
+
+```python
+import litellm
+from litellm import CustomLLM, completion, get_llm_provider
+
+
+class MyCustomLLM(CustomLLM):
+    async def acompletion(self, *args, **kwargs) -> litellm.ModelResponse:
+        return litellm.completion(
+            model="gpt-3.5-turbo",
+            messages=[{"role": "user", "content": "Hello world"}],
+            mock_response="Hi!",
+        )  # type: ignore
+
+
+my_custom_llm = MyCustomLLM()
+```
+
+2. Add to `config.yaml` 
+
+In the config below, we pass
+
+python_filename: `custom_handler.py`
+custom_handler_instance_name: `my_custom_llm`. This is defined in Step 1
+
+custom_handler: `custom_handler.my_custom_llm`
+
+```yaml
+model_list:
+  - model_name: "test-model"             
+    litellm_params:
+      model: "openai/text-embedding-ada-002"
+  - model_name: "my-custom-model"
+    litellm_params:
+      model: "my-custom-llm/my-model"
+
+litellm_settings:
+  custom_provider_map:
+  - {"provider": "my-custom-llm", "custom_handler": custom_handler.my_custom_llm}
+```
+
+```bash
+litellm --config /path/to/config.yaml
+```
+
+3. Test it! 
+
+```bash
+curl -L -X POST 'http://0.0.0.0:4000/v1/messages' \
+-H 'anthropic-version: 2023-06-01' \
+-H 'content-type: application/json' \
+-H 'Authorization: Bearer sk-1234' \
+-d '{
+   "model": "my-custom-model",
+     "max_tokens": 1024,
+     "messages": [{
+         "role": "user",
+         "content": [
+         {
+             "type": "text",
+             "text": "What are the key findings in this document 12?"
+         }]
+     }]
+}'
+```
+
+Expected Response
+
+```json
+{
+    "id": "chatcmpl-Bm4qEp4h4vCe7Zi4Gud1MAxTWgibO",
+    "type": "message",
+    "role": "assistant",
+    "model": "gpt-3.5-turbo-0125",
+    "stop_sequence": null,
+    "usage": {
+        "input_tokens": 18,
+        "output_tokens": 44
+    },
+    "content": [
+        {
+            "type": "text",
+            "text": "Without the specific document being provided, it is not possible to determine the key findings within it. If you can provide the content or a summary of document 12, I would be happy to help identify the key findings."
+        }
+    ],
+    "stop_reason": "end_turn"
+}
+```
+
+
 ## Additional Parameters
 
 Additional parameters are passed inside `optional_params` key in the `completion` or `image_generation` function.

docs/my-website/docs/providers/elevenlabs.md

@@ -0,0 +1,231 @@
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+
+# ElevenLabs
+
+ElevenLabs provides high-quality AI voice technology, including speech-to-text capabilities through their transcription API.
+
+| Property | Details |
+|----------|---------|
+| Description | ElevenLabs offers advanced AI voice technology with speech-to-text transcription capabilities that support multiple languages and speaker diarization. |
+| Provider Route on LiteLLM | `elevenlabs/` |
+| Provider Doc | [ElevenLabs API ↗](https://elevenlabs.io/docs/api-reference) |
+| Supported Endpoints | `/audio/transcriptions` |
+
+## Quick Start
+
+### LiteLLM Python SDK
+
+<Tabs>
+<TabItem value="basic" label="Basic Usage">
+
+```python showLineNumbers title="Basic audio transcription with ElevenLabs"
+import litellm
+
+# Transcribe audio file
+with open("audio.mp3", "rb") as audio_file:
+    response = litellm.transcription(
+        model="elevenlabs/scribe_v1",
+        file=audio_file,
+        api_key="your-elevenlabs-api-key"  # or set ELEVENLABS_API_KEY env var
+    )
+
+print(response.text)
+```
+
+</TabItem>
+
+<TabItem value="advanced" label="Advanced Features">
+
+```python showLineNumbers title="Audio transcription with advanced features"
+import litellm
+
+# Transcribe with speaker diarization and language specification
+with open("audio.wav", "rb") as audio_file:
+    response = litellm.transcription(
+        model="elevenlabs/scribe_v1",
+        file=audio_file,
+        language="en",           # Language hint (maps to language_code)
+        temperature=0.3,         # Control randomness in transcription
+        diarize=True,           # Enable speaker diarization
+        api_key="your-elevenlabs-api-key"
+    )
+
+print(f"Transcription: {response.text}")
+print(f"Language: {response.language}")
+
+# Access word-level timestamps if available
+if hasattr(response, 'words') and response.words:
+    for word_info in response.words:
+        print(f"Word: {word_info['word']}, Start: {word_info['start']}, End: {word_info['end']}")
+```
+
+</TabItem>
+
+<TabItem value="async" label="Async Usage">
+
+```python showLineNumbers title="Async audio transcription"
+import litellm
+import asyncio
+
+async def transcribe_audio():
+    with open("audio.mp3", "rb") as audio_file:
+        response = await litellm.atranscription(
+            model="elevenlabs/scribe_v1",
+            file=audio_file,
+            api_key="your-elevenlabs-api-key"
+        )
+    
+    return response.text
+
+# Run async transcription
+result = asyncio.run(transcribe_audio())
+print(result)
+```
+
+</TabItem>
+</Tabs>
+
+### LiteLLM Proxy
+
+#### 1. Configure your proxy
+
+<Tabs>
+<TabItem value="config-yaml" label="config.yaml">
+
+```yaml showLineNumbers title="ElevenLabs configuration in config.yaml"
+model_list:
+  - model_name: elevenlabs-transcription
+    litellm_params:
+      model: elevenlabs/scribe_v1
+      api_key: os.environ/ELEVENLABS_API_KEY
+
+general_settings:
+  master_key: your-master-key
+```
+
+</TabItem>
+
+<TabItem value="env-vars" label="Environment Variables">
+
+```bash showLineNumbers title="Required environment variables"
+export ELEVENLABS_API_KEY="your-elevenlabs-api-key"
+export LITELLM_MASTER_KEY="your-master-key"
+```
+
+</TabItem>
+</Tabs>
+
+#### 2. Start the proxy
+
+```bash showLineNumbers title="Start LiteLLM proxy server"
+litellm --config config.yaml
+
+# Proxy will be available at http://localhost:4000
+```
+
+#### 3. Make transcription requests
+
+<Tabs>
+<TabItem value="curl" label="Curl">
+
+```bash showLineNumbers title="Audio transcription with curl"
+curl http://localhost:4000/v1/audio/transcriptions \
+  -H "Authorization: Bearer $LITELLM_API_KEY" \
+  -H "Content-Type: multipart/form-data" \
+  -F file="@audio.mp3" \
+  -F model="elevenlabs-transcription" \
+  -F language="en" \
+  -F temperature="0.3"
+```
+
+</TabItem>
+
+<TabItem value="openai-sdk" label="OpenAI Python SDK">
+
+```python showLineNumbers title="Using OpenAI SDK with LiteLLM proxy"
+from openai import OpenAI
+
+# Initialize client with your LiteLLM proxy URL
+client = OpenAI(
+    base_url="http://localhost:4000",
+    api_key="your-litellm-api-key"
+)
+
+# Transcribe audio file
+with open("audio.mp3", "rb") as audio_file:
+    response = client.audio.transcriptions.create(
+        model="elevenlabs-transcription",
+        file=audio_file,
+        language="en",
+        temperature=0.3,
+        # ElevenLabs-specific parameters
+        diarize=True,
+        speaker_boost=True,
+        custom_vocabulary="technical,AI,machine learning"
+    )
+
+print(response.text)
+```
+
+</TabItem>
+
+<TabItem value="javascript" label="JavaScript/Node.js">
+
+```javascript showLineNumbers title="Audio transcription with JavaScript"
+import OpenAI from 'openai';
+import fs from 'fs';
+
+const openai = new OpenAI({
+  baseURL: 'http://localhost:4000',
+  apiKey: 'your-litellm-api-key'
+});
+
+async function transcribeAudio() {
+  const response = await openai.audio.transcriptions.create({
+    file: fs.createReadStream('audio.mp3'),
+    model: 'elevenlabs-transcription',
+    language: 'en',
+    temperature: 0.3,
+    diarize: true,
+    speaker_boost: true
+  });
+
+  console.log(response.text);
+}
+
+transcribeAudio();
+```
+
+</TabItem>
+</Tabs>
+
+## Response Format
+
+ElevenLabs returns transcription responses in OpenAI-compatible format:
+
+```json showLineNumbers title="Example transcription response"
+{
+  "text": "Hello, this is a sample transcription with multiple speakers.",
+  "task": "transcribe",
+  "language": "en",
+  "words": [
+    {
+      "word": "Hello",
+      "start": 0.0,
+      "end": 0.5
+    },
+    {
+      "word": "this",
+      "start": 0.5,
+      "end": 0.8
+    }
+  ]
+}
+```
+
+### Common Issues
+
+1. **Invalid API Key**: Ensure `ELEVENLABS_API_KEY` is set correctly
+
+

docs/my-website/docs/providers/featherless_ai.md

@@ -0,0 +1,56 @@
+# Featherless AI
+https://featherless.ai/
+
+:::tip
+
+**We support ALL Featherless AI models, just set `model=featherless_ai/<any-model-on-featherless>` as a prefix when sending litellm requests. For the complete supported model list, visit https://featherless.ai/models **
+
+:::
+
+
+## API Key
+```python
+# env variable
+os.environ['FEATHERLESS_AI_API_KEY']
+```
+
+## Sample Usage
+```python
+from litellm import completion
+import os
+
+os.environ['FEATHERLESS_AI_API_KEY'] = ""
+response = completion(
+    model="featherless_ai/featherless-ai/Qwerky-72B", 
+    messages=[{"role": "user", "content": "write code for saying hi from LiteLLM"}]
+)
+```
+
+## Sample Usage - Streaming
+```python
+from litellm import completion
+import os
+
+os.environ['FEATHERLESS_AI_API_KEY'] = ""
+response = completion(
+    model="featherless_ai/featherless-ai/Qwerky-72B", 
+    messages=[{"role": "user", "content": "write code for saying hi from LiteLLM"}],
+    stream=True
+)
+
+for chunk in response:
+    print(chunk)
+```
+
+## Chat Models
+| Model Name                                 | Function Call                                                                                  |
+|---------------------------------------------|-----------------------------------------------------------------------------------------------|
+| featherless-ai/Qwerky-72B                   | `completion(model="featherless_ai/featherless-ai/Qwerky-72B", messages)`                      |
+| featherless-ai/Qwerky-QwQ-32B               | `completion(model="featherless_ai/featherless-ai/Qwerky-QwQ-32B", messages)`                  |
+| Qwen/Qwen2.5-72B-Instruct                   | `completion(model="featherless_ai/Qwen/Qwen2.5-72B-Instruct", messages)`                      |
+| all-hands/openhands-lm-32b-v0.1             | `completion(model="featherless_ai/all-hands/openhands-lm-32b-v0.1", messages)`                |
+| Qwen/Qwen2.5-Coder-32B-Instruct             | `completion(model="featherless_ai/Qwen/Qwen2.5-Coder-32B-Instruct", messages)`                |
+| deepseek-ai/DeepSeek-V3-0324                | `completion(model="featherless_ai/deepseek-ai/DeepSeek-V3-0324", messages)`                   |
+| mistralai/Mistral-Small-24B-Instruct-2501   | `completion(model="featherless_ai/mistralai/Mistral-Small-24B-Instruct-2501", messages)`      |
+| mistralai/Mistral-Nemo-Instruct-2407        | `completion(model="featherless_ai/mistralai/Mistral-Nemo-Instruct-2407", messages)`           |
+| ProdeusUnity/Stellar-Odyssey-12b-v0.0       | `completion(model="featherless_ai/ProdeusUnity/Stellar-Odyssey-12b-v0.0", messages)`          |

docs/my-website/docs/providers/gemini.md

@@ -8,7 +8,7 @@ import TabItem from '@theme/TabItem';
 |-------|-------|
 | Description | Google AI Studio is a fully-managed AI development platform for building and using generative AI. |
 | Provider Route on LiteLLM | `gemini/` |
-| Provider Doc | [Google AI Studio ↗](https://ai.google.dev/aistudio) |
+| Provider Doc | [Google AI Studio ↗](https://aistudio.google.com/) |
 | API Endpoint for Provider | https://generativelanguage.googleapis.com |
 | Supported OpenAI Endpoints | `/chat/completions`, [`/embeddings`](../embedding/supported_embedding#gemini-ai-embedding-models), `/completions` |
 | Pass-through Endpoint | [Supported](../pass_through/google_ai_studio.md) |
@@ -51,6 +51,7 @@ response = completion(
 - frequency_penalty
 - modalities
 - reasoning_content
+- audio (for TTS models only)
 
 **Anthropic Params**
 - thinking (used to set max budget tokens across anthropic/gemini models)
@@ -63,10 +64,13 @@ response = completion(
 
 LiteLLM translates OpenAI's `reasoning_effort` to Gemini's `thinking` parameter. [Code](https://github.com/BerriAI/litellm/blob/620664921902d7a9bfb29897a7b27c1a7ef4ddfb/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py#L362)
 
+Added an additional non-OpenAI standard "disable" value for non-reasoning Gemini requests.
+
 **Mapping**
 
 | reasoning_effort | thinking |
 | ---------------- | -------- |
+| "disable"        | "budget_tokens": 0    |
 | "low"            | "budget_tokens": 1024 |
 | "medium"         | "budget_tokens": 2048 |
 | "high"           | "budget_tokens": 4096 |
@@ -198,6 +202,119 @@ curl http://0.0.0.0:4000/v1/chat/completions \
 
 
 
+## Text-to-Speech (TTS) Audio Output
+
+:::info
+
+LiteLLM supports Gemini TTS models that can generate audio responses using the OpenAI-compatible `audio` parameter format.
+
+:::
+
+### Supported Models
+
+LiteLLM supports Gemini TTS models with audio capabilities (e.g. `gemini-2.5-flash-preview-tts` and `gemini-2.5-pro-preview-tts`). For the complete list of available TTS models and voices, see the [official Gemini TTS documentation](https://ai.google.dev/gemini-api/docs/speech-generation).
+
+### Limitations
+
+:::warning
+
+**Important Limitations**:
+- Gemini TTS models only support the `pcm16` audio format
+- **Streaming support has not been added** to TTS models yet
+- The `modalities` parameter must be set to `['audio']` for TTS requests
+
+:::
+
+### Quick Start
+
+<Tabs>
+<TabItem value="sdk" label="SDK">
+
+```python
+from litellm import completion
+import os
+
+os.environ['GEMINI_API_KEY'] = "your-api-key"
+
+response = completion(
+    model="gemini/gemini-2.5-flash-preview-tts",
+    messages=[{"role": "user", "content": "Say hello in a friendly voice"}],
+    modalities=["audio"],  # Required for TTS models
+    audio={
+        "voice": "Kore",
+        "format": "pcm16"  # Required: must be "pcm16"
+    }
+)
+
+print(response)
+```
+
+</TabItem>
+<TabItem value="proxy" label="PROXY">
+
+1. Setup config.yaml
+
+```yaml
+model_list:
+  - model_name: gemini-tts-flash
+    litellm_params:
+      model: gemini/gemini-2.5-flash-preview-tts
+      api_key: os.environ/GEMINI_API_KEY
+  - model_name: gemini-tts-pro
+    litellm_params:
+      model: gemini/gemini-2.5-pro-preview-tts
+      api_key: os.environ/GEMINI_API_KEY
+```
+
+2. Start proxy
+
+```bash
+litellm --config /path/to/config.yaml
+```
+
+3. Make TTS request
+
+```bash
+curl http://0.0.0.0:4000/v1/chat/completions \
+  -H "Content-Type: application/json" \
+  -H "Authorization: Bearer <YOUR-LITELLM-KEY>" \
+  -d '{
+    "model": "gemini-tts-flash",
+    "messages": [{"role": "user", "content": "Say hello in a friendly voice"}],
+    "modalities": ["audio"],
+    "audio": {
+      "voice": "Kore",
+      "format": "pcm16"
+    }
+  }'
+```
+
+</TabItem>
+</Tabs>
+
+### Advanced Usage
+
+You can combine TTS with other Gemini features:
+
+```python
+response = completion(
+    model="gemini/gemini-2.5-pro-preview-tts",
+    messages=[
+        {"role": "system", "content": "You are a helpful assistant that speaks clearly."},
+        {"role": "user", "content": "Explain quantum computing in simple terms"}
+    ],
+    modalities=["audio"],
+    audio={
+        "voice": "Charon",
+        "format": "pcm16"
+    },
+    temperature=0.7,
+    max_tokens=150
+)
+```
+
+For more information about Gemini's TTS capabilities and available voices, see the [official Gemini TTS documentation](https://ai.google.dev/gemini-api/docs/speech-generation).
+
 ## Passing Gemini Specific Params
 ### Response schema 
 LiteLLM supports sending `response_schema` as a param for Gemini-1.5-Pro on Google AI Studio. 
@@ -643,6 +760,66 @@ curl -X POST 'http://0.0.0.0:4000/chat/completions' \
 </TabItem>
 </Tabs>
 
+### URL Context 
+
+<Tabs>
+<TabItem value="sdk" label="SDK">
+
+```python
+from litellm import completion
+import os
+
+os.environ["GEMINI_API_KEY"] = ".."
+
+# 👇 ADD URL CONTEXT
+tools = [{"urlContext": {}}]
+
+response = completion(
+    model="gemini/gemini-2.0-flash",
+    messages=[{"role": "user", "content": "Summarize this document: https://ai.google.dev/gemini-api/docs/models"}],
+    tools=tools,
+)
+
+print(response)
+
+# Access URL context metadata
+url_context_metadata = response.model_extra['vertex_ai_url_context_metadata']
+urlMetadata = url_context_metadata[0]['urlMetadata'][0]
+print(f"Retrieved URL: {urlMetadata['retrievedUrl']}")
+print(f"Retrieval Status: {urlMetadata['urlRetrievalStatus']}")
+```
+
+</TabItem>
+<TabItem value="proxy" label="PROXY">
+
+1. Setup config.yaml
+```yaml
+model_list:
+  - model_name: gemini-2.0-flash
+    litellm_params:
+      model: gemini/gemini-2.0-flash
+      api_key: os.environ/GEMINI_API_KEY
+```
+
+2. Start Proxy
+```bash
+$ litellm --config /path/to/config.yaml
+```
+
+3. Make Request!
+```bash
+curl -X POST 'http://0.0.0.0:4000/chat/completions' \
+  -H "Content-Type: application/json" \
+  -H "Authorization: Bearer <YOUR-LITELLM-KEY>" \
+  -d '{
+    "model": "gemini-2.0-flash",
+    "messages": [{"role": "user", "content": "Summarize this document: https://ai.google.dev/gemini-api/docs/models"}],
+    "tools": [{"urlContext": {}}]
+  }'
+```
+</TabItem>
+</Tabs>
+
 ### Google Search Retrieval
 
 

docs/my-website/docs/providers/github.md

@@ -7,6 +7,7 @@ https://github.com/marketplace/models
 :::tip
 
 **We support ALL Github models, just set `model=github/<any-model-on-github>` as a prefix when sending litellm requests**
+Ignore company prefix: meta/Llama-3.2-11B-Vision-Instruct becomes model=github/Llama-3.2-11B-Vision-Instruct
 
 :::
 
@@ -23,7 +24,7 @@ import os
 
 os.environ['GITHUB_API_KEY'] = ""
 response = completion(
-    model="github/llama3-8b-8192", 
+    model="github/Llama-3.2-11B-Vision-Instruct", 
     messages=[
        {"role": "user", "content": "hello from litellm"}
    ],
@@ -38,7 +39,7 @@ import os
 
 os.environ['GITHUB_API_KEY'] = ""
 response = completion(
-    model="github/llama3-8b-8192", 
+    model="github/Llama-3.2-11B-Vision-Instruct", 
     messages=[
        {"role": "user", "content": "hello from litellm"}
    ],
@@ -57,9 +58,9 @@ for chunk in response:
 
 ```yaml
 model_list:
-  - model_name: github-llama3-8b-8192 # Model Alias to use for requests
+  - model_name: github-Llama-3.2-11B-Vision-Instruct # Model Alias to use for requests
     litellm_params:
-      model: github/llama3-8b-8192
+      model: github/Llama-3.2-11B-Vision-Instruct
       api_key: "os.environ/GITHUB_API_KEY" # ensure you have `GITHUB_API_KEY` in your .env
 ```
 
@@ -80,7 +81,7 @@ Make request to litellm proxy
 curl --location 'http://0.0.0.0:4000/chat/completions' \
 --header 'Content-Type: application/json' \
 --data ' {
-      "model": "github-llama3-8b-8192",
+      "model": "github-Llama-3.2-11B-Vision-Instruct",
       "messages": [
         {
           "role": "user",
@@ -100,7 +101,7 @@ client = openai.OpenAI(
     base_url="http://0.0.0.0:4000"
 )
 
-response = client.chat.completions.create(model="github-llama3-8b-8192", messages = [
+response = client.chat.completions.create(model="github-Llama-3.2-11B-Vision-Instruct", messages = [
     {
         "role": "user",
         "content": "this is a test request, write a short poem"
@@ -124,7 +125,7 @@ from langchain.schema import HumanMessage, SystemMessage
 
 chat = ChatOpenAI(
     openai_api_base="http://0.0.0.0:4000", # set openai_api_base to the LiteLLM Proxy
-    model = "github-llama3-8b-8192",
+    model = "github-Llama-3.2-11B-Vision-Instruct",
     temperature=0.1
 )
 
@@ -152,7 +153,7 @@ We support ALL Github models, just set `github/` as a prefix when sending comple
 |--------------------|---------------------------------------------------------|
 | llama-3.1-8b-instant     | `completion(model="github/llama-3.1-8b-instant", messages)`     | 
 | llama-3.1-70b-versatile    | `completion(model="github/llama-3.1-70b-versatile", messages)`    | 
-| llama3-8b-8192     | `completion(model="github/llama3-8b-8192", messages)`     | 
+| Llama-3.2-11B-Vision-Instruct     | `completion(model="github/Llama-3.2-11B-Vision-Instruct", messages)`     | 
 | llama3-70b-8192    | `completion(model="github/llama3-70b-8192", messages)`    | 
 | llama2-70b-4096    | `completion(model="github/llama2-70b-4096", messages)`    | 
 | mixtral-8x7b-32768 | `completion(model="github/mixtral-8x7b-32768", messages)` |
@@ -214,7 +215,7 @@ tools = [
     }
 ]
 response = litellm.completion(
-    model="github/llama3-8b-8192",
+    model="github/Llama-3.2-11B-Vision-Instruct",
     messages=messages,
     tools=tools,
     tool_choice="auto",  # auto is default, but we'll be explicit
@@ -254,7 +255,7 @@ if tool_calls:
         )  # extend conversation with function response
     print(f"messages: {messages}")
     second_response = litellm.completion(
-        model="github/llama3-8b-8192", messages=messages
+        model="github/Llama-3.2-11B-Vision-Instruct", messages=messages
     )  # get a new response from the model where it can see the function response
     print("second response\n", second_response)
 ```

docs/my-website/docs/providers/google_ai_studio/realtime.md

@@ -0,0 +1,92 @@
+# Gemini Realtime API - Google AI Studio
+
+| Feature | Description | Comments |
+| --- | --- | --- |
+| Proxy | ✅ |  |
+| SDK | ⌛️ | Experimental access via `litellm._arealtime`. |
+
+
+## Proxy Usage
+
+### Add model to config 
+
+```yaml
+model_list:
+  - model_name: "gemini-2.0-flash"
+    litellm_params:
+      model: gemini/gemini-2.0-flash-live-001
+    model_info:
+      mode: realtime
+```
+
+### Start proxy 
+
+```bash
+litellm --config /path/to/config.yaml 
+
+# RUNNING on http://0.0.0.0:8000
+```
+
+### Test 
+
+Run this script using node - `node test.js`
+
+```js
+// test.js
+const WebSocket = require("ws");
+
+const url = "ws://0.0.0.0:4000/v1/realtime?model=openai-gemini-2.0-flash";
+
+const ws = new WebSocket(url, {
+    headers: {
+        "api-key": `${LITELLM_API_KEY}`,
+        "OpenAI-Beta": "realtime=v1",
+    },
+});
+
+ws.on("open", function open() {
+    console.log("Connected to server.");
+    ws.send(JSON.stringify({
+        type: "response.create",
+        response: {
+            modalities: ["text"],
+            instructions: "Please assist the user.",
+        }
+    }));
+});
+
+ws.on("message", function incoming(message) {
+    console.log(JSON.parse(message.toString()));
+});
+
+ws.on("error", function handleError(error) {
+    console.error("Error: ", error);
+});
+```
+
+## Limitations 
+
+- Does not support audio transcription. 
+- Does not support tool calling 
+
+## Supported OpenAI Realtime Events
+
+- `session.created`
+- `response.created`
+- `response.output_item.added`
+- `conversation.item.created`
+- `response.content_part.added`
+- `response.text.delta`
+- `response.audio.delta`
+- `response.text.done`
+- `response.audio.done`
+- `response.content_part.done`
+- `response.output_item.done`
+- `response.done`
+
+
+
+## [Supported Session Params](https://github.com/BerriAI/litellm/blob/e87b536d038f77c2a2206fd7433e275c487179ee/litellm/llms/gemini/realtime/transformation.py#L155)
+
+## More Examples
+### [Gemini Realtime API with Audio Input/Output](../../../docs/tutorials/gemini_realtime_with_audio)

docs/my-website/docs/providers/huggingface_rerank.md

@@ -0,0 +1,263 @@
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+import Image from '@theme/IdealImage';
+
+# HuggingFace Rerank
+
+HuggingFace Rerank allows you to use reranking models hosted on Hugging Face infrastructure or your custom endpoints to reorder documents based on their relevance to a query.
+
+| Property | Details |
+|----------|---------|
+| Description | HuggingFace Rerank enables semantic reranking of documents using models hosted on Hugging Face infrastructure or custom endpoints. |
+| Provider Route on LiteLLM | `huggingface/` in model name |
+| Provider Doc | [Hugging Face Hub ↗](https://huggingface.co/models?pipeline_tag=sentence-similarity) |
+
+## Quick Start
+
+### LiteLLM Python SDK
+
+```python showLineNumbers title="Example using LiteLLM Python SDK"
+import litellm
+import os
+
+# Set your HuggingFace token
+os.environ["HF_TOKEN"] = "hf_xxxxxx"
+
+# Basic rerank usage
+response = litellm.rerank(
+    model="huggingface/BAAI/bge-reranker-base",
+    query="What is the capital of the United States?",
+    documents=[
+        "Carson City is the capital city of the American state of Nevada.",
+        "The Commonwealth of the Northern Mariana Islands is a group of islands in the Pacific Ocean. Its capital is Saipan.",
+        "Washington, D.C. is the capital of the United States.",
+        "Capital punishment has existed in the United States since before it was a country.",
+    ],
+    top_n=3,
+)
+
+print(response)
+```
+
+### Custom Endpoint Usage
+
+```python showLineNumbers title="Using custom HuggingFace endpoint"
+import litellm
+
+response = litellm.rerank(
+    model="huggingface/BAAI/bge-reranker-base",
+    query="hello",
+    documents=["hello", "world"],
+    top_n=2,
+    api_base="https://my-custom-hf-endpoint.com",
+    api_key="test_api_key",
+)
+
+print(response)
+```
+
+### Async Usage
+
+```python showLineNumbers title="Async rerank example"
+import litellm
+import asyncio
+import os
+
+os.environ["HF_TOKEN"] = "hf_xxxxxx"
+
+async def async_rerank_example():
+    response = await litellm.arerank(
+        model="huggingface/BAAI/bge-reranker-base",
+        query="What is the capital of the United States?",
+        documents=[
+            "Carson City is the capital city of the American state of Nevada.",
+            "The Commonwealth of the Northern Mariana Islands is a group of islands in the Pacific Ocean. Its capital is Saipan.",
+            "Washington, D.C. is the capital of the United States.",
+            "Capital punishment has existed in the United States since before it was a country.",
+        ],
+        top_n=3,
+    )
+    print(response)
+
+asyncio.run(async_rerank_example())
+```
+
+## LiteLLM Proxy
+
+### 1. Configure your model in config.yaml
+
+<Tabs>
+<TabItem value="config-yaml" label="config.yaml">
+
+```yaml
+model_list:
+  - model_name: bge-reranker-base
+    litellm_params:
+      model: huggingface/BAAI/bge-reranker-base
+      api_key: os.environ/HF_TOKEN
+  - model_name: bge-reranker-large  
+    litellm_params:
+      model: huggingface/BAAI/bge-reranker-large
+      api_key: os.environ/HF_TOKEN
+  - model_name: custom-reranker
+    litellm_params:
+      model: huggingface/BAAI/bge-reranker-base
+      api_base: https://my-custom-hf-endpoint.com
+      api_key: your-custom-api-key
+```
+
+</TabItem>
+</Tabs>
+
+### 2. Start the proxy
+
+```bash
+export HF_TOKEN="hf_xxxxxx"
+litellm --config /path/to/config.yaml
+
+# RUNNING on http://0.0.0.0:4000
+```
+
+### 3. Make rerank requests
+
+<Tabs>
+<TabItem value="curl" label="Curl">
+
+```bash
+curl http://localhost:4000/rerank \
+  -H "Content-Type: application/json" \
+  -H "Authorization: Bearer $LITELLM_API_KEY" \
+  -d '{
+    "model": "bge-reranker-base",
+    "query": "What is the capital of the United States?",
+    "documents": [
+        "Carson City is the capital city of the American state of Nevada.",
+        "The Commonwealth of the Northern Mariana Islands is a group of islands in the Pacific Ocean. Its capital is Saipan.",
+        "Washington, D.C. is the capital of the United States.",
+        "Capital punishment has existed in the United States since before it was a country."
+    ],
+    "top_n": 3
+  }'
+```
+
+</TabItem>
+
+<TabItem value="python-sdk" label="Python SDK">
+
+```python
+import litellm
+
+# Initialize with your LiteLLM proxy URL
+response = litellm.rerank(
+    model="bge-reranker-base",
+    query="What is the capital of the United States?",
+    documents=[
+        "Carson City is the capital city of the American state of Nevada.",
+        "The Commonwealth of the Northern Mariana Islands is a group of islands in the Pacific Ocean. Its capital is Saipan.",
+        "Washington, D.C. is the capital of the United States.",
+        "Capital punishment has existed in the United States since before it was a country.",
+    ],
+    top_n=3,
+    api_base="http://localhost:4000",
+    api_key="your-litellm-api-key"
+)
+
+print(response)
+```
+
+</TabItem>
+
+<TabItem value="requests" label="Using requests library">
+
+```python
+import requests
+
+url = "http://localhost:4000/rerank"
+headers = {
+    "Authorization": "Bearer your-litellm-api-key",
+    "Content-Type": "application/json"
+}
+
+data = {
+    "model": "bge-reranker-base",
+    "query": "What is the capital of the United States?",
+    "documents": [
+        "Carson City is the capital city of the American state of Nevada.",
+        "The Commonwealth of the Northern Mariana Islands is a group of islands in the Pacific Ocean. Its capital is Saipan.",
+        "Washington, D.C. is the capital of the United States.",
+        "Capital punishment has existed in the United States since before it was a country."
+    ],
+    "top_n": 3
+}
+
+response = requests.post(url, headers=headers, json=data)
+print(response.json())
+```
+
+</TabItem>
+</Tabs>
+
+
+
+## Configuration Options
+
+### Authentication
+
+#### Using HuggingFace Token (Serverless)
+```python
+import os
+os.environ["HF_TOKEN"] = "hf_xxxxxx"
+
+# Or pass directly
+litellm.rerank(
+    model="huggingface/BAAI/bge-reranker-base",
+    api_key="hf_xxxxxx",
+    # ... other params
+)
+```
+
+#### Using Custom Endpoint
+```python
+litellm.rerank(
+    model="huggingface/BAAI/bge-reranker-base",
+    api_base="https://your-custom-endpoint.com",
+    api_key="your-custom-key",
+    # ... other params
+)
+```
+
+
+
+## Response Format
+
+The response follows the standard rerank API format:
+
+```json
+{
+  "results": [
+    {
+      "index": 3,
+      "relevance_score": 0.999071
+    },
+    {
+      "index": 4,
+      "relevance_score": 0.7867867
+    },
+    {
+      "index": 0,
+      "relevance_score": 0.32713068
+    }
+  ],
+  "id": "07734bd2-2473-4f07-94e1-0d9f0e6843cf",
+  "meta": {
+    "api_version": {
+      "version": "2",
+      "is_experimental": false
+    },
+    "billed_units": {
+      "search_units": 1
+    }
+  }
+}
+```
+

docs/my-website/docs/providers/litellm_proxy.md

@@ -155,6 +155,59 @@ response = litellm.rerank(
     api_key="your-litellm-proxy-api-key"
 )
 ```
-## **Usage with Langchain, LLamaindex, OpenAI Js, Anthropic SDK, Instructor**
 
-#### [Follow this doc to see how to use litellm proxy with langchain, llamaindex, anthropic etc](../proxy/user_keys)
+
+## Integration with Other Libraries
+
+LiteLLM Proxy works seamlessly with Langchain, LlamaIndex, OpenAI JS, Anthropic SDK, Instructor, and more.
+
+[Learn how to use LiteLLM proxy with these libraries →](../proxy/user_keys)
+
+## Send all SDK requests to LiteLLM Proxy
+
+:::info
+
+Requires v1.72.1 or higher.
+
+:::
+
+Use this when calling LiteLLM Proxy from any library / codebase already using the LiteLLM SDK.
+
+These flags will route all requests through your LiteLLM proxy, regardless of the model specified.
+
+When enabled, requests will use `LITELLM_PROXY_API_BASE` with `LITELLM_PROXY_API_KEY` as the authentication.
+
+### Option 1: Set Globally in Code
+
+```python
+# Set the flag globally for all requests
+litellm.use_litellm_proxy = True
+
+response = litellm.completion(
+    model="vertex_ai/gemini-2.0-flash-001",
+    messages=[{"role": "user", "content": "Hello, how are you?"}]
+)
+```
+
+### Option 2: Control via Environment Variable
+
+```python
+# Control proxy usage through environment variable
+os.environ["USE_LITELLM_PROXY"] = "True"
+
+response = litellm.completion(
+    model="vertex_ai/gemini-2.0-flash-001",
+    messages=[{"role": "user", "content": "Hello, how are you?"}]
+)
+```
+
+### Option 3: Set Per Request
+
+```python
+# Enable proxy for specific requests only
+response = litellm.completion(
+    model="vertex_ai/gemini-2.0-flash-001",
+    messages=[{"role": "user", "content": "Hello, how are you?"}],
+    use_litellm_proxy=True
+)
+```

docs/my-website/docs/providers/llamafile.md

@@ -0,0 +1,158 @@
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+
+# Llamafile
+
+LiteLLM supports all models on Llamafile.
+
+| Property                  | Details                                                                                                                              |
+|---------------------------|--------------------------------------------------------------------------------------------------------------------------------------|
+| Description               | llamafile lets you distribute and run LLMs with a single file. [Docs](https://github.com/Mozilla-Ocho/llamafile/blob/main/README.md) |
+| Provider Route on LiteLLM | `llamafile/` (for OpenAI compatible server)                                                                                          |
+| Provider Doc              | [llamafile ↗](https://github.com/Mozilla-Ocho/llamafile/blob/main/llama.cpp/server/README.md#api-endpoints)                          |
+| Supported Endpoints       | `/chat/completions`, `/embeddings`, `/completions`                                                                                   |
+
+
+# Quick Start
+
+## Usage - litellm.completion (calling OpenAI compatible endpoint)
+llamafile Provides an OpenAI compatible endpoint for chat completions - here's how to call it with LiteLLM
+
+To use litellm to call llamafile add the following to your completion call
+
+* `model="llamafile/<your-llamafile-model-name>"` 
+* `api_base = "your-hosted-llamafile"`
+
+```python
+import litellm 
+
+response = litellm.completion(
+            model="llamafile/mistralai/mistral-7b-instruct-v0.2", # pass the llamafile model name for completeness
+            messages=messages,
+            api_base="http://localhost:8080/v1",
+            temperature=0.2,
+            max_tokens=80)
+
+print(response)
+```
+
+
+## Usage -  LiteLLM Proxy Server (calling OpenAI compatible endpoint)
+
+Here's how to call an OpenAI-Compatible Endpoint with the LiteLLM Proxy Server
+
+1. Modify the config.yaml 
+
+  ```yaml
+  model_list:
+    - model_name: my-model
+      litellm_params:
+        model: llamafile/mistralai/mistral-7b-instruct-v0.2 # add llamafile/ prefix to route as OpenAI provider
+        api_base: http://localhost:8080/v1 # add api base for OpenAI compatible provider
+  ```
+
+1. Start the proxy 
+
+  ```bash
+  $ litellm --config /path/to/config.yaml
+  ```
+
+1. Send Request to LiteLLM Proxy Server
+
+  <Tabs>
+
+  <TabItem value="openai" label="OpenAI Python v1.0.0+">
+
+  ```python
+  import openai
+  client = openai.OpenAI(
+      api_key="sk-1234", # pass litellm proxy key, if you're using virtual keys
+      base_url="http://0.0.0.0:4000" # litellm-proxy-base url
+  )
+
+  response = client.chat.completions.create(
+      model="my-model",
+      messages = [
+          {
+              "role": "user",
+              "content": "what llm are you"
+          }
+      ],
+  )
+
+  print(response)
+  ```
+  </TabItem>
+
+  <TabItem value="curl" label="curl">
+
+  ```shell
+  curl --location 'http://0.0.0.0:4000/chat/completions' \
+      --header 'Authorization: Bearer sk-1234' \
+      --header 'Content-Type: application/json' \
+      --data '{
+      "model": "my-model",
+      "messages": [
+          {
+          "role": "user",
+          "content": "what llm are you"
+          }
+      ],
+  }'
+  ```
+  </TabItem>
+
+  </Tabs>
+
+
+## Embeddings
+
+<Tabs>
+<TabItem value="sdk" label="SDK">
+
+```python
+from litellm import embedding   
+import os
+
+os.environ["LLAMAFILE_API_BASE"] = "http://localhost:8080/v1"
+
+
+embedding = embedding(model="llamafile/sentence-transformers/all-MiniLM-L6-v2", input=["Hello world"])
+
+print(embedding)
+```
+
+</TabItem>
+<TabItem value="proxy" label="PROXY">
+
+1. Setup config.yaml
+
+```yaml
+model_list:
+    - model_name: my-model
+      litellm_params:
+        model: llamafile/sentence-transformers/all-MiniLM-L6-v2 # add llamafile/ prefix to route as OpenAI provider
+        api_base: http://localhost:8080/v1 # add api base for OpenAI compatible provider
+```
+
+1. Start the proxy 
+
+```bash
+$ litellm --config /path/to/config.yaml
+
+# RUNNING on http://0.0.0.0:4000
+```
+
+1. Test it! 
+
+```bash
+curl -L -X POST 'http://0.0.0.0:4000/embeddings' \
+-H 'Authorization: Bearer sk-1234' \
+-H 'Content-Type: application/json' \
+-d '{"input": ["hello world"], "model": "my-model"}'
+```
+
+[See OpenAI SDK/Langchain/etc. examples](../proxy/user_keys.md#embeddings)
+
+</TabItem>
+</Tabs>

docs/my-website/docs/providers/lm_studio.md

@@ -153,3 +153,26 @@ response = embedding(
 )
 print(response)
 ```
+
+
+## Structured Output
+
+LM Studio supports structured outputs via JSON Schema. You can pass a pydantic model or a raw schema using `response_format`.
+LiteLLM sends the schema as `{ "type": "json_schema", "json_schema": {"schema": <your schema>} }`.
+
+```python
+from pydantic import BaseModel
+from litellm import completion
+
+class Book(BaseModel):
+    title: str
+    author: str
+    year: int
+
+response = completion(
+    model="lm_studio/llama-3-8b-instruct",
+    messages=[{"role": "user", "content": "Tell me about The Hobbit"}],
+    response_format=Book,
+)
+print(response.choices[0].message.content)
+```

docs/my-website/docs/providers/meta_llama.md

@@ -0,0 +1,303 @@
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+
+# Meta Llama
+
+| Property | Details |
+|-------|-------|
+| Description | Meta's Llama API provides access to Meta's family of large language models. |
+| Provider Route on LiteLLM | `meta_llama/` |
+| Supported Endpoints | `/chat/completions`, `/completions`, `/responses` |
+| API Reference | [Llama API Reference ↗](https://llama.developer.meta.com?utm_source=partner-litellm&utm_medium=website) |
+
+## Required Variables
+
+```python showLineNumbers title="Environment Variables"
+os.environ["LLAMA_API_KEY"] = ""  # your Meta Llama API key
+```
+
+## Supported Models
+
+:::info
+All models listed here https://llama.developer.meta.com/docs/models/ are supported. We actively maintain the list of models, token window, etc. [here](https://github.com/BerriAI/litellm/blob/main/model_prices_and_context_window.json).
+
+:::
+
+
+| Model ID | Input context length | Output context length | Input Modalities | Output Modalities |
+| --- | --- | --- | --- | --- |
+| `Llama-4-Scout-17B-16E-Instruct-FP8` | 128k | 4028 | Text, Image | Text |
+| `Llama-4-Maverick-17B-128E-Instruct-FP8` | 128k | 4028 | Text, Image | Text |
+| `Llama-3.3-70B-Instruct` | 128k | 4028 | Text | Text |
+| `Llama-3.3-8B-Instruct` | 128k | 4028 | Text | Text |
+
+## Usage - LiteLLM Python SDK
+
+### Non-streaming
+
+```python showLineNumbers title="Meta Llama Non-streaming Completion"
+import os
+import litellm
+from litellm import completion
+
+os.environ["LLAMA_API_KEY"] = ""  # your Meta Llama API key
+
+messages = [{"content": "Hello, how are you?", "role": "user"}]
+
+# Meta Llama call
+response = completion(model="meta_llama/Llama-4-Maverick-17B-128E-Instruct-FP8", messages=messages)
+```
+
+### Streaming
+
+```python showLineNumbers title="Meta Llama Streaming Completion"
+import os
+import litellm
+from litellm import completion
+
+os.environ["LLAMA_API_KEY"] = ""  # your Meta Llama API key
+
+messages = [{"content": "Hello, how are you?", "role": "user"}]
+
+# Meta Llama call with streaming
+response = completion(
+    model="meta_llama/Llama-4-Maverick-17B-128E-Instruct-FP8",
+    messages=messages,
+    stream=True
+)
+
+for chunk in response:
+    print(chunk)
+```
+
+### Function Calling
+
+```python showLineNumbers title="Meta Llama Function Calling"
+import os
+import litellm
+from litellm import completion
+
+os.environ["LLAMA_API_KEY"] = ""  # your Meta Llama API key
+
+messages = [{"content": "What's the weather like in San Francisco?", "role": "user"}]
+
+# Define the function
+tools = [
+    {
+        "type": "function",
+        "function": {
+            "name": "get_weather",
+            "description": "Get the current weather in a given location",
+            "parameters": {
+                "type": "object",
+                "properties": {
+                    "location": {
+                        "type": "string",
+                        "description": "The city and state, e.g. San Francisco, CA"
+                    },
+                    "unit": {
+                        "type": "string",
+                        "enum": ["celsius", "fahrenheit"]
+                    }
+                },
+                "required": ["location"]
+            }
+        }
+    }
+]
+
+# Meta Llama call with function calling
+response = completion(
+    model="meta_llama/Llama-4-Maverick-17B-128E-Instruct-FP8",
+    messages=messages,
+    tools=tools,
+    tool_choice="auto"
+)
+
+print(response.choices[0].message.tool_calls)
+```
+
+### Tool Use
+
+```python showLineNumbers title="Meta Llama Tool Use"
+import os
+import litellm
+from litellm import completion
+
+os.environ["LLAMA_API_KEY"] = ""  # your Meta Llama API key
+
+messages = [{"content": "Create a chart showing the population growth of New York City from 2010 to 2020", "role": "user"}]
+
+# Define the tools
+tools = [
+    {
+        "type": "function",
+        "function": {
+            "name": "create_chart",
+            "description": "Create a chart with the provided data",
+            "parameters": {
+                "type": "object",
+                "properties": {
+                    "chart_type": {
+                        "type": "string",
+                        "enum": ["bar", "line", "pie", "scatter"],
+                        "description": "The type of chart to create"
+                    },
+                    "title": {
+                        "type": "string",
+                        "description": "The title of the chart"
+                    },
+                    "data": {
+                        "type": "object",
+                        "description": "The data to plot in the chart"
+                    }
+                },
+                "required": ["chart_type", "title", "data"]
+            }
+        }
+    }
+]
+
+# Meta Llama call with tool use
+response = completion(
+    model="meta_llama/Llama-4-Maverick-17B-128E-Instruct-FP8",
+    messages=messages,
+    tools=tools,
+    tool_choice="auto"
+)
+
+print(response.choices[0].message.content)
+```
+
+## Usage - LiteLLM Proxy
+
+
+Add the following to your LiteLLM Proxy configuration file:
+
+```yaml showLineNumbers title="config.yaml"
+model_list:
+  - model_name: meta_llama/Llama-3.3-70B-Instruct
+    litellm_params:
+      model: meta_llama/Llama-3.3-70B-Instruct
+      api_key: os.environ/LLAMA_API_KEY
+
+  - model_name: meta_llama/Llama-3.3-8B-Instruct
+    litellm_params:
+      model: meta_llama/Llama-3.3-8B-Instruct
+      api_key: os.environ/LLAMA_API_KEY
+```
+
+Start your LiteLLM Proxy server:
+
+```bash showLineNumbers title="Start LiteLLM Proxy"
+litellm --config config.yaml
+
+# RUNNING on http://0.0.0.0:4000
+```
+
+<Tabs>
+<TabItem value="openai-sdk" label="OpenAI SDK">
+
+```python showLineNumbers title="Meta Llama via Proxy - Non-streaming"
+from openai import OpenAI
+
+# Initialize client with your proxy URL
+client = OpenAI(
+    base_url="http://localhost:4000",  # Your proxy URL
+    api_key="your-proxy-api-key"       # Your proxy API key
+)
+
+# Non-streaming response
+response = client.chat.completions.create(
+    model="meta_llama/Llama-4-Maverick-17B-128E-Instruct-FP8",
+    messages=[{"role": "user", "content": "Write a short poem about AI."}]
+)
+
+print(response.choices[0].message.content)
+```
+
+```python showLineNumbers title="Meta Llama via Proxy - Streaming"
+from openai import OpenAI
+
+# Initialize client with your proxy URL
+client = OpenAI(
+    base_url="http://localhost:4000",  # Your proxy URL
+    api_key="your-proxy-api-key"       # Your proxy API key
+)
+
+# Streaming response
+response = client.chat.completions.create(
+    model="meta_llama/Llama-4-Maverick-17B-128E-Instruct-FP8",
+    messages=[{"role": "user", "content": "Write a short poem about AI."}],
+    stream=True
+)
+
+for chunk in response:
+    if chunk.choices[0].delta.content is not None:
+        print(chunk.choices[0].delta.content, end="")
+```
+
+</TabItem>
+
+<TabItem value="litellm-sdk" label="LiteLLM SDK">
+
+```python showLineNumbers title="Meta Llama via Proxy - LiteLLM SDK"
+import litellm
+
+# Configure LiteLLM to use your proxy
+response = litellm.completion(
+    model="litellm_proxy/meta_llama/Llama-3.3-70B-Instruct",
+    messages=[{"role": "user", "content": "Write a short poem about AI."}],
+    api_base="http://localhost:4000",
+    api_key="your-proxy-api-key"
+)
+
+print(response.choices[0].message.content)
+```
+
+```python showLineNumbers title="Meta Llama via Proxy - LiteLLM SDK Streaming"
+import litellm
+
+# Configure LiteLLM to use your proxy with streaming
+response = litellm.completion(
+    model="litellm_proxy/meta_llama/Llama-3.3-70B-Instruct",
+    messages=[{"role": "user", "content": "Write a short poem about AI."}],
+    api_base="http://localhost:4000",
+    api_key="your-proxy-api-key",
+    stream=True
+)
+
+for chunk in response:
+    if hasattr(chunk.choices[0], 'delta') and chunk.choices[0].delta.content is not None:
+        print(chunk.choices[0].delta.content, end="")
+```
+
+</TabItem>
+
+<TabItem value="curl" label="cURL">
+
+```bash showLineNumbers title="Meta Llama via Proxy - cURL"
+curl http://localhost:4000/v1/chat/completions \
+  -H "Content-Type: application/json" \
+  -H "Authorization: Bearer your-proxy-api-key" \
+  -d '{
+    "model": "meta_llama/Llama-3.3-70B-Instruct",
+    "messages": [{"role": "user", "content": "Write a short poem about AI."}]
+  }'
+```
+
+```bash showLineNumbers title="Meta Llama via Proxy - cURL Streaming"
+curl http://localhost:4000/v1/chat/completions \
+  -H "Content-Type: application/json" \
+  -H "Authorization: Bearer your-proxy-api-key" \
+  -d '{
+    "model": "meta_llama/Llama-3.3-70B-Instruct",
+    "messages": [{"role": "user", "content": "Write a short poem about AI."}],
+    "stream": true
+  }'
+```
+
+</TabItem>
+</Tabs>
+
+For more detailed information on using the LiteLLM Proxy, see the [LiteLLM Proxy documentation](../providers/litellm_proxy).

docs/my-website/docs/providers/mistral.md

@@ -144,20 +144,22 @@ All models listed here https://docs.mistral.ai/platform/endpoints are supported.
 :::
 
 
-| Model Name     | Function Call                                                |
-|----------------|--------------------------------------------------------------|
-| Mistral Small  | `completion(model="mistral/mistral-small-latest", messages)` |
-| Mistral Medium | `completion(model="mistral/mistral-medium-latest", messages)`|
-| Mistral Large 2  | `completion(model="mistral/mistral-large-2407", messages)` |
-| Mistral Large Latest  | `completion(model="mistral/mistral-large-latest", messages)` |
-| Mistral 7B     | `completion(model="mistral/open-mistral-7b", messages)`      |
-| Mixtral 8x7B   | `completion(model="mistral/open-mixtral-8x7b", messages)`    |
-| Mixtral 8x22B  | `completion(model="mistral/open-mixtral-8x22b", messages)`   |
-| Codestral      | `completion(model="mistral/codestral-latest", messages)`     |
-| Mistral NeMo      | `completion(model="mistral/open-mistral-nemo", messages)`     |
-| Mistral NeMo 2407      | `completion(model="mistral/open-mistral-nemo-2407", messages)`     |
-| Codestral Mamba      | `completion(model="mistral/open-codestral-mamba", messages)`     |
-| Codestral Mamba    | `completion(model="mistral/codestral-mamba-latest"", messages)`     |
+| Model Name     | Function Call                                                | Reasoning Support |
+|----------------|--------------------------------------------------------------|-------------------|
+| Mistral Small  | `completion(model="mistral/mistral-small-latest", messages)` | No |
+| Mistral Medium | `completion(model="mistral/mistral-medium-latest", messages)`| No |
+| Mistral Large 2  | `completion(model="mistral/mistral-large-2407", messages)` | No |
+| Mistral Large Latest  | `completion(model="mistral/mistral-large-latest", messages)` | No |
+| **Magistral Small**  | `completion(model="mistral/magistral-small-2506", messages)` | Yes |
+| **Magistral Medium** | `completion(model="mistral/magistral-medium-2506", messages)`| Yes |
+| Mistral 7B     | `completion(model="mistral/open-mistral-7b", messages)`      | No |
+| Mixtral 8x7B   | `completion(model="mistral/open-mixtral-8x7b", messages)`    | No |
+| Mixtral 8x22B  | `completion(model="mistral/open-mixtral-8x22b", messages)`   | No |
+| Codestral      | `completion(model="mistral/codestral-latest", messages)`     | No |
+| Mistral NeMo      | `completion(model="mistral/open-mistral-nemo", messages)`     | No |
+| Mistral NeMo 2407      | `completion(model="mistral/open-mistral-nemo-2407", messages)`     | No |
+| Codestral Mamba      | `completion(model="mistral/open-codestral-mamba", messages)`     | No |
+| Codestral Mamba    | `completion(model="mistral/codestral-mamba-latest"", messages)`     | No |
 
 ## Function Calling 
 
@@ -203,6 +205,112 @@ assert isinstance(
 )
 ```
 
+## Reasoning
+
+Mistral does not directly support reasoning, instead it recommends a specific [system prompt](https://docs.mistral.ai/capabilities/reasoning/) to use with their magistral models. By setting the `reasoning_effort` parameter, LiteLLM will prepend the system prompt to the request. 
+
+If an existing system message is provided, LiteLLM will send both as a list of system messages (you can verify this by enabling `litellm._turn_on_debug()`).
+
+### Supported Models
+
+| Model Name     | Function Call                                                |
+|----------------|--------------------------------------------------------------|
+| Magistral Small  | `completion(model="mistral/magistral-small-2506", messages)` |
+| Magistral Medium | `completion(model="mistral/magistral-medium-2506", messages)`|
+
+### Using Reasoning Effort
+
+The `reasoning_effort` parameter controls how much effort the model puts into reasoning. When used with magistral models.
+
+```python
+from litellm import completion
+import os
+
+os.environ['MISTRAL_API_KEY'] = "your-api-key"
+
+response = completion(
+    model="mistral/magistral-medium-2506",
+    messages=[
+        {"role": "user", "content": "What is 15 multiplied by 7?"}
+    ],
+    reasoning_effort="medium"  # Options: "low", "medium", "high"
+)
+
+print(response)
+```
+
+### Example with System Message
+
+If you already have a system message, LiteLLM will prepend the reasoning instructions:
+
+```python
+response = completion(
+    model="mistral/magistral-medium-2506",
+    messages=[
+        {"role": "system", "content": "You are a helpful math tutor."},
+        {"role": "user", "content": "Explain how to solve quadratic equations."}
+    ],
+    reasoning_effort="high"
+)
+
+# The system message becomes:
+# "When solving problems, think step-by-step in <think> tags before providing your final answer...
+#  
+#  You are a helpful math tutor."
+```
+
+### Usage with LiteLLM Proxy
+
+You can also use reasoning capabilities through the LiteLLM proxy:
+
+<Tabs>
+<TabItem value="Curl" label="Curl Request">
+
+```shell
+curl --location 'http://0.0.0.0:4000/chat/completions' \
+--header 'Content-Type: application/json' \
+--data '{
+      "model": "magistral-medium-2506",
+      "messages": [
+        {
+          "role": "user",
+          "content": "What is the square root of 144? Show your reasoning."
+        }
+      ],
+      "reasoning_effort": "medium"
+    }'
+```
+</TabItem>
+<TabItem value="openai" label="OpenAI v1.0.0+">
+
+```python
+import openai
+client = openai.OpenAI(
+    api_key="anything",
+    base_url="http://0.0.0.0:4000"
+)
+
+response = client.chat.completions.create(
+    model="magistral-medium-2506", 
+    messages=[
+        {
+            "role": "user",
+            "content": "Calculate the area of a circle with radius 5. Show your work."
+        }
+    ],
+    reasoning_effort="high"
+)
+
+print(response)
+```
+</TabItem>
+</Tabs>
+
+### Important Notes
+
+- **Model Compatibility**: Reasoning parameters only work with magistral models
+- **Backward Compatibility**: Non-magistral models will ignore reasoning parameters and work normally
+
 ## Sample Usage - Embedding
 ```python
 from litellm import embedding

docs/my-website/docs/providers/nebius.md

@@ -0,0 +1,195 @@
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+
+# Nebius AI Studio
+https://docs.nebius.com/studio/inference/quickstart
+
+:::tip
+
+**Litellm provides support to all models from Nebius AI Studio. To use a model, set `model=nebius/<any-model-on-nebius-ai-studio>` as a prefix for litellm requests. The full list of supported models is provided at https://studio.nebius.ai/ **
+
+:::
+
+## API Key
+```python
+import os
+# env variable
+os.environ['NEBIUS_API_KEY']
+```
+
+## Sample Usage: Text Generation
+```python
+from litellm import completion
+import os
+
+os.environ['NEBIUS_API_KEY'] = "insert-your-nebius-ai-studio-api-key"
+response = completion(
+    model="nebius/Qwen/Qwen3-235B-A22B",
+    messages=[
+        {
+            "role": "user",
+            "content": "What character was Wall-e in love with?",
+        }
+    ],
+    max_tokens=10,
+    response_format={ "type": "json_object" },
+    seed=123,
+    stop=["\n\n"],
+    temperature=0.6,  # either set temperature or `top_p`
+    top_p=0.01,  # to get as deterministic results as possible
+    tool_choice="auto",
+    tools=[],
+    user="user",
+)
+print(response)
+```
+
+## Sample Usage - Streaming
+```python
+from litellm import completion
+import os
+
+os.environ['NEBIUS_API_KEY'] = ""
+response = completion(
+    model="nebius/Qwen/Qwen3-235B-A22B",
+    messages=[
+        {
+            "role": "user",
+            "content": "What character was Wall-e in love with?",
+        }
+    ],
+    stream=True,
+    max_tokens=10,
+    response_format={ "type": "json_object" },
+    seed=123,
+    stop=["\n\n"],
+    temperature=0.6,  # either set temperature or `top_p`
+    top_p=0.01,  # to get as deterministic results as possible
+    tool_choice="auto",
+    tools=[],
+    user="user",
+)
+
+for chunk in response:
+    print(chunk)
+```
+
+## Sample Usage - Embedding
+```python
+from litellm import embedding
+import os
+
+os.environ['NEBIUS_API_KEY'] = ""
+response = embedding(
+    model="nebius/BAAI/bge-en-icl",
+    input=["What character was Wall-e in love with?"],
+)
+print(response)
+```
+
+
+## Usage with LiteLLM Proxy Server
+
+Here's how to call a Nebius AI Studio model with the LiteLLM Proxy Server
+
+1. Modify the config.yaml 
+
+  ```yaml
+  model_list:
+    - model_name: my-model
+      litellm_params:
+        model: nebius/<your-model-name>  # add nebius/ prefix to use Nebius AI Studio as provider
+        api_key: api-key                 # api key to send your model
+  ```
+2. Start the proxy 
+  ```bash
+  $ litellm --config /path/to/config.yaml
+  ```
+
+3. Send Request to LiteLLM Proxy Server
+
+  <Tabs>
+
+  <TabItem value="openai" label="OpenAI Python v1.0.0+">
+
+  ```python
+  import openai
+  client = openai.OpenAI(
+      api_key="litellm-proxy-key",             # pass litellm proxy key, if you're using virtual keys
+      base_url="http://0.0.0.0:4000" # litellm-proxy-base url
+  )
+
+  response = client.chat.completions.create(
+      model="my-model",
+      messages = [
+          {
+              "role": "user",
+              "content": "What character was Wall-e in love with?"
+          }
+      ],
+  )
+
+  print(response)
+  ```
+  </TabItem>
+
+  <TabItem value="curl" label="curl">
+
+  ```shell
+  curl --location 'http://0.0.0.0:4000/chat/completions' \
+      --header 'Authorization: litellm-proxy-key' \
+      --header 'Content-Type: application/json' \
+      --data '{
+      "model": "my-model",
+      "messages": [
+          {
+          "role": "user",
+          "content": "What character was Wall-e in love with?"
+          }
+      ],
+  }'
+  ```
+  </TabItem>
+
+  </Tabs>
+
+## Supported Parameters
+
+The Nebius provider supports the following parameters:
+
+### Chat Completion Parameters
+
+| Parameter | Type | Description |
+| --------- | ---- | ----------- |
+| frequency_penalty | number | Penalizes new tokens based on their frequency in the text |
+| function_call | string/object | Controls how the model calls functions |
+| functions | array | List of functions for which the model may generate JSON inputs |
+| logit_bias | map | Modifies the likelihood of specified tokens |
+| max_tokens | integer | Maximum number of tokens to generate |
+| n | integer | Number of completions to generate |
+| presence_penalty | number | Penalizes tokens based on if they appear in the text so far |
+| response_format | object | Format of the response, e.g., `{"type": "json"}` |
+| seed | integer | Sampling seed for deterministic results |
+| stop | string/array | Sequences where the API will stop generating tokens |
+| stream | boolean | Whether to stream the response |
+| temperature | number | Controls randomness (0-2) |
+| top_p | number | Controls nucleus sampling |
+| tool_choice | string/object | Controls which (if any) function to call |
+| tools | array | List of tools the model can use |
+| user | string | User identifier |
+
+### Embedding Parameters
+
+| Parameter | Type | Description |
+| --------- | ---- | ----------- |
+| input | string/array | Text to embed |
+| user | string | User identifier |
+
+## Error Handling
+
+The integration uses the standard LiteLLM error handling. Common errors include:
+
+- **Authentication Error**: Check your API key
+- **Model Not Found**: Ensure you're using a valid model name
+- **Rate Limit Error**: You've exceeded your rate limits
+- **Timeout Error**: Request took too long to complete

docs/my-website/docs/providers/novita.md

@@ -0,0 +1,234 @@
+import Image from '@theme/IdealImage';
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+
+# Novita AI
+
+| Property | Details |
+|-------|-------|
+| Description | Novita AI is an AI cloud platform that helps developers easily deploy AI models through a simple API, backed by affordable and reliable GPU cloud infrastructure. LiteLLM supports all models from [Novita AI](https://novita.ai/models/llm?utm_source=github_litellm&utm_medium=github_readme&utm_campaign=github_link) |
+| Provider Route on LiteLLM | `novita/` |
+| Provider Doc | [Novita AI Docs ↗](https://novita.ai/docs/guides/introduction) |
+| API Endpoint for Provider | https://api.novita.ai/v3/openai |
+| Supported OpenAI Endpoints | `/chat/completions`, `/completions` |
+
+<br />
+
+## API Keys
+
+Get your API key [here](https://novita.ai/settings/key-management)
+```python
+import os
+os.environ["NOVITA_API_KEY"] = "your-api-key"
+```
+
+## Supported OpenAI Params
+- max_tokens
+- stream
+- stream_options
+- n
+- seed
+- frequency_penalty
+- presence_penalty
+- repetition_penalty
+- stop
+- temperature
+- top_p
+- top_k
+- min_p
+- logit_bias
+- logprobs
+- top_logprobs
+- tools
+- response_format
+- separate_reasoning
+
+
+## Sample Usage
+
+<Tabs>
+<TabItem value="sdk" label="SDK">
+
+```python
+import os
+from litellm import completion
+os.environ["NOVITA_API_KEY"] = ""
+
+response = completion(
+    model="novita/deepseek/deepseek-r1-turbo",
+    messages=[{"role": "user", "content": "List 5 popular cookie recipes."}]
+)
+
+content = response.get('choices', [{}])[0].get('message', {}).get('content')
+print(content)
+```
+
+</TabItem>
+<TabItem value="proxy" label="PROXY">
+
+1. Add model to config.yaml
+```yaml
+model_list:
+  - model_name: deepseek-r1-turbo
+    litellm_params:
+      model: novita/deepseek/deepseek-r1-turbo
+      api_key: os.environ/NOVITA_API_KEY
+```
+
+2. Start Proxy 
+
+```
+$ litellm --config /path/to/config.yaml
+```
+
+3. Make Request!
+
+```bash
+curl -X POST 'http://0.0.0.0:4000/chat/completions' \
+-H 'Content-Type: application/json' \
+-H 'Authorization: Bearer sk_sujEQQEjTRxGUiMLN3TJh2KadRX4pw2TLWRoIKeoYZ0' \
+-d '{
+  "model": "deepseek-r1-turbo",
+  "messages": [
+      {"role": "user", "content": "List 5 popular cookie recipes."}
+  ]
+}
+'
+```
+
+</TabItem>
+</Tabs>
+
+
+## Tool Calling
+
+```python
+from litellm import completion
+import os
+# set env
+os.environ["NOVITA_API_KEY"] = ""
+
+tools = [
+    {
+        "type": "function",
+        "function": {
+            "name": "get_current_weather",
+            "description": "Get the current weather in a given location",
+            "parameters": {
+                "type": "object",
+                "properties": {
+                    "location": {
+                        "type": "string",
+                        "description": "The city and state, e.g. San Francisco, CA",
+                    },
+                    "unit": {"type": "string", "enum": ["celsius", "fahrenheit"]},
+                },
+                "required": ["location"],
+            },
+        },
+    }
+]
+messages = [{"role": "user", "content": "What's the weather like in Boston today?"}]
+
+response = completion(
+    model="novita/deepseek/deepseek-r1-turbo",
+    messages=messages,
+    tools=tools,
+)
+# Add any assertions, here to check response args
+print(response)
+assert isinstance(response.choices[0].message.tool_calls[0].function.name, str)
+assert isinstance(
+    response.choices[0].message.tool_calls[0].function.arguments, str
+)
+
+```
+
+## JSON Mode
+
+<Tabs>
+<TabItem value="sdk" label="SDK">
+
+```python
+from litellm import completion 
+import json 
+import os 
+
+os.environ['NOVITA_API_KEY'] = ""
+
+messages = [
+    {
+        "role": "user",
+        "content": "List 5 popular cookie recipes."
+    }
+]
+
+completion(
+    model="novita/deepseek/deepseek-r1-turbo", 
+    messages=messages, 
+    response_format={"type": "json_object"} # 👈 KEY CHANGE
+)
+
+print(json.loads(completion.choices[0].message.content))
+```
+
+</TabItem>
+<TabItem value="proxy" label="PROXY">
+
+1. Add model to config.yaml
+```yaml
+model_list:
+  - model_name: deepseek-r1-turbo
+    litellm_params:
+      model: novita/deepseek/deepseek-r1-turbo
+      api_key: os.environ/NOVITA_API_KEY
+```
+
+2. Start Proxy 
+
+```
+$ litellm --config /path/to/config.yaml
+```
+
+3. Make Request!
+
+```bash
+curl -X POST 'http://0.0.0.0:4000/chat/completions' \
+-H 'Content-Type: application/json' \
+-H 'Authorization: Bearer sk-1234' \
+-d '{
+  "model": "deepseek-r1-turbo",
+  "messages": [
+      {"role": "user", "content": "List 5 popular cookie recipes."}
+  ],
+  "response_format": {"type": "json_object"}
+}
+'
+```
+
+</TabItem>
+</Tabs>
+
+
+## Chat Models
+
+🚨 LiteLLM supports ALL Novita AI models, send `model=novita/<your-novita-model>` to send it to Novita AI. See all Novita AI models [here](https://novita.ai/models/llm?utm_source=github_litellm&utm_medium=github_readme&utm_campaign=github_link)
+
+| Model Name                | Function Call                                       |
+|---------------------------|-----------------------------------------------------|
+| novita/deepseek/deepseek-r1-turbo | `completion('novita/deepseek/deepseek-r1-turbo', messages)` | `os.environ['NOVITA_API_KEY']` |
+| novita/deepseek/deepseek-v3-turbo | `completion('novita/deepseek/deepseek-v3-turbo', messages)` | `os.environ['NOVITA_API_KEY']` |
+| novita/deepseek/deepseek-v3-0324 | `completion('novita/deepseek/deepseek-v3-0324', messages)` | `os.environ['NOVITA_API_KEY']` |
+| novita/qwen/qwen3-235b-a22b-fp8 | `completion('novita/qwen/qwen/qwen3-235b-a22b-fp8', messages)` | `os.environ['NOVITA_API_KEY']` |
+| novita/qwen/qwen3-30b-a3b-fp8 | `completion('novita/qwen/qwen3-30b-a3b-fp8', messages)` | `os.environ['NOVITA_API_KEY']` |
+| novita/qwen/qwen/qwen3-32b-fp8 | `completion('novita/qwen/qwen3-32b-fp8', messages)` | `os.environ['NOVITA_API_KEY']` |
+| novita/qwen/qwen3-30b-a3b-fp8 | `completion('novita/qwen/qwen3-30b-a3b-fp8', messages)` | `os.environ['NOVITA_API_KEY']` |
+| novita/qwen/qwen2.5-vl-72b-instruct | `completion('novita/qwen/qwen2.5-vl-72b-instruct', messages)` | `os.environ['NOVITA_API_KEY']` |
+| novita/meta-llama/llama-4-maverick-17b-128e-instruct-fp8 | `completion('novita/meta-llama/llama-4-maverick-17b-128e-instruct-fp8', messages)` | `os.environ['NOVITA_API_KEY']` |
+| novita/meta-llama/llama-3.3-70b-instruct | `completion('novita/meta-llama/llama-3.3-70b-instruct', messages)` | `os.environ['NOVITA_API_KEY']` |
+| novita/meta-llama/llama-3.1-8b-instruct | `completion('novita/meta-llama/llama-3.1-8b-instruct', messages)` | `os.environ['NOVITA_API_KEY']` |
+| novita/meta-llama/llama-3.1-8b-instruct-max | `completion('novita/meta-llama/llama-3.1-8b-instruct-max', messages)` | `os.environ['NOVITA_API_KEY']` |
+| novita/meta-llama/llama-3.1-70b-instruct | `completion('novita/meta-llama/llama-3.1-70b-instruct', messages)` | `os.environ['NOVITA_API_KEY']` |
+| novita/gryphe/mythomax-l2-13b | `completion('novita/gryphe/mythomax-l2-13b', messages)` | `os.environ['NOVITA_API_KEY']` |
+| novita/google/gemma-3-27b-it | `completion('novita/google/gemma-3-27b-it', messages)` | `os.environ['NOVITA_API_KEY']` |
+| novita/mistralai/mistral-nemo | `completion('novita/mistralai/mistral-nemo', messages)` | `os.environ['NOVITA_API_KEY']` |

docs/my-website/docs/providers/nscale.md

@@ -0,0 +1,180 @@
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+
+# Nscale (EU Sovereign)
+
+https://docs.nscale.com/docs/inference/chat
+
+:::tip
+
+**We support ALL Nscale models, just set `model=nscale/<any-model-on-nscale>` as a prefix when sending litellm requests**
+
+:::
+
+| Property | Details |
+|-------|-------|
+| Description | European-domiciled full-stack AI cloud platform for LLMs and image generation. |
+| Provider Route on LiteLLM | `nscale/` |
+| Supported Endpoints | `/chat/completions`, `/images/generations` |
+| API Reference | [Nscale docs](https://docs.nscale.com/docs/getting-started/overview) |
+
+## Required Variables
+
+```python showLineNumbers title="Environment Variables"
+os.environ["NSCALE_API_KEY"] = ""  # your Nscale API key
+```
+
+## Explore Available Models  
+
+Explore our full list of text and multimodal AI models — all available at highly competitive pricing:
+📚 [Full List of Models](https://docs.nscale.com/docs/inference/serverless-models/current)  
+
+
+## Key Features
+- **EU Sovereign**: Full data sovereignty and compliance with European regulations
+- **Ultra-Low Cost (starting at $0.01 / M tokens)**: Extremely competitive pricing for both text and image generation models
+- **Production Grade**: Reliable serverless deployments with full isolation
+- **No Setup Required**: Instant access to compute without infrastructure management
+- **Full Control**: Your data remains private and isolated
+
+## Usage - LiteLLM Python SDK
+
+### Text Generation
+
+```python showLineNumbers title="Nscale Text Generation"
+from litellm import completion
+import os
+
+os.environ["NSCALE_API_KEY"] = ""  # your Nscale API key
+response = completion(
+    model="nscale/meta-llama/Llama-4-Scout-17B-16E-Instruct",
+    messages=[{"role": "user", "content": "What is LiteLLM?"}]
+)
+print(response)
+```
+
+```python showLineNumbers title="Nscale Text Generation - Streaming"
+from litellm import completion
+import os
+
+os.environ["NSCALE_API_KEY"] = ""  # your Nscale API key
+stream = completion(
+    model="nscale/meta-llama/Llama-4-Scout-17B-16E-Instruct",
+    messages=[{"role": "user", "content": "What is LiteLLM?"}],
+    stream=True
+)
+
+for chunk in stream:
+    if chunk.choices[0].delta.content is not None:
+        print(chunk.choices[0].delta.content, end="")
+```
+
+### Image Generation
+
+```python showLineNumbers title="Nscale Image Generation"
+from litellm import image_generation
+import os
+
+os.environ["NSCALE_API_KEY"] = ""  # your Nscale API key
+response = image_generation(
+    model="nscale/stabilityai/stable-diffusion-xl-base-1.0",
+    prompt="A beautiful sunset over mountains",
+    n=1,
+    size="1024x1024"
+)
+print(response)
+```
+
+## Usage - LiteLLM Proxy
+
+Add the following to your LiteLLM Proxy configuration file:
+
+```yaml showLineNumbers title="config.yaml"
+model_list:
+  - model_name: nscale/meta-llama/Llama-4-Scout-17B-16E-Instruct
+    litellm_params:
+      model: nscale/meta-llama/Llama-4-Scout-17B-16E-Instruct
+      api_key: os.environ/NSCALE_API_KEY
+  - model_name: nscale/meta-llama/Llama-3.3-70B-Instruct
+    litellm_params:
+      model: nscale/meta-llama/Llama-3.3-70B-Instruct
+      api_key: os.environ/NSCALE_API_KEY
+  - model_name: nscale/stabilityai/stable-diffusion-xl-base-1.0
+    litellm_params:
+      model: nscale/stabilityai/stable-diffusion-xl-base-1.0
+      api_key: os.environ/NSCALE_API_KEY
+```
+
+Start your LiteLLM Proxy server:
+
+```bash showLineNumbers title="Start LiteLLM Proxy"
+litellm --config config.yaml
+
+# RUNNING on http://0.0.0.0:4000
+```
+
+<Tabs>
+<TabItem value="openai-sdk" label="OpenAI SDK">
+
+```python showLineNumbers title="Nscale via Proxy - Non-streaming"
+from openai import OpenAI
+
+# Initialize client with your proxy URL
+client = OpenAI(
+    base_url="http://localhost:4000",  # Your proxy URL
+    api_key="your-proxy-api-key"       # Your proxy API key
+)
+
+# Non-streaming response
+response = client.chat.completions.create(
+    model="nscale/meta-llama/Llama-4-Scout-17B-16E-Instruct",
+    messages=[{"role": "user", "content": "What is LiteLLM?"}]
+)
+
+print(response.choices[0].message.content)
+```
+
+</TabItem>
+
+<TabItem value="litellm-sdk" label="LiteLLM SDK">
+
+```python showLineNumbers title="Nscale via Proxy - LiteLLM SDK"
+import litellm
+
+# Configure LiteLLM to use your proxy
+response = litellm.completion(
+    model="litellm_proxy/nscale/meta-llama/Llama-4-Scout-17B-16E-Instruct",
+    messages=[{"role": "user", "content": "What is LiteLLM?"}],
+    api_base="http://localhost:4000",
+    api_key="your-proxy-api-key"
+)
+
+print(response.choices[0].message.content)
+```
+
+</TabItem>
+
+<TabItem value="curl" label="cURL">
+
+```bash showLineNumbers title="Nscale via Proxy - cURL"
+curl http://localhost:4000/v1/chat/completions \
+  -H "Content-Type: application/json" \
+  -H "Authorization: Bearer your-proxy-api-key" \
+  -d '{
+    "model": "nscale/meta-llama/Llama-4-Scout-17B-16E-Instruct",
+    "messages": [{"role": "user", "content": "What is LiteLLM?"}]
+  }'
+```
+
+</TabItem>
+</Tabs>
+
+## Getting Started
+1. Create an account at [console.nscale.com](https://console.nscale.com)
+2. Claim free credit
+3. Create an API key in settings
+4. Start making API calls using LiteLLM
+
+## Additional Resources
+- [Nscale Documentation](https://docs.nscale.com/docs/getting-started/overview)
+- [Blog: Sovereign Serverless](https://www.nscale.com/blog/sovereign-serverless-how-we-designed-full-isolation-without-sacrificing-performance) 

docs/my-website/docs/providers/nvidia_nim.md

@@ -10,10 +10,19 @@ https://docs.api.nvidia.com/nim/reference/
 
 :::
 
+| Property | Details |
+|-------|-------|
+| Description | Nvidia NIM is a platform that provides a simple API for deploying and using AI models. LiteLLM supports all models from [Nvidia NIM](https://developer.nvidia.com/nim/) |
+| Provider Route on LiteLLM | `nvidia_nim/` |
+| Provider Doc | [Nvidia NIM Docs ↗](https://developer.nvidia.com/nim/) |
+| API Endpoint for Provider | https://integrate.api.nvidia.com/v1/ |
+| Supported OpenAI Endpoints | `/chat/completions`, `/completions`, `/responses`, `/embeddings` |
+
 ## API Key
 ```python
 # env variable
-os.environ['NVIDIA_NIM_API_KEY']
+os.environ['NVIDIA_NIM_API_KEY'] = ""
+os.environ['NVIDIA_NIM_API_BASE'] = "" # [OPTIONAL] - default is https://integrate.api.nvidia.com/v1/
 ```
 
 ## Sample Usage
@@ -100,6 +109,7 @@ Here's how to call an Nvidia NIM Endpoint with the LiteLLM Proxy Server
       litellm_params:
         model: nvidia_nim/<your-model-name>  # add nvidia_nim/ prefix to route as Nvidia NIM provider
         api_key: api-key                 # api key to send your model
+       # api_base: "" # [OPTIONAL] - default is https://integrate.api.nvidia.com/v1/
   ```