diff --git a/.circleci/config.yml b/.circleci/config.yml index 23a62df4789..790efc79862 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -28,34 +28,28 @@ commands: - setup_google_dns - restore_cache: keys: - - v2-litellm-deps-{{ checksum "requirements.txt" }}-{{ checksum ".circleci/config.yml" }} - - v2-litellm-deps- + - v3-litellm-uv-deps-{{ checksum "requirements.txt" }}-{{ checksum ".circleci/config.yml" }} - run: name: Install Dependencies command: | - python -m pip install --upgrade pip - python -m pip install -r requirements.txt - pip install "pytest-mock==3.12.0" - pip install "pytest==7.3.1" - pip install "pytest-retry==1.6.3" - pip install "pytest-cov==5.0.0" - pip install "pytest-asyncio==0.21.1" - pip install "respx==0.22.0" - pip install "hypercorn==0.17.3" - pip install "pydantic==2.11.0" - pip install "mcp==1.25.0" - pip install "requests-mock>=1.12.1" - pip install "responses==0.25.7" - pip install "pytest-xdist==3.6.1" - pip install "pytest-timeout==2.2.0" - pip install "semantic_router==0.1.10" - pip install "fastapi-offline==1.7.3" - pip install "a2a" + python -m pip install --upgrade pip uv + # Use uv for the heavy requirements.txt (10-100x faster than pip) + uv pip install --system -r requirements.txt + # Use pip for test deps (small set, avoids uv strict-resolution + # conflicts with transitive dep pins like openai<2 and pydantic>=2.11.5) + pip install "pytest-mock==3.12.0" "pytest==7.3.1" "pytest-retry==1.6.3" \ + "pytest-asyncio==0.21.1" "respx==0.22.0" "hypercorn==0.17.3" \ + "pydantic==2.11.0" "mcp==1.25.0" "requests-mock>=1.12.1" \ + "responses==0.25.7" "pytest-xdist==3.6.1" "pytest-timeout==2.2.0" \ + "pytest-cov==5.0.0" "semantic_router==0.1.10" "fastapi-offline==1.7.3" \ + "a2a" "parameterized>=0.9.0" - setup_litellm_enterprise_pip - save_cache: paths: - - ~/.cache/pip - key: v2-litellm-deps-{{ checksum "requirements.txt" }}-{{ checksum ".circleci/config.yml" }} + - ~/.local/lib + - ~/.local/bin + - ~/.cache/uv + key: v3-litellm-uv-deps-{{ checksum "requirements.txt" }}-{{ checksum ".circleci/config.yml" }} jobs: # Add Windows testing job @@ -100,8 +94,8 @@ jobs: - run: name: Install Dependencies command: | - python -m pip install --upgrade pip - python -m pip install -r requirements.txt + python -m pip install --upgrade pip uv + uv pip install --system -r requirements.txt pip uninstall fastuuid -y pip install "mypy==1.18.2" - run: @@ -120,6 +114,7 @@ jobs: username: ${DOCKERHUB_USERNAME} password: ${DOCKERHUB_PASSWORD} working_directory: ~/project + resource_class: medium steps: - checkout - setup_google_dns @@ -154,50 +149,17 @@ jobs: command: | python -m pip install --upgrade pip python -m pip install -r .circleci/requirements.txt - pip install "pytest==7.3.1" - pip install "pytest-retry==1.6.3" - pip install "pytest-asyncio==0.21.1" - pip install "pytest-cov==5.0.0" - pip install "mypy==1.18.2" - pip install "google-generativeai==0.3.2" - pip install "google-cloud-aiplatform==1.43.0" - pip install pyarrow - pip install "boto3==1.36.0" - pip install "aioboto3==13.4.0" - pip install langchain - pip install lunary==0.2.5 - pip install "azure-identity==1.16.1" - pip install "langfuse==2.59.7" - pip install "logfire==0.29.0" - pip install numpydoc - pip install traceloop-sdk==0.21.1 - pip install opentelemetry-api==1.25.0 - pip install opentelemetry-sdk==1.25.0 - pip install opentelemetry-exporter-otlp==1.25.0 - pip install openai==1.100.1 - pip install prisma==0.11.0 - pip install "detect_secrets==1.5.0" - pip install "httpx==0.24.1" - pip install "respx==0.22.0" - pip install fastapi - pip install "gunicorn==21.2.0" - pip install "anyio==4.2.0" - pip install "aiodynamo==23.10.1" - pip install "asyncio==3.4.3" - pip install "apscheduler==3.10.4" - pip install "PyGithub==1.59.1" - pip install argon2-cffi - pip install "pytest-mock==3.12.0" - pip install python-multipart - pip install google-cloud-aiplatform - pip install prometheus-client==0.20.0 - pip install "pydantic==2.10.2" - pip install "diskcache==5.6.1" - pip install "Pillow==10.3.0" - pip install "jsonschema==4.22.0" - pip install "pytest-xdist==3.6.1" - pip install "pytest-timeout==2.2.0" - pip install "websockets==13.1.0" + pip install "pytest==7.3.1" "pytest-retry==1.6.3" "pytest-asyncio==0.21.1" "pytest-cov==5.0.0" \ + "mypy==1.18.2" "google-generativeai==0.3.2" "google-cloud-aiplatform==1.43.0" pyarrow \ + "boto3==1.36.0" "aioboto3==13.4.0" langchain lunary==0.2.5 \ + "azure-identity==1.16.1" "langfuse==2.59.7" "logfire==0.29.0" numpydoc \ + traceloop-sdk==0.21.1 openai==1.100.1 prisma==0.11.0 \ + "detect_secrets==1.5.0" "respx==0.22.0" fastapi \ + "gunicorn==21.2.0" "aiodynamo==23.10.1" "asyncio==3.4.3" \ + "apscheduler==3.10.4" "PyGithub==1.59.1" argon2-cffi "pytest-mock==3.12.0" \ + python-multipart prometheus-client==0.20.0 "pydantic==2.10.2" \ + "diskcache==5.6.1" "Pillow==10.3.0" "jsonschema==4.22.0" \ + "pytest-xdist==3.6.1" "pytest-timeout==2.2.0" "websockets==13.1.0" pip install semantic_router --no-deps pip install aurelio_sdk --no-deps pip uninstall posthog -y @@ -243,7 +205,7 @@ jobs: -n 4 \ --timeout=300 \ --timeout_method=thread" - no_output_timeout: 120m + no_output_timeout: 15m - run: name: Rename the coverage files command: | @@ -282,50 +244,17 @@ jobs: command: | python -m pip install --upgrade pip python -m pip install -r .circleci/requirements.txt - pip install "pytest==7.3.1" - pip install "pytest-retry==1.6.3" - pip install "pytest-asyncio==0.21.1" - pip install "pytest-cov==5.0.0" - pip install "mypy==1.18.2" - pip install "google-generativeai==0.3.2" - pip install "google-cloud-aiplatform==1.43.0" - pip install pyarrow - pip install "boto3==1.36.0" - pip install "aioboto3==13.4.0" - pip install langchain - pip install lunary==0.2.5 - pip install "azure-identity==1.16.1" - pip install "langfuse==2.59.7" - pip install "logfire==0.29.0" - pip install numpydoc - pip install traceloop-sdk==0.21.1 - pip install opentelemetry-api==1.25.0 - pip install opentelemetry-sdk==1.25.0 - pip install opentelemetry-exporter-otlp==1.25.0 - pip install openai==1.100.1 - pip install prisma==0.11.0 - pip install "detect_secrets==1.5.0" - pip install "httpx==0.24.1" - pip install "respx==0.22.0" - pip install fastapi - pip install "gunicorn==21.2.0" - pip install "anyio==4.2.0" - pip install "aiodynamo==23.10.1" - pip install "asyncio==3.4.3" - pip install "apscheduler==3.10.4" - pip install "PyGithub==1.59.1" - pip install argon2-cffi - pip install "pytest-mock==3.12.0" - pip install python-multipart - pip install google-cloud-aiplatform - pip install prometheus-client==0.20.0 - pip install "pydantic==2.10.2" - pip install "diskcache==5.6.1" - pip install "Pillow==10.3.0" - pip install "jsonschema==4.22.0" - pip install "pytest-xdist==3.6.1" - pip install "pytest-timeout==2.2.0" - pip install "websockets==13.1.0" + pip install "pytest==7.3.1" "pytest-retry==1.6.3" "pytest-asyncio==0.21.1" "pytest-cov==5.0.0" \ + "mypy==1.18.2" "google-generativeai==0.3.2" "google-cloud-aiplatform==1.43.0" pyarrow \ + "boto3==1.36.0" "aioboto3==13.4.0" langchain lunary==0.2.5 \ + "azure-identity==1.16.1" "langfuse==2.59.7" "logfire==0.29.0" numpydoc \ + traceloop-sdk==0.21.1 openai==1.100.1 prisma==0.11.0 \ + "detect_secrets==1.5.0" "respx==0.22.0" fastapi \ + "gunicorn==21.2.0" "aiodynamo==23.10.1" "asyncio==3.4.3" \ + "apscheduler==3.10.4" "PyGithub==1.59.1" argon2-cffi "pytest-mock==3.12.0" \ + python-multipart prometheus-client==0.20.0 "pydantic==2.10.2" \ + "diskcache==5.6.1" "Pillow==10.3.0" "jsonschema==4.22.0" \ + "pytest-xdist==3.6.1" "pytest-timeout==2.2.0" "websockets==13.1.0" pip install semantic_router --no-deps pip install aurelio_sdk --no-deps pip uninstall posthog -y @@ -371,7 +300,7 @@ jobs: -n 4 \ --timeout=300 \ --timeout_method=thread" - no_output_timeout: 120m + no_output_timeout: 15m - run: name: Rename the coverage files command: | @@ -393,6 +322,7 @@ jobs: username: ${DOCKERHUB_USERNAME} password: ${DOCKERHUB_PASSWORD} working_directory: ~/project + resource_class: medium steps: - checkout @@ -471,29 +401,20 @@ jobs: command: | pwd ls - python -m pytest -vv tests/local_testing --cov=litellm --cov-report=xml -x --junitxml=test-results/junit.xml --durations=5 -k "langfuse" - no_output_timeout: 120m - - run: - name: Rename the coverage files - command: | - mv coverage.xml langfuse_coverage.xml - mv .coverage langfuse_coverage - + python -m pytest -v tests/local_testing -x --junitxml=test-results/junit.xml --durations=5 -k "langfuse" + no_output_timeout: 15m # Store test results - store_test_results: path: test-results - - persist_to_workspace: - root: . - paths: - - langfuse_coverage.xml - - langfuse_coverage caching_unit_tests: docker: - image: cimg/python:3.11 auth: username: ${DOCKERHUB_USERNAME} password: ${DOCKERHUB_PASSWORD} + resource_class: large working_directory: ~/project + parallelism: 2 steps: - checkout @@ -511,7 +432,8 @@ jobs: - restore_cache: keys: - - v1-dependencies-{{ checksum ".circleci/requirements.txt" }} + - v2-caching-deps-{{ checksum ".circleci/requirements.txt" }} + - v2-caching-deps- - run: name: Install Dependencies command: | @@ -559,11 +481,13 @@ jobs: pip install "Pillow==10.3.0" pip install "jsonschema==4.22.0" pip install "websockets==13.1.0" + pip install "pytest-xdist==3.6.1" - setup_litellm_enterprise_pip - save_cache: paths: - - ./venv - key: v1-dependencies-{{ checksum ".circleci/requirements.txt" }} + - /home/circleci/.pyenv/versions + - /home/circleci/.local + key: v2-caching-deps-{{ checksum ".circleci/requirements.txt" }} - run: name: Run prisma ./docker/entrypoint.sh command: | @@ -578,22 +502,23 @@ jobs: command: | pwd ls - python -m pytest -vv tests/local_testing --cov=litellm --cov-report=xml -x --junitxml=test-results/junit.xml --durations=5 -k "caching or cache" - no_output_timeout: 120m - - run: - name: Rename the coverage files - command: | - mv coverage.xml caching_coverage.xml - mv .coverage caching_coverage + mkdir -p test-results + + TEST_FILES=$(circleci tests glob "tests/local_testing/**/test_*.py") + + echo "$TEST_FILES" | circleci tests run \ + --split-by=timings \ + --verbose \ + --command="xargs python -m pytest \ + -v \ + --junitxml=test-results/junit.xml \ + --durations=5 \ + -k 'caching or cache'" + no_output_timeout: 15m # Store test results - store_test_results: path: test-results - - persist_to_workspace: - root: . - paths: - - caching_coverage.xml - - caching_coverage auth_ui_unit_tests: docker: - image: cimg/python:3.11 @@ -608,12 +533,12 @@ jobs: - run: name: Install Dependencies command: | - python -m pip install --upgrade pip - python -m pip install -r requirements.txt + python -m pip install --upgrade pip uv + uv pip install --system -r requirements.txt pip install "pytest==7.3.1" pip install "pytest-retry==1.6.3" pip install "pytest-asyncio==0.21.1" - pip install "pytest-cov==5.0.0" + pip install "pytest-xdist==3.6.1" - save_cache: paths: - ./venv @@ -631,25 +556,13 @@ jobs: command: | pwd ls - python -m pytest -vv tests/proxy_admin_ui_tests -x --cov=litellm --cov-report=xml --junitxml=test-results/junit.xml --durations=5 - no_output_timeout: 120m - - - run: - name: Rename the coverage files - command: | - mv coverage.xml auth_ui_unit_tests_coverage.xml - mv .coverage auth_ui_unit_tests_coverage + python -m pytest -v tests/proxy_admin_ui_tests -x --junitxml=test-results/junit.xml --durations=5 -n 2 + no_output_timeout: 15m # Store test results - store_test_results: path: test-results - - persist_to_workspace: - root: . - paths: - - auth_ui_unit_tests_coverage.xml - - auth_ui_unit_tests_coverage - litellm_router_testing: # Runs all tests with the "router" keyword docker: - image: cimg/python:3.11 @@ -657,22 +570,32 @@ jobs: username: ${DOCKERHUB_USERNAME} password: ${DOCKERHUB_PASSWORD} working_directory: ~/project + resource_class: large + parallelism: 4 steps: - checkout - setup_google_dns + - restore_cache: + keys: + - v1-router-testing-deps-{{ checksum "requirements.txt" }} - run: name: Install Dependencies command: | - python -m pip install --upgrade pip - python -m pip install -r requirements.txt + python -m pip install --upgrade pip uv + uv pip install --system -r requirements.txt pip install "pytest==7.3.1" pip install "respx==0.22.0" - pip install "pytest-cov==5.0.0" pip install "pytest-retry==1.6.3" pip install "pytest-asyncio==0.21.1" pip install "pytest-xdist==3.6.1" + pip install "pytest-timeout==2.2.0" pip install semantic_router --no-deps pip install aurelio_sdk --no-deps + - save_cache: + paths: + - /home/circleci/.pyenv + - /home/circleci/.local + key: v1-router-testing-deps-{{ checksum "requirements.txt" }} # Run pytest and generate JUnit XML report - setup_litellm_enterprise_pip - run: @@ -680,23 +603,23 @@ jobs: command: | pwd ls - python -m pytest tests/local_testing --cov=litellm --cov-report=xml -vv -k "router" -v --junitxml=test-results/junit.xml --durations=5 - no_output_timeout: 120m - - run: - name: Rename the coverage files - command: | - mv coverage.xml litellm_router_coverage.xml - mv .coverage litellm_router_coverage + TEST_FILES=$(circleci tests glob "tests/local_testing/**/test_*.py") + + echo "$TEST_FILES" | circleci tests run \ + --split-by=timings \ + --verbose \ + --command="xargs python -m pytest \ + -v \ + -k 'router' \ + -n 4 \ + --junitxml=test-results/junit.xml \ + --durations=5 \ + --timeout=300 --timeout_method=thread" + no_output_timeout: 15m # Store test results - store_test_results: path: test-results - - persist_to_workspace: - root: . - paths: - - litellm_router_coverage.xml - - litellm_router_coverage - litellm_router_unit_testing: # Runs all tests with the "router" keyword docker: - image: cimg/python:3.11 @@ -704,23 +627,31 @@ jobs: username: ${DOCKERHUB_USERNAME} password: ${DOCKERHUB_PASSWORD} working_directory: ~/project + resource_class: large steps: - checkout - setup_google_dns + - restore_cache: + keys: + - v1-router-unit-deps-{{ checksum "requirements.txt" }} - run: name: Install Dependencies command: | - python -m pip install --upgrade pip - python -m pip install -r requirements.txt + python -m pip install --upgrade pip uv + uv pip install --system -r requirements.txt pip install "pytest==7.3.1" pip install "respx==0.22.0" - pip install "pytest-cov==5.0.0" pip install "pytest-retry==1.6.3" pip install "pytest-asyncio==0.21.1" pip install semantic_router --no-deps pip install aurelio_sdk --no-deps pip install "pytest-xdist==3.6.1" + - save_cache: + paths: + - /home/circleci/.pyenv + - /home/circleci/.local + key: v1-router-unit-deps-{{ checksum "requirements.txt" }} # Run pytest and generate JUnit XML report - setup_litellm_enterprise_pip - run: @@ -728,27 +659,26 @@ jobs: command: | pwd ls - python -m pytest -vv tests/router_unit_tests --cov=litellm --cov-report=xml -x -s --junitxml=test-results/junit.xml --durations=5 - no_output_timeout: 120m - - run: - name: Rename the coverage files - command: | - mv coverage.xml litellm_router_unit_coverage.xml - mv .coverage litellm_router_unit_coverage + python -m pytest -v tests/router_unit_tests -x --junitxml=test-results/junit.xml --durations=5 -n 4 + no_output_timeout: 15m # Store test results - store_test_results: path: test-results - - - persist_to_workspace: - root: . - paths: - - litellm_router_unit_coverage.xml - - litellm_router_unit_coverage litellm_security_tests: - machine: - image: ubuntu-2204:2023.10.1 + docker: + - image: cimg/python:3.13 + auth: + username: ${DOCKERHUB_USERNAME} + password: ${DOCKERHUB_PASSWORD} + - image: cimg/postgres:14.0 + environment: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + POSTGRES_DB: circle_test resource_class: xlarge working_directory: ~/project + environment: + DATABASE_URL: "postgresql://postgres:postgres@localhost:5432/circle_test" steps: - checkout - setup_google_dns @@ -756,87 +686,33 @@ jobs: name: Show git commit hash command: | echo "Git commit hash: $CIRCLE_SHA1" - - run: - name: Install Docker CLI (In case it's not already installed) - command: | - curl -fsSL https://get.docker.com | sh - sudo usermod -aG docker $USER - docker version - - run: - name: Install Python 3.13 - command: | - curl https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh --output miniconda.sh - bash miniconda.sh -b -p $HOME/miniconda - export PATH="$HOME/miniconda/bin:$PATH" - conda init bash - source ~/.bashrc - conda create -n myenv python=3.13 -y - conda activate myenv - python --version + - setup_remote_docker: + docker_layer_caching: true + - restore_cache: + keys: + - v3-litellm-uv-deps-{{ checksum "requirements.txt" }}-{{ checksum ".circleci/config.yml" }} - run: name: Install Dependencies command: | - export PATH="$HOME/miniconda/bin:$PATH" - source $HOME/miniconda/etc/profile.d/conda.sh - conda activate myenv - python --version - which python - pip install --upgrade typing-extensions>=4.12.0 - pip install "pytest==7.3.1" - pip install "pytest-asyncio==0.21.1" - pip install aiohttp - python -m pip install --upgrade pip - python -m pip install -r requirements.txt - pip install "pytest==7.3.1" - pip install "pytest-retry==1.6.3" - pip install "pytest-mock==3.12.0" - pip install "pytest-asyncio==0.21.1" - pip install "mypy==1.18.2" - pip install "google-generativeai==0.3.2" - pip install "google-cloud-aiplatform==1.43.0" - pip install pyarrow - pip install "boto3==1.36.0" - pip install "aioboto3==13.4.0" - pip install langchain - pip install "langfuse>=2.0.0" - pip install "logfire==0.29.0" - pip install numpydoc - pip install prisma - pip install fastapi - pip install jsonschema - pip install "httpx==0.24.1" - pip install "gunicorn==21.2.0" - pip install "anyio==3.7.1" - pip install "aiodynamo==23.10.1" - pip install "asyncio==3.4.3" - pip install "PyGithub==1.59.1" - pip install "openai==1.100.1" - pip install "pytest-cov==5.0.0" - pip install "apscheduler" + python -m pip install --upgrade pip uv + uv pip install --system -r requirements.txt + pip install "pytest==7.3.1" "pytest-retry==1.6.3" "pytest-mock==3.12.0" \ + "pytest-asyncio==0.21.1" "pytest-cov==5.0.0" + - save_cache: + paths: + - ~/.local/lib + - ~/.local/bin + - ~/.cache/uv + key: v3-litellm-uv-deps-{{ checksum "requirements.txt" }}-{{ checksum ".circleci/config.yml" }} - run: name: Install dockerize command: | wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz rm dockerize-linux-amd64-v0.6.1.tar.gz - - run: - name: Start PostgreSQL Database - command: | - docker run -d \ - --name postgres-db \ - -e POSTGRES_USER=postgres \ - -e POSTGRES_PASSWORD=postgres \ - -e POSTGRES_DB=circle_test \ - -p 5432:5432 \ - postgres:14 - run: name: Wait for PostgreSQL to be ready command: dockerize -wait tcp://localhost:5432 -timeout 1m - - run: - name: Set DATABASE_URL environment variable - command: | - echo 'export DATABASE_URL="postgresql://postgres:postgres@localhost:5432/circle_test"' >> $BASH_ENV - source $BASH_ENV - run: name: Run Security Scans command: | @@ -845,9 +721,6 @@ jobs: - run: name: Run prisma ./docker/entrypoint.sh command: | - export PATH="$HOME/miniconda/bin:$PATH" - source $HOME/miniconda/etc/profile.d/conda.sh - conda activate myenv set +e chmod +x docker/entrypoint.sh ./docker/entrypoint.sh @@ -856,26 +729,11 @@ jobs: - run: name: Run tests command: | - export PATH="$HOME/miniconda/bin:$PATH" - source $HOME/miniconda/etc/profile.d/conda.sh - conda activate myenv - pwd - ls - python -m pytest tests/proxy_security_tests --cov=litellm --cov-report=xml -vv -x -v --junitxml=test-results/junit.xml --durations=5 - no_output_timeout: 120m - - run: - name: Rename the coverage files - command: | - mv coverage.xml litellm_security_tests_coverage.xml - mv .coverage litellm_security_tests_coverage + python -m pytest tests/proxy_security_tests -v -x --junitxml=test-results/junit.xml --durations=5 + no_output_timeout: 15m # Store test results - store_test_results: path: test-results - - persist_to_workspace: - root: . - paths: - - litellm_security_tests_coverage.xml - - litellm_security_tests_coverage # Split proxy unit tests into 3 jobs for faster execution and better debugging # test_key_generate_prisma runs separately without parallel execution to avoid event loop issues with logging worker litellm_proxy_unit_testing_key_generation: @@ -885,7 +743,7 @@ jobs: username: ${DOCKERHUB_USERNAME} password: ${DOCKERHUB_PASSWORD} working_directory: ~/project - resource_class: large + resource_class: medium steps: - checkout - setup_google_dns @@ -971,7 +829,7 @@ jobs: ls # Run without -n flag to avoid pytest-xdist event loop conflicts with logging worker python -m pytest tests/proxy_unit_tests/test_key_generate_prisma.py --cov=litellm --cov-report=xml --junitxml=test-results/junit-key-generation.xml --durations=10 --timeout=300 -vv --log-cli-level=INFO - no_output_timeout: 120m + no_output_timeout: 15m - run: name: Rename the coverage files command: | @@ -991,7 +849,7 @@ jobs: username: ${DOCKERHUB_USERNAME} password: ${DOCKERHUB_PASSWORD} working_directory: ~/project - resource_class: large + resource_class: xlarge steps: - checkout - setup_google_dns @@ -1072,25 +930,14 @@ jobs: ./docker/entrypoint.sh set -e - run: - name: Run proxy unit tests (part 1 - auth checks only, key generation in separate job) + name: Run proxy unit tests (part 1 - auth checks) command: | pwd ls - # Run auth tests with parallel execution (test_key_generate_prisma moved to separate job to avoid event loop issues) - python -m pytest tests/proxy_unit_tests/test_auth_checks.py tests/proxy_unit_tests/test_user_api_key_auth.py --cov=litellm --cov-report=xml --junitxml=test-results/junit-part1.xml --durations=10 -n 8 --timeout=300 -vv --log-cli-level=INFO - no_output_timeout: 120m - - run: - name: Rename the coverage files - command: | - mv coverage.xml litellm_proxy_unit_tests_part1_coverage.xml - mv .coverage litellm_proxy_unit_tests_part1_coverage + python -m pytest tests/proxy_unit_tests/test_auth_checks.py tests/proxy_unit_tests/test_user_api_key_auth.py --junitxml=test-results/junit-part1.xml --durations=10 -n 8 --timeout=300 -v + no_output_timeout: 15m - store_test_results: path: test-results - - persist_to_workspace: - root: . - paths: - - litellm_proxy_unit_tests_part1_coverage.xml - - litellm_proxy_unit_tests_part1_coverage litellm_proxy_unit_testing_part2: docker: - image: cimg/python:3.11 @@ -1098,7 +945,7 @@ jobs: username: ${DOCKERHUB_USERNAME} password: ${DOCKERHUB_PASSWORD} working_directory: ~/project - resource_class: large + resource_class: xlarge steps: - checkout - setup_google_dns @@ -1183,20 +1030,10 @@ jobs: command: | pwd ls - python -m pytest tests/proxy_unit_tests --ignore=tests/proxy_unit_tests/test_key_generate_prisma.py --ignore=tests/proxy_unit_tests/test_auth_checks.py --ignore=tests/proxy_unit_tests/test_user_api_key_auth.py --cov=litellm --cov-report=xml --junitxml=test-results/junit-part2.xml --durations=10 -n 4 --timeout=300 -vv --log-cli-level=INFO - no_output_timeout: 120m - - run: - name: Rename the coverage files - command: | - mv coverage.xml litellm_proxy_unit_tests_part2_coverage.xml - mv .coverage litellm_proxy_unit_tests_part2_coverage + python -m pytest tests/proxy_unit_tests --ignore=tests/proxy_unit_tests/test_key_generate_prisma.py --ignore=tests/proxy_unit_tests/test_auth_checks.py --ignore=tests/proxy_unit_tests/test_user_api_key_auth.py --junitxml=test-results/junit-part2.xml --durations=10 -n 8 --timeout=300 -v + no_output_timeout: 15m - store_test_results: path: test-results - - persist_to_workspace: - root: . - paths: - - litellm_proxy_unit_tests_part2_coverage.xml - - litellm_proxy_unit_tests_part2_coverage litellm_assistants_api_testing: # Runs all tests with the "assistants" keyword docker: - image: cimg/python:3.13.1 @@ -1204,6 +1041,7 @@ jobs: username: ${DOCKERHUB_USERNAME} password: ${DOCKERHUB_PASSWORD} working_directory: ~/project + resource_class: medium steps: - checkout @@ -1211,15 +1049,13 @@ jobs: - run: name: Install Dependencies command: | - python -m pip install --upgrade pip - pip install wheel - pip install --upgrade pip wheel setuptools - python -m pip install -r requirements.txt + python -m pip install --upgrade pip uv + pip install wheel setuptools + uv pip install --system -r requirements.txt pip install "pytest==7.3.1" pip install "respx==0.22.0" pip install "pytest-retry==1.6.3" pip install "pytest-asyncio==0.21.1" - pip install "pytest-cov==5.0.0" # Run pytest and generate JUnit XML report - setup_litellm_enterprise_pip - run: @@ -1227,21 +1063,11 @@ jobs: command: | pwd ls - python -m pytest tests/local_testing/ -vv -k "assistants" --cov=litellm --cov-report=xml -x -s -v --junitxml=test-results/junit.xml --durations=5 - no_output_timeout: 120m - - run: - name: Rename the coverage files - command: | - mv coverage.xml litellm_assistants_api_coverage.xml - mv .coverage litellm_assistants_api_coverage + python -m pytest tests/local_testing/ -v -k "assistants" -x --junitxml=test-results/junit.xml --durations=5 + no_output_timeout: 15m # Store test results - store_test_results: path: test-results - - persist_to_workspace: - root: . - paths: - - litellm_assistants_api_coverage.xml - - litellm_assistants_api_coverage llm_translation_testing: docker: - image: cimg/python:3.11 @@ -1249,22 +1075,30 @@ jobs: username: ${DOCKERHUB_USERNAME} password: ${DOCKERHUB_PASSWORD} working_directory: ~/project + resource_class: large steps: - checkout - setup_google_dns + - restore_cache: + keys: + - v1-llm-translation-deps-{{ checksum "requirements.txt" }} - run: name: Install Dependencies command: | - python -m pip install --upgrade pip - python -m pip install -r requirements.txt + python -m pip install --upgrade pip uv + uv pip install --system -r requirements.txt pip install "pytest==7.3.1" pip install "pytest-retry==1.6.3" - pip install "pytest-cov==5.0.0" pip install "pytest-asyncio==0.21.1" pip install "respx==0.22.0" pip install "pytest-xdist==3.6.1" pip install "pytest-timeout==2.2.0" + - save_cache: + paths: + - /home/circleci/.pyenv + - /home/circleci/.local + key: v1-llm-translation-deps-{{ checksum "requirements.txt" }} # Run pytest and generate JUnit XML report - run: name: Run tests @@ -1281,22 +1115,12 @@ jobs: for dir in "${IGNORE_DIRS[@]}"; do IGNORE_ARGS="$IGNORE_ARGS --ignore=$dir" done - python -m pytest -vv tests/llm_translation $IGNORE_ARGS --cov=litellm --cov-report=xml -v --junitxml=test-results/junit.xml --durations=20 -n 4 --timeout=120 --timeout_method=thread - no_output_timeout: 120m - - run: - name: Rename the coverage files - command: | - mv coverage.xml llm_translation_coverage.xml - mv .coverage llm_translation_coverage + python -m pytest -v tests/llm_translation $IGNORE_ARGS --junitxml=test-results/junit.xml --durations=20 -n 8 --timeout=120 --timeout_method=thread --retries 2 --retry-delay 5 + no_output_timeout: 15m # Store test results - store_test_results: path: test-results - - persist_to_workspace: - root: . - paths: - - llm_translation_coverage.xml - - llm_translation_coverage realtime_translation_testing: docker: - image: cimg/python:3.11 @@ -1311,16 +1135,9 @@ jobs: - run: name: Install Dependencies command: | - python -m pip install --upgrade pip - python -m pip install -r requirements.txt - pip install "pytest==7.3.1" - pip install "pytest-retry==1.6.3" - pip install "pytest-cov==5.0.0" - pip install "pytest-asyncio==0.21.1" - pip install "respx==0.22.0" - pip install "pytest-xdist==3.6.1" - pip install "pytest-timeout==2.2.0" - pip install "websockets" + python -m pip install --upgrade pip uv + uv pip install --system -r requirements.txt + pip install "pytest==7.3.1" "pytest-retry==1.6.3" "pytest-cov==5.0.0" "pytest-asyncio==0.21.1" "respx==0.22.0" "pytest-xdist==3.6.1" "pytest-timeout==2.2.0" "websockets" # Run pytest and generate JUnit XML report - run: name: Run realtime tests @@ -1330,7 +1147,7 @@ jobs: # Add --timeout to kill hanging tests after 120s (2 min) # Add --durations=20 to show 20 slowest tests for debugging python -m pytest -vv tests/llm_translation/realtime --cov=litellm --cov-report=xml -v --junitxml=test-results/junit.xml --durations=20 -n 4 --timeout=120 --timeout_method=thread - no_output_timeout: 120m + no_output_timeout: 15m - run: name: Rename the coverage files command: | @@ -1359,8 +1176,8 @@ jobs: - run: name: Install Dependencies command: | - python -m pip install --upgrade pip - python -m pip install -r requirements.txt + python -m pip install --upgrade pip uv + uv pip install --system -r requirements.txt pip install "pytest==7.3.1" pip install "pytest-retry==1.6.3" pip install "pytest-cov==5.0.0" @@ -1368,14 +1185,15 @@ jobs: pip install "respx==0.22.0" pip install "pydantic==2.11.0" pip install "mcp==1.25.0" + pip install "pytest-xdist==3.6.1" # Run pytest and generate JUnit XML report - run: name: Run tests command: | pwd ls - python -m pytest -vv tests/mcp_tests --cov=litellm --cov-report=xml -x -s -v --junitxml=test-results/junit.xml --durations=5 - no_output_timeout: 120m + python -m pytest -vv tests/mcp_tests --cov=litellm --cov-report=xml -x -s -v --junitxml=test-results/junit.xml --durations=5 -n 2 + no_output_timeout: 15m - run: name: Rename the coverage files command: | @@ -1404,8 +1222,8 @@ jobs: - run: name: Install Dependencies command: | - python -m pip install --upgrade pip - python -m pip install -r requirements.txt + python -m pip install --upgrade pip uv + uv pip install --system -r requirements.txt pip install "pytest==7.3.1" pip install "pytest-retry==1.6.3" pip install "pytest-cov==5.0.0" @@ -1420,7 +1238,7 @@ jobs: pwd ls python -m pytest -vv tests/agent_tests --ignore=tests/agent_tests/local_only_agent_tests --cov=litellm --cov-report=xml -x -s -v --junitxml=test-results/junit.xml --durations=5 - no_output_timeout: 120m + no_output_timeout: 15m - run: name: Rename the coverage files command: | @@ -1449,8 +1267,8 @@ jobs: - run: name: Install Dependencies command: | - python -m pip install --upgrade pip - python -m pip install -r requirements.txt + python -m pip install --upgrade pip uv + uv pip install --system -r requirements.txt pip install "pytest==7.3.1" pip install "pytest-retry==1.6.3" pip install "pytest-cov==5.0.0" @@ -1458,15 +1276,18 @@ jobs: pip install "respx==0.22.0" pip install "pydantic==2.10.2" pip install "boto3==1.36.0" - pip install "semantic_router==0.1.10" + pip install "semantic_router==0.1.10" --no-deps + pip install aurelio_sdk + pip install "pytest-xdist==3.6.1" + pip install "pytest-timeout==2.2.0" # Run pytest and generate JUnit XML report - run: name: Run tests command: | pwd ls - python -m pytest -vv tests/guardrails_tests --cov=litellm --cov-report=xml -x -s -v --junitxml=test-results/junit.xml --durations=5 - no_output_timeout: 120m + LITELLM_LOG=WARNING python -m pytest tests/guardrails_tests -vv --cov=litellm --cov-report=xml --junitxml=test-results/junit.xml --durations=5 -n 2 --timeout=120 --timeout_method=thread + no_output_timeout: 15m - run: name: Rename the coverage files command: | @@ -1496,8 +1317,8 @@ jobs: - run: name: Install Dependencies command: | - python -m pip install --upgrade pip - python -m pip install -r requirements.txt + python -m pip install --upgrade pip uv + uv pip install --system -r requirements.txt pip install "pytest==7.3.1" pip install "pytest-retry==1.6.3" pip install "pytest-cov==5.0.0" @@ -1510,8 +1331,8 @@ jobs: command: | pwd ls - python -m pytest -vv tests/unified_google_tests --cov=litellm --cov-report=xml -x -s -v --junitxml=test-results/junit.xml --durations=5 - no_output_timeout: 120m + python -m pytest -vv tests/unified_google_tests --cov=litellm --cov-report=xml -x -s -v --junitxml=test-results/junit.xml --durations=5 --retries 3 --retry-delay 5 + no_output_timeout: 15m - run: name: Rename the coverage files command: | @@ -1534,42 +1355,41 @@ jobs: username: ${DOCKERHUB_USERNAME} password: ${DOCKERHUB_PASSWORD} working_directory: ~/project + resource_class: large steps: - checkout - setup_google_dns + - restore_cache: + keys: + - v1-llm-responses-deps-{{ checksum "requirements.txt" }} - run: name: Install Dependencies command: | - python -m pip install --upgrade pip - python -m pip install -r requirements.txt + python -m pip install --upgrade pip uv + uv pip install --system -r requirements.txt pip install "pytest==7.3.1" pip install "pytest-retry==1.6.3" - pip install "pytest-cov==5.0.0" pip install "pytest-asyncio==0.21.1" pip install "respx==0.22.0" + pip install "pytest-xdist==3.6.1" + - save_cache: + paths: + - /home/circleci/.pyenv + - /home/circleci/.local + key: v1-llm-responses-deps-{{ checksum "requirements.txt" }} # Run pytest and generate JUnit XML report - run: name: Run tests command: | pwd ls - python -m pytest -vv tests/llm_responses_api_testing --cov=litellm --cov-report=xml -x -s -v --junitxml=test-results/junit.xml --durations=5 - no_output_timeout: 120m - - run: - name: Rename the coverage files - command: | - mv coverage.xml llm_responses_api_coverage.xml - mv .coverage llm_responses_api_coverage + python -m pytest -v tests/llm_responses_api_testing -x --junitxml=test-results/junit.xml --durations=5 -n 8 + no_output_timeout: 15m # Store test results - store_test_results: path: test-results - - persist_to_workspace: - root: . - paths: - - llm_responses_api_coverage.xml - - llm_responses_api_coverage ocr_testing: docker: - image: cimg/python:3.11 @@ -1584,21 +1404,17 @@ jobs: - run: name: Install Dependencies command: | - python -m pip install --upgrade pip - python -m pip install -r requirements.txt - pip install "pytest==7.3.1" - pip install "pytest-retry==1.6.3" - pip install "pytest-cov==5.0.0" - pip install "pytest-asyncio==0.21.1" - pip install "respx==0.22.0" + python -m pip install --upgrade pip uv + uv pip install --system -r requirements.txt + pip install "pytest==7.3.1" "pytest-retry==1.6.3" "pytest-cov==5.0.0" "pytest-asyncio==0.21.1" "respx==0.22.0" "pytest-xdist==3.6.1" # Run pytest and generate JUnit XML report - run: name: Run tests command: | pwd ls - python -m pytest -vv tests/ocr_tests --cov=litellm --cov-report=xml -x -s -v --junitxml=test-results/junit.xml --durations=5 - no_output_timeout: 120m + python -m pytest -vv tests/ocr_tests --cov=litellm --cov-report=xml -x -v --junitxml=test-results/junit.xml --durations=5 -n 4 + no_output_timeout: 15m - run: name: Rename the coverage files command: | @@ -1627,21 +1443,17 @@ jobs: - run: name: Install Dependencies command: | - python -m pip install --upgrade pip - python -m pip install -r requirements.txt - pip install "pytest==7.3.1" - pip install "pytest-retry==1.6.3" - pip install "pytest-cov==5.0.0" - pip install "pytest-asyncio==0.21.1" - pip install "respx==0.22.0" + python -m pip install --upgrade pip uv + uv pip install --system -r requirements.txt + pip install "pytest==7.3.1" "pytest-retry==1.6.3" "pytest-cov==5.0.0" "pytest-asyncio==0.21.1" "respx==0.22.0" "pytest-xdist==3.6.1" # Run pytest and generate JUnit XML report - run: name: Run tests command: | pwd ls - python -m pytest -vv tests/search_tests --cov=litellm --cov-report=xml -x -s -v --junitxml=test-results/junit.xml --durations=5 - no_output_timeout: 120m + python -m pytest -vv tests/search_tests --cov=litellm --cov-report=xml -x -v --junitxml=test-results/junit.xml --durations=5 -n 4 + no_output_timeout: 15m - run: name: Rename the coverage files command: | @@ -1656,7 +1468,7 @@ jobs: paths: - search_coverage.xml - search_coverage - # Split litellm_mapped_tests into 3 parallel jobs for 3x faster execution + # Split litellm_mapped_tests into parallel jobs litellm_mapped_tests_proxy_part1: docker: - image: cimg/python:3.11 @@ -1664,7 +1476,7 @@ jobs: username: ${DOCKERHUB_USERNAME} password: ${DOCKERHUB_PASSWORD} working_directory: ~/project - resource_class: xlarge + resource_class: large steps: - setup_litellm_test_deps - run: @@ -1672,20 +1484,10 @@ jobs: command: | prisma generate export PYTHONUNBUFFERED=1 - python -m pytest tests/test_litellm/proxy/guardrails tests/test_litellm/proxy/management_endpoints tests/test_litellm/proxy/_experimental tests/test_litellm/proxy/client tests/test_litellm/proxy/auth --cov=litellm --cov-report=xml --junitxml=test-results/junit-proxy-part1.xml --durations=10 -n 8 --maxfail=5 --timeout=60 -vv --log-cli-level=WARNING -r A - no_output_timeout: 60m - - run: - name: Rename the coverage files - command: | - mv coverage.xml litellm_proxy_tests_part1_coverage.xml - mv .coverage litellm_proxy_tests_part1_coverage + python -m pytest tests/test_litellm/proxy/guardrails tests/test_litellm/proxy/management_endpoints tests/test_litellm/proxy/_experimental tests/test_litellm/proxy/client tests/test_litellm/proxy/auth --junitxml=test-results/junit-proxy-part1.xml --durations=10 -n 4 --maxfail=5 --timeout=60 -vv --log-cli-level=WARNING -r A + no_output_timeout: 15m - store_test_results: path: test-results - - persist_to_workspace: - root: . - paths: - - litellm_proxy_tests_part1_coverage.xml - - litellm_proxy_tests_part1_coverage litellm_mapped_tests_proxy_part2: docker: - image: cimg/python:3.11 @@ -1693,7 +1495,7 @@ jobs: username: ${DOCKERHUB_USERNAME} password: ${DOCKERHUB_PASSWORD} working_directory: ~/project - resource_class: xlarge + resource_class: large steps: - setup_litellm_test_deps - run: @@ -1701,20 +1503,10 @@ jobs: command: | prisma generate export PYTHONUNBUFFERED=1 - python -m pytest tests/test_litellm/proxy --ignore=tests/test_litellm/proxy/guardrails --ignore=tests/test_litellm/proxy/management_endpoints --ignore=tests/test_litellm/proxy/_experimental --ignore=tests/test_litellm/proxy/client --ignore=tests/test_litellm/proxy/auth --cov=litellm --cov-report=xml --junitxml=test-results/junit-proxy-part2.xml --durations=10 -n 4 --maxfail=5 --timeout=120 -vv --log-cli-level=WARNING -r A - no_output_timeout: 60m - - run: - name: Rename the coverage files - command: | - mv coverage.xml litellm_proxy_tests_part2_coverage.xml - mv .coverage litellm_proxy_tests_part2_coverage + python -m pytest tests/test_litellm/proxy --ignore=tests/test_litellm/proxy/guardrails --ignore=tests/test_litellm/proxy/management_endpoints --ignore=tests/test_litellm/proxy/_experimental --ignore=tests/test_litellm/proxy/client --ignore=tests/test_litellm/proxy/auth --junitxml=test-results/junit-proxy-part2.xml --durations=10 -n 4 --maxfail=5 --timeout=120 -vv --log-cli-level=WARNING -r A + no_output_timeout: 15m - store_test_results: path: test-results - - persist_to_workspace: - root: . - paths: - - litellm_proxy_tests_part2_coverage.xml - - litellm_proxy_tests_part2_coverage litellm_mapped_tests_llms: docker: - image: cimg/python:3.11 @@ -1722,26 +1514,16 @@ jobs: username: ${DOCKERHUB_USERNAME} password: ${DOCKERHUB_PASSWORD} working_directory: ~/project - resource_class: xlarge + resource_class: large steps: - setup_litellm_test_deps - run: name: Run LLM provider tests command: | - python -m pytest tests/test_litellm/llms --cov=litellm --cov-report=xml --junitxml=test-results/junit-llms.xml --durations=10 -n 16 --maxfail=5 --timeout=300 -vv --log-cli-level=WARNING - no_output_timeout: 120m - - run: - name: Rename the coverage files - command: | - mv coverage.xml litellm_llms_tests_coverage.xml - mv .coverage litellm_llms_tests_coverage + python -m pytest tests/test_litellm/llms --junitxml=test-results/junit-llms.xml --durations=10 -n 4 --maxfail=5 --timeout=300 -vv --log-cli-level=WARNING + no_output_timeout: 15m - store_test_results: path: test-results - - persist_to_workspace: - root: . - paths: - - litellm_llms_tests_coverage.xml - - litellm_llms_tests_coverage litellm_mapped_tests_core: docker: - image: cimg/python:3.11 @@ -1749,26 +1531,16 @@ jobs: username: ${DOCKERHUB_USERNAME} password: ${DOCKERHUB_PASSWORD} working_directory: ~/project - resource_class: xlarge + resource_class: large steps: - setup_litellm_test_deps - run: name: Run core tests command: | - python -m pytest tests/test_litellm --ignore=tests/test_litellm/proxy --ignore=tests/test_litellm/llms --ignore=tests/test_litellm/integrations --ignore=tests/test_litellm/litellm_core_utils --ignore=tests/test_litellm/experimental_mcp_client --cov=litellm --cov-report=xml --junitxml=test-results/junit-core.xml --durations=10 -n 16 --maxfail=5 --timeout=300 -vv --log-cli-level=WARNING - no_output_timeout: 120m - - run: - name: Rename the coverage files - command: | - mv coverage.xml litellm_core_tests_coverage.xml - mv .coverage litellm_core_tests_coverage + python -m pytest tests/test_litellm --ignore=tests/test_litellm/proxy --ignore=tests/test_litellm/llms --ignore=tests/test_litellm/integrations --ignore=tests/test_litellm/litellm_core_utils --ignore=tests/test_litellm/experimental_mcp_client --junitxml=test-results/junit-core.xml --durations=10 -n 4 --maxfail=5 --timeout=300 -vv --log-cli-level=WARNING + no_output_timeout: 15m - store_test_results: path: test-results - - persist_to_workspace: - root: . - paths: - - litellm_core_tests_coverage.xml - - litellm_core_tests_coverage litellm_mapped_tests_litellm_core_utils: docker: - image: cimg/python:3.11 @@ -1776,26 +1548,16 @@ jobs: username: ${DOCKERHUB_USERNAME} password: ${DOCKERHUB_PASSWORD} working_directory: ~/project - resource_class: xlarge + resource_class: large steps: - setup_litellm_test_deps - run: name: Run litellm_core_utils tests command: | - python -m pytest tests/test_litellm/litellm_core_utils --cov=litellm --cov-report=xml --junitxml=test-results/junit-litellm-core-utils.xml --durations=10 -n 16 --maxfail=5 --timeout=300 -vv --log-cli-level=WARNING - no_output_timeout: 120m - - run: - name: Rename the coverage files - command: | - mv coverage.xml litellm_core_utils_tests_coverage.xml - mv .coverage litellm_core_utils_tests_coverage + python -m pytest tests/test_litellm/litellm_core_utils --junitxml=test-results/junit-litellm-core-utils.xml --durations=10 -n 4 --maxfail=5 --timeout=300 -vv --log-cli-level=WARNING + no_output_timeout: 15m - store_test_results: path: test-results - - persist_to_workspace: - root: . - paths: - - litellm_core_utils_tests_coverage.xml - - litellm_core_utils_tests_coverage litellm_mapped_tests_mcps: docker: - image: cimg/python:3.11 @@ -1803,14 +1565,14 @@ jobs: username: ${DOCKERHUB_USERNAME} password: ${DOCKERHUB_PASSWORD} working_directory: ~/project - resource_class: xlarge + resource_class: medium steps: - setup_litellm_test_deps - run: name: Run MCP client tests command: | - python -m pytest tests/test_litellm/experimental_mcp_client --cov=litellm --cov-report=xml --junitxml=test-results/junit-mcps.xml --durations=10 -n 4 --maxfail=5 --timeout=300 -vv --log-cli-level=WARNING - no_output_timeout: 120m + python -m pytest tests/test_litellm/experimental_mcp_client --cov=litellm --cov-report=xml --junitxml=test-results/junit-mcps.xml --durations=10 -n 2 --maxfail=5 --timeout=300 -vv --log-cli-level=WARNING + no_output_timeout: 15m - run: name: Rename the coverage files command: | @@ -1830,26 +1592,16 @@ jobs: username: ${DOCKERHUB_USERNAME} password: ${DOCKERHUB_PASSWORD} working_directory: ~/project - resource_class: xlarge + resource_class: large steps: - setup_litellm_test_deps - run: name: Run integrations tests command: | - python -m pytest tests/test_litellm/integrations --cov=litellm --cov-report=xml --junitxml=test-results/junit-integrations.xml --durations=10 -n 16 --maxfail=5 --timeout=300 -vv --log-cli-level=WARNING - no_output_timeout: 120m - - run: - name: Rename the coverage files - command: | - mv coverage.xml litellm_integrations_tests_coverage.xml - mv .coverage litellm_integrations_tests_coverage + python -m pytest tests/test_litellm/integrations --junitxml=test-results/junit-integrations.xml --durations=10 -n 4 --maxfail=5 --timeout=300 -vv --log-cli-level=WARNING + no_output_timeout: 15m - store_test_results: path: test-results - - persist_to_workspace: - root: . - paths: - - litellm_integrations_tests_coverage.xml - - litellm_integrations_tests_coverage litellm_mapped_enterprise_tests: docker: - image: cimg/python:3.11 @@ -1857,6 +1609,7 @@ jobs: username: ${DOCKERHUB_USERNAME} password: ${DOCKERHUB_PASSWORD} working_directory: ~/project + resource_class: large steps: - checkout @@ -1864,8 +1617,8 @@ jobs: - run: name: Install Dependencies command: | - python -m pip install --upgrade pip - python -m pip install -r requirements.txt + python -m pip install --upgrade pip uv + uv pip install --system -r requirements.txt pip install "pytest-mock==3.12.0" pip install "pytest==7.3.1" pip install "pytest-retry==1.6.3" @@ -1878,7 +1631,8 @@ jobs: pip install "requests-mock>=1.12.1" pip install "responses==0.25.7" pip install "pytest-xdist==3.6.1" - pip install "semantic_router==0.1.10" + pip install "semantic_router==0.1.10" --no-deps + pip install aurelio_sdk pip install "fastapi-offline==1.7.3" - setup_litellm_enterprise_pip - run: @@ -1887,22 +1641,11 @@ jobs: pwd ls prisma generate - python -m pytest -vv tests/enterprise --cov=litellm --cov-report=xml -x -s -v --junitxml=test-results/junit-enterprise.xml --durations=10 -n 8 - no_output_timeout: 120m - - run: - name: Rename the coverage files - command: | - mv coverage.xml litellm_mapped_tests_coverage.xml - mv .coverage litellm_mapped_tests_coverage - + python -m pytest -v tests/enterprise -x --junitxml=test-results/junit-enterprise.xml --durations=10 -n 4 + no_output_timeout: 15m # Store test results - store_test_results: path: test-results - - persist_to_workspace: - root: . - paths: - - litellm_mapped_tests_coverage.xml - - litellm_mapped_tests_coverage batches_testing: docker: - image: cimg/python:3.11 @@ -1917,8 +1660,8 @@ jobs: - run: name: Install Dependencies command: | - python -m pip install --upgrade pip - python -m pip install -r requirements.txt + python -m pip install --upgrade pip uv + uv pip install --system -r requirements.txt pip install "respx==0.22.0" pip install "pytest==7.3.1" pip install "pytest-retry==1.6.3" @@ -1926,14 +1669,15 @@ jobs: pip install "pytest-cov==5.0.0" pip install "google-generativeai==0.3.2" pip install "google-cloud-aiplatform==1.43.0" + pip install "pytest-xdist==3.6.1" # Run pytest and generate JUnit XML report - run: name: Run tests command: | pwd ls - python -m pytest -vv tests/batches_tests --cov=litellm --cov-report=xml -x -s -v --junitxml=test-results/junit.xml --durations=5 - no_output_timeout: 120m + python -m pytest -vv tests/batches_tests --cov=litellm --cov-report=xml -x -s -v --junitxml=test-results/junit.xml --durations=5 -n 2 + no_output_timeout: 15m - run: name: Rename the coverage files command: | @@ -1962,9 +1706,9 @@ jobs: - run: name: Install Dependencies command: | - python -m pip install --upgrade pip + python -m pip install --upgrade pip uv + uv pip install --system -r requirements.txt pip install numpydoc - python -m pip install -r requirements.txt pip install "respx==0.22.0" pip install "pytest==7.3.1" pip install "pytest-retry==1.6.3" @@ -1973,14 +1717,15 @@ jobs: pip install "google-generativeai==0.3.2" pip install "google-cloud-aiplatform==1.43.0" pip install pytest-mock + pip install "pytest-xdist==3.6.1" # Run pytest and generate JUnit XML report - run: name: Run tests command: | pwd ls - python -m pytest -vv tests/litellm_utils_tests --cov=litellm --cov-report=xml -x -s -v --junitxml=test-results/junit.xml --durations=5 - no_output_timeout: 120m + python -m pytest -vv tests/litellm_utils_tests --cov=litellm --cov-report=xml -x -s -v --junitxml=test-results/junit.xml --durations=5 -n 2 + no_output_timeout: 15m - run: name: Rename the coverage files command: | @@ -2010,21 +1755,17 @@ jobs: - run: name: Install Dependencies command: | - python -m pip install --upgrade pip - python -m pip install -r requirements.txt - pip install "pytest==7.3.1" - pip install "pytest-retry==1.6.3" - pip install "pytest-cov==5.0.0" - pip install "pytest-asyncio==0.21.1" - pip install "respx==0.22.0" + python -m pip install --upgrade pip uv + uv pip install --system -r requirements.txt + pip install "pytest==7.3.1" "pytest-retry==1.6.3" "pytest-cov==5.0.0" "pytest-asyncio==0.21.1" "respx==0.22.0" "pytest-xdist==3.6.1" # Run pytest and generate JUnit XML report - run: name: Run tests command: | pwd ls - python -m pytest -vv tests/pass_through_unit_tests --cov=litellm --cov-report=xml -x -s -v --junitxml=test-results/junit.xml --durations=5 - no_output_timeout: 120m + python -m pytest -vv tests/pass_through_unit_tests --cov=litellm --cov-report=xml -x -v --junitxml=test-results/junit.xml --durations=5 -n 4 + no_output_timeout: 15m - run: name: Rename the coverage files command: | @@ -2046,6 +1787,7 @@ jobs: username: ${DOCKERHUB_USERNAME} password: ${DOCKERHUB_PASSWORD} working_directory: ~/project + resource_class: large steps: - checkout @@ -2053,8 +1795,8 @@ jobs: - run: name: Install Dependencies command: | - python -m pip install --upgrade pip - python -m pip install -r requirements.txt + python -m pip install --upgrade pip uv + uv pip install --system -r requirements.txt pip install "pytest==7.3.1" pip install "pytest-retry==1.6.3" pip install "pytest-cov==5.0.0" @@ -2067,22 +1809,11 @@ jobs: command: | pwd ls - python -m pytest -vv tests/image_gen_tests -n 4 --cov=litellm --cov-report=xml -x -v --junitxml=test-results/junit.xml --durations=5 - no_output_timeout: 120m - - run: - name: Rename the coverage files - command: | - mv coverage.xml image_gen_coverage.xml - mv .coverage image_gen_coverage - + python -m pytest -v tests/image_gen_tests -n 4 -x --junitxml=test-results/junit.xml --durations=5 + no_output_timeout: 15m # Store test results - store_test_results: path: test-results - - persist_to_workspace: - root: . - paths: - - image_gen_coverage.xml - - image_gen_coverage logging_testing: docker: - image: cimg/python:3.11 @@ -2097,8 +1828,8 @@ jobs: - run: name: Install Dependencies command: | - python -m pip install --upgrade pip - python -m pip install -r requirements.txt + python -m pip install --upgrade pip uv + uv pip install --system -r requirements.txt pip install "pytest==7.3.1" pip install "pytest-retry==1.6.3" pip install "pytest-cov==5.0.0" @@ -2111,6 +1842,7 @@ jobs: pip install "anthropic==0.52.0" pip install "blockbuster==1.5.24" pip install "pytest-xdist==3.6.1" + pip install "pytest-timeout==2.2.0" # Run pytest and generate JUnit XML report - setup_litellm_enterprise_pip - run: @@ -2118,13 +1850,13 @@ jobs: command: | pwd ls - python -m pytest -vv tests/logging_callback_tests --cov=litellm -n 4 --cov-report=xml -s -v --junitxml=test-results/junit.xml --durations=5 - no_output_timeout: 120m + LITELLM_LOG=WARNING python -m pytest tests/logging_callback_tests -vv --cov=litellm --cov-report=xml -n 4 --junitxml=test-results/junit.xml --durations=5 --timeout=120 --timeout_method=thread + no_output_timeout: 15m - run: name: Rename the coverage files command: | - mv coverage.xml logging_coverage.xml - mv .coverage logging_coverage + mv coverage.xml logging_coverage.xml || true + mv .coverage logging_coverage || true # Store test results - store_test_results: @@ -2148,8 +1880,8 @@ jobs: - run: name: Install Dependencies command: | - python -m pip install --upgrade pip - python -m pip install -r requirements.txt + python -m pip install --upgrade pip uv + uv pip install --system -r requirements.txt pip install "pytest==7.3.1" pip install "pytest-retry==1.6.3" pip install "pytest-cov==5.0.0" @@ -2162,7 +1894,7 @@ jobs: pwd ls python -m pytest -vv tests/audio_tests --cov=litellm --cov-report=xml -x -s -v --junitxml=test-results/junit.xml --durations=5 - no_output_timeout: 120m + no_output_timeout: 15m - run: name: Rename the coverage files command: | @@ -2219,6 +1951,7 @@ jobs: username: ${DOCKERHUB_USERNAME} password: ${DOCKERHUB_PASSWORD} working_directory: ~/project + resource_class: medium steps: - checkout @@ -2226,9 +1959,9 @@ jobs: - run: name: Install Dependencies command: | - python -m pip install --upgrade pip - python -m pip install wheel setuptools - python -m pip install -r requirements.txt + python -m pip install --upgrade pip uv + pip install wheel setuptools + uv pip install --system -r requirements.txt pip install "pytest==7.3.1" pip install "pytest-retry==1.6.3" pip install "pytest-asyncio==0.21.1" @@ -2240,7 +1973,7 @@ jobs: command: | pwd ls - python -m pytest -vv tests/local_testing/test_basic_python_version.py + python -m pytest -v tests/local_testing/test_basic_python_version.py helm_chart_testing: machine: image: ubuntu-2204:2023.10.1 # Use machine executor instead of docker @@ -2249,6 +1982,8 @@ jobs: steps: - checkout + - attach_workspace: + at: ~/project - setup_google_dns # Install Helm - run: @@ -2279,10 +2014,11 @@ jobs: kind create cluster --name litellm-test - run: - name: Build Docker image for helm tests + name: Load Docker Database Image for helm tests command: | + zstd -d litellm-docker-database.tar.zst --stdout | docker load IMAGE_TAG=${CIRCLE_SHA1:-ci} - docker build -t litellm-ci:${IMAGE_TAG} -f docker/Dockerfile.database . + docker tag litellm-docker-database:ci litellm-ci:${IMAGE_TAG} - run: name: Load Docker image into Kind @@ -2378,7 +2114,7 @@ jobs: db_migration_disable_update_check: machine: image: ubuntu-2204:2023.10.1 - resource_class: xlarge + resource_class: medium working_directory: ~/project steps: - checkout @@ -2406,7 +2142,7 @@ jobs: - run: name: Load Docker Database Image command: | - gunzip -c litellm-docker-database.tar.gz | docker load + zstd -d litellm-docker-database.tar.zst --stdout | docker load docker images | grep litellm-docker-database - run: name: Run Docker container @@ -2452,23 +2188,19 @@ jobs: - run: name: Run Basic Proxy Startup Tests (Health Readiness and Chat Completion) command: | - python -m pytest -vv tests/basic_proxy_startup_tests -x --junitxml=test-results/junit-2.xml --durations=5 - no_output_timeout: 120m + python -m pytest -v tests/basic_proxy_startup_tests -x --junitxml=test-results/junit-2.xml --durations=5 + no_output_timeout: 15m build_and_test: machine: image: ubuntu-2204:2023.10.1 - resource_class: xlarge + resource_class: large working_directory: ~/project steps: - checkout + - attach_workspace: + at: ~/project - setup_google_dns - - run: - name: Install Docker CLI (In case it's not already installed) - command: | - curl -fsSL https://get.docker.com | sh - sudo usermod -aG docker $USER - docker version - run: name: Install Python 3.9 command: | @@ -2534,8 +2266,10 @@ jobs: name: Wait for PostgreSQL to be ready command: dockerize -wait tcp://localhost:5432 -timeout 1m - run: - name: Build Docker image - command: docker build -t my-app:latest -f ./docker/Dockerfile.database . + name: Load Docker Database Image + command: | + zstd -d litellm-docker-database.tar.zst --stdout | docker load + docker tag litellm-docker-database:ci my-app:latest - run: name: Run Docker container command: | @@ -2590,8 +2324,8 @@ jobs: command: | pwd ls - python -m pytest -s -vv tests/*.py -x --junitxml=test-results/junit.xml -n 4 --durations=5 --ignore=tests/otel_tests --ignore=tests/spend_tracking_tests --ignore=tests/pass_through_tests --ignore=tests/proxy_admin_ui_tests --ignore=tests/load_tests --ignore=tests/llm_translation --ignore=tests/llm_responses_api_testing --ignore=tests/mcp_tests --ignore=tests/guardrails_tests --ignore=tests/image_gen_tests --ignore=tests/pass_through_unit_tests - no_output_timeout: 120m + python -m pytest -s -v tests/*.py -x --junitxml=test-results/junit.xml -n 4 --durations=5 --ignore=tests/otel_tests --ignore=tests/spend_tracking_tests --ignore=tests/pass_through_tests --ignore=tests/proxy_admin_ui_tests --ignore=tests/load_tests --ignore=tests/llm_translation --ignore=tests/llm_responses_api_testing --ignore=tests/mcp_tests --ignore=tests/guardrails_tests --ignore=tests/image_gen_tests --ignore=tests/pass_through_unit_tests + no_output_timeout: 15m # Store test results - store_test_results: @@ -2599,7 +2333,7 @@ jobs: e2e_openai_endpoints: machine: image: ubuntu-2204:2023.10.1 - resource_class: xlarge + resource_class: large working_directory: ~/project steps: - checkout @@ -2680,7 +2414,7 @@ jobs: - run: name: Load Docker Database Image command: | - gunzip -c litellm-docker-database.tar.gz | docker load + zstd -d litellm-docker-database.tar.zst --stdout | docker load docker images | grep litellm-docker-database - run: name: Run Docker container @@ -2738,7 +2472,7 @@ jobs: pwd ls python -m pytest -s -vv tests/openai_endpoints_tests --junitxml=test-results/junit.xml --durations=5 - no_output_timeout: 120m + no_output_timeout: 15m # Store test results - store_test_results: @@ -2746,7 +2480,7 @@ jobs: proxy_logging_guardrails_model_info_tests: machine: image: ubuntu-2204:2023.10.1 - resource_class: xlarge + resource_class: large working_directory: ~/project steps: - checkout @@ -2824,7 +2558,7 @@ jobs: - run: name: Load Docker Database Image command: | - gunzip -c litellm-docker-database.tar.gz | docker load + zstd -d litellm-docker-database.tar.zst --stdout | docker load docker images | grep litellm-docker-database - run: name: Run Docker container @@ -2882,9 +2616,8 @@ jobs: command: | pwd ls - python -m pytest -vv tests/otel_tests -x --junitxml=test-results/junit.xml --durations=5 - no_output_timeout: - 120m + python -m pytest -v tests/otel_tests -x --junitxml=test-results/junit.xml --durations=5 + no_output_timeout: 15m # Clean up first container - run: name: Stop and remove first container @@ -2926,8 +2659,8 @@ jobs: - run: name: Run second round of tests command: | - python -m pytest -vv tests/basic_proxy_startup_tests -x --junitxml=test-results/junit-2.xml --durations=5 - no_output_timeout: 120m + python -m pytest -v tests/basic_proxy_startup_tests -x --junitxml=test-results/junit-2.xml --durations=5 + no_output_timeout: 15m # Store test results - store_test_results: @@ -2935,7 +2668,7 @@ jobs: proxy_spend_accuracy_tests: machine: image: ubuntu-2204:2023.10.1 - resource_class: xlarge + resource_class: large working_directory: ~/project steps: - checkout @@ -2989,7 +2722,7 @@ jobs: - run: name: Load Docker Database Image command: | - gunzip -c litellm-docker-database.tar.gz | docker load + zstd -d litellm-docker-database.tar.zst --stdout | docker load docker images | grep litellm-docker-database - run: name: Run Docker container @@ -3036,8 +2769,7 @@ jobs: pwd ls python -m pytest -vv tests/spend_tracking_tests -x --junitxml=test-results/junit.xml --durations=5 - no_output_timeout: - 120m + no_output_timeout: 15m # Clean up first container - run: name: Stop and remove first container @@ -3048,7 +2780,7 @@ jobs: proxy_multi_instance_tests: machine: image: ubuntu-2204:2023.10.1 - resource_class: xlarge + resource_class: large working_directory: ~/project steps: - checkout @@ -3106,7 +2838,7 @@ jobs: - run: name: Load Docker Database Image command: | - gunzip -c litellm-docker-database.tar.gz | docker load + zstd -d litellm-docker-database.tar.zst --stdout | docker load docker images | grep litellm-docker-database - run: name: Run Docker container 1 @@ -3176,8 +2908,7 @@ jobs: pwd ls python -m pytest -vv tests/multi_instance_e2e_tests -x --junitxml=test-results/junit.xml --durations=5 - no_output_timeout: - 120m + no_output_timeout: 15m # Clean up first container # Store test results - store_test_results: @@ -3186,7 +2917,7 @@ jobs: proxy_store_model_in_db_tests: machine: image: ubuntu-2204:2023.10.1 - resource_class: xlarge + resource_class: large working_directory: ~/project steps: - checkout @@ -3246,7 +2977,7 @@ jobs: - run: name: Load Docker Database Image command: | - gunzip -c litellm-docker-database.tar.gz | docker load + zstd -d litellm-docker-database.tar.zst --stdout | docker load docker images | grep litellm-docker-database - run: name: Run Docker container @@ -3287,7 +3018,7 @@ jobs: pwd ls python -m pytest -vv tests/store_model_in_db_tests -x --junitxml=test-results/junit.xml --durations=5 - no_output_timeout: 120m + no_output_timeout: 15m - run: name: Stop and remove containers command: | @@ -3303,7 +3034,7 @@ jobs: # Change from docker to machine executor machine: image: ubuntu-2204:2023.10.1 - resource_class: xlarge + resource_class: large working_directory: ~/project steps: - checkout @@ -3323,16 +3054,9 @@ jobs: - run: name: Install Dependencies command: | - pip install "pytest==7.3.1" - pip install "pytest-asyncio==0.21.1" - pip install aiohttp python -m pip install --upgrade pip - pip install "pytest==7.3.1" - pip install "pytest-retry==1.6.3" - pip install "pytest-mock==3.12.0" - pip install "pytest-asyncio==0.21.1" - pip install "mypy==1.18.2" - pip install apscheduler + pip install "pytest==7.3.1" "pytest-asyncio==0.21.1" "pytest-retry==1.6.3" \ + "pytest-mock==3.12.0" "mypy==1.18.2" aiohttp apscheduler - run: name: Build Docker image command: | @@ -3389,8 +3113,7 @@ jobs: name: Run tests command: | python -m pytest -vv tests/basic_proxy_startup_tests -x --junitxml=test-results/junit-2.xml --durations=5 - no_output_timeout: - 120m + no_output_timeout: 15m # Clean up first container - run: name: Stop and remove first container @@ -3400,17 +3123,11 @@ jobs: proxy_pass_through_endpoint_tests: machine: image: ubuntu-2204:2023.10.1 - resource_class: xlarge + resource_class: large working_directory: ~/project steps: - checkout - setup_google_dns - - run: - name: Install Docker CLI (In case it's not already installed) - command: | - curl -fsSL https://get.docker.com | sh - sudo usermod -aG docker $USER - docker version - run: name: Install Python 3.10 command: | @@ -3482,7 +3199,7 @@ jobs: - run: name: Load Docker Database Image command: | - gunzip -c litellm-docker-database.tar.gz | docker load + zstd -d litellm-docker-database.tar.zst --stdout | docker load docker images | grep litellm-docker-database - run: name: Run Docker container @@ -3501,6 +3218,7 @@ jobs: -e DD_API_KEY=$DD_API_KEY \ -e DD_SITE=$DD_SITE \ -e LITELLM_LICENSE=$LITELLM_LICENSE \ + -e LITELLM_USE_CHAT_COMPLETIONS_URL_FOR_ANTHROPIC_MESSAGES=true \ --add-host host.docker.internal:host-gateway \ --name my-app \ -v $(pwd)/litellm/proxy/example_config_yaml/pass_through_config.yaml:/app/config.yaml \ @@ -3578,8 +3296,8 @@ jobs: conda activate myenv pwd ls - python -m pytest -vv tests/pass_through_tests/ -x --junitxml=test-results/junit.xml --durations=5 - no_output_timeout: 120m + python -m pytest -v tests/pass_through_tests/ -x --junitxml=test-results/junit.xml --durations=5 + no_output_timeout: 15m # Store test results - store_test_results: @@ -3588,7 +3306,7 @@ jobs: proxy_e2e_anthropic_messages_tests: machine: image: ubuntu-2204:2023.10.1 - resource_class: xlarge + resource_class: large working_directory: ~/project steps: - checkout @@ -3646,7 +3364,7 @@ jobs: - run: name: Load Docker Database Image command: | - gunzip -c litellm-docker-database.tar.gz | docker load + zstd -d litellm-docker-database.tar.zst --stdout | docker load docker images | grep litellm-docker-database - run: name: Run Docker container with test config @@ -3685,7 +3403,7 @@ jobs: pwd ls python -m pytest -vv tests/proxy_e2e_anthropic_messages_tests/ -x -s --junitxml=test-results/junit.xml --durations=5 - no_output_timeout: 120m + no_output_timeout: 15m # Store test results - store_test_results: @@ -3694,7 +3412,7 @@ jobs: proxy_e2e_azure_batches_tests: machine: image: ubuntu-2204:2023.10.1 - resource_class: xlarge + resource_class: large working_directory: ~/project steps: - checkout @@ -3797,7 +3515,7 @@ jobs: --maxfail=3 \ --durations=10 \ --junitxml=test-results/junit.xml - no_output_timeout: 30m + no_output_timeout: 15m upload-coverage: docker: @@ -3820,7 +3538,7 @@ jobs: python -m venv venv . venv/bin/activate pip install coverage - coverage combine llm_translation_coverage realtime_translation_coverage llm_responses_api_coverage ocr_coverage search_coverage mcp_coverage litellm_mcps_tests_coverage logging_coverage audio_coverage litellm_router_coverage litellm_router_unit_coverage local_testing_part1_coverage local_testing_part2_coverage litellm_assistants_api_coverage auth_ui_unit_tests_coverage langfuse_coverage caching_coverage litellm_proxy_unit_tests_part1_coverage litellm_proxy_unit_tests_part2_coverage image_gen_coverage pass_through_unit_tests_coverage batches_coverage litellm_security_tests_coverage guardrails_coverage litellm_mapped_tests_coverage + coverage combine realtime_translation_coverage ocr_coverage search_coverage mcp_coverage litellm_mcps_tests_coverage logging_coverage audio_coverage local_testing_part1_coverage local_testing_part2_coverage pass_through_unit_tests_coverage batches_coverage guardrails_coverage coverage xml - codecov/upload: file: ./coverage.xml @@ -4019,83 +3737,87 @@ jobs: twine upload --verbose dist/* ui_build: - machine: - image: ubuntu-2204:2023.10.1 - resource_class: xlarge + docker: + - image: cimg/node:20.19 + auth: + username: ${DOCKERHUB_USERNAME} + password: ${DOCKERHUB_PASSWORD} + resource_class: medium+ working_directory: ~/project steps: - checkout - setup_google_dns + - restore_cache: + keys: + - ui-build-deps-v1-{{ checksum "ui/litellm-dashboard/package-lock.json" }} + - ui-build-deps-v1- + - restore_cache: + keys: + - ui-nextjs-cache-v1-{{ checksum "ui/litellm-dashboard/package-lock.json" }} + - ui-nextjs-cache-v1- + - run: + name: Install dependencies + command: | + cd ui/litellm-dashboard + npm ci + - save_cache: + key: ui-build-deps-v1-{{ checksum "ui/litellm-dashboard/package-lock.json" }} + paths: + - ui/litellm-dashboard/node_modules - run: name: Build UI command: | - # Set up nvm - export NVM_DIR="/opt/circleci/.nvm" - source "$NVM_DIR/nvm.sh" - source "$NVM_DIR/bash_completion" - - # Install and use Node version - nvm install v20 - nvm use v20 - cd ui/litellm-dashboard - - # Remove node_modules and package-lock to ensure clean install (fixes dependency resolution issues) - rm -rf node_modules package-lock.json - - # Install dependencies first - npm install - - # Now source the build script source ./build_ui.sh + - save_cache: + key: ui-nextjs-cache-v1-{{ checksum "ui/litellm-dashboard/package-lock.json" }} + paths: + - ui/litellm-dashboard/.next/cache - persist_to_workspace: root: . paths: - litellm/proxy/_experimental/out ui_unit_tests: - machine: - image: ubuntu-2204:2023.10.1 + docker: + - image: cimg/node:20.19 + auth: + username: ${DOCKERHUB_USERNAME} + password: ${DOCKERHUB_PASSWORD} resource_class: xlarge working_directory: ~/project steps: - checkout - setup_google_dns + - restore_cache: + keys: + - ui-unit-deps-v1-{{ checksum "ui/litellm-dashboard/package-lock.json" }} + - ui-unit-deps-v1- + - run: + name: Install dependencies + command: | + cd ui/litellm-dashboard + npm ci + - save_cache: + key: ui-unit-deps-v1-{{ checksum "ui/litellm-dashboard/package-lock.json" }} + paths: + - ui/litellm-dashboard/node_modules - run: name: Run UI unit tests (Vitest) command: | - # Use Node 20 (several deps require >=20) - export NVM_DIR="/opt/circleci/.nvm" - source "$NVM_DIR/nvm.sh" - nvm install 20 - nvm use 20 - cd ui/litellm-dashboard - # Remove node_modules and package-lock to ensure clean install (fixes optional deps issue) - rm -rf node_modules package-lock.json - npm install - # CI run, with both LCOV (Codecov) and HTML (artifact you can click) - CI=true npm run test -- --run --coverage \ - --coverage.provider=v8 \ - --coverage.reporter=lcov \ - --coverage.reporter=html \ - --coverage.reportsDirectory=coverage/html + CI=true npm run test -- --run \ + --pool forks --poolOptions.forks.maxForks=8 build_docker_database_image: machine: - image: ubuntu-2204:2023.10.1 - resource_class: xlarge + image: ubuntu-2204:2024.04.1 + resource_class: large working_directory: ~/project steps: - checkout - - run: - name: Upgrade Docker - command: | - curl -fsSL https://get.docker.com | sh - docker version - - run: name: Build Docker image command: | @@ -4106,17 +3828,17 @@ jobs: - run: name: Save Docker image to workspace root command: | - docker save litellm-docker-database:ci | gzip > litellm-docker-database.tar.gz + docker save litellm-docker-database:ci | zstd -1 -T0 > litellm-docker-database.tar.zst - persist_to_workspace: root: . paths: - - litellm-docker-database.tar.gz + - litellm-docker-database.tar.zst e2e_ui_testing: machine: image: ubuntu-2204:2023.10.1 - resource_class: xlarge + resource_class: large working_directory: ~/project parameters: browser: @@ -4129,7 +3851,7 @@ jobs: - run: name: Load Docker Database Image command: | - gunzip -c litellm-docker-database.tar.gz | docker load + zstd -d litellm-docker-database.tar.zst --stdout | docker load docker images | grep litellm-docker-database - run: name: Install Dependencies @@ -4201,7 +3923,7 @@ jobs: --config ui/litellm-dashboard/e2e_tests/playwright.config.ts \ --reporter=html \ --output=test-results - no_output_timeout: 120m + no_output_timeout: 15m - store_artifacts: path: test-results destination: playwright-results @@ -4213,7 +3935,7 @@ jobs: prisma_schema_sync: machine: image: ubuntu-2204:2023.10.1 - resource_class: xlarge + resource_class: medium working_directory: ~/project steps: - checkout @@ -4223,7 +3945,7 @@ jobs: - run: name: Load Docker Database Image command: | - gunzip -c litellm-docker-database.tar.gz | docker load + zstd -d litellm-docker-database.tar.zst --stdout | docker load docker images | grep litellm-docker-database - run: name: Install Neon CLI @@ -4267,36 +3989,16 @@ jobs: name: Stop schema sync container command: docker stop schema-sync - test_nonroot_image: - machine: - image: ubuntu-2204:2023.10.1 - resource_class: xlarge - working_directory: ~/project - steps: - - checkout - - setup_google_dns - - run: - name: Build Docker image - command: | - docker build -t non_root_image:latest . -f ./docker/Dockerfile.non_root - - run: - name: Install Container Structure Test - command: | - curl -LO https://github.com/GoogleContainerTools/container-structure-test/releases/download/v1.19.3/container-structure-test-linux-amd64 - chmod +x container-structure-test-linux-amd64 - sudo mv container-structure-test-linux-amd64 /usr/local/bin/container-structure-test - - run: - name: Run Container Structure Test - command: | - container-structure-test test --image non_root_image:latest --config docker/tests/nonroot.yaml test_bad_database_url: machine: image: ubuntu-2204:2023.10.1 - resource_class: xlarge + resource_class: medium working_directory: ~/project steps: - checkout + - attach_workspace: + at: ~/project - setup_google_dns - run: name: Install dockerize @@ -4318,9 +4020,10 @@ jobs: name: Wait for PostgreSQL to be ready command: dockerize -wait tcp://localhost:5432 -timeout 1m - run: - name: Build Docker image + name: Load Docker Database Image command: | - docker build -t myapp . -f ./docker/Dockerfile.non_root + zstd -d litellm-docker-database.tar.zst --stdout | docker load + docker tag litellm-docker-database:ci myapp:latest - run: name: Run Docker container with bad DATABASE_URL command: | @@ -4502,6 +4205,8 @@ workflows: - main - /litellm_.*/ - build_and_test: + requires: + - build_docker_database_image filters: branches: only: @@ -4715,13 +4420,11 @@ workflows: - /litellm_.*/ - upload-coverage: requires: - - llm_translation_testing - realtime_translation_testing - mcp_testing - agent_testing - google_generate_content_endpoint_testing - guardrails_testing - - llm_responses_api_testing - ocr_testing - search_testing - litellm_mapped_tests_proxy_part1 @@ -4738,18 +4441,12 @@ workflows: - image_gen_testing - logging_testing - audio_testing - - litellm_router_testing - - litellm_router_unit_testing - caching_unit_tests - litellm_proxy_unit_testing_key_generation - - litellm_proxy_unit_testing_part1 - - litellm_proxy_unit_testing_part2 - - litellm_security_tests - langfuse_logging_unit_tests - local_testing_part1 - local_testing_part2 - litellm_assistants_api_testing - - auth_ui_unit_tests - db_migration_disable_update_check: requires: - build_docker_database_image @@ -4771,12 +4468,16 @@ workflows: - main - /litellm_.*/ - helm_chart_testing: + requires: + - build_docker_database_image filters: branches: only: - main - /litellm_.*/ - test_bad_database_url: + requires: + - build_docker_database_image filters: branches: only: @@ -4825,6 +4526,7 @@ workflows: - langfuse_logging_unit_tests - litellm_assistants_api_testing - auth_ui_unit_tests + - ui_unit_tests - db_migration_disable_update_check - e2e_ui_testing_chromium - e2e_ui_testing_firefox diff --git a/.circleci/requirements.txt b/.circleci/requirements.txt index a5ec74424fe..ab4c3995772 100644 --- a/.circleci/requirements.txt +++ b/.circleci/requirements.txt @@ -17,4 +17,5 @@ mcp==1.25.0 # for MCP server semantic_router==0.1.10 # for auto-routing with litellm fastuuid==0.12.0 responses==0.25.7 # for proxy client tests -pytest-retry==1.6.3 # for automatic test retries \ No newline at end of file +pytest-retry==1.6.3 # for automatic test retries +litellm-proxy-extras # for prisma migrations \ No newline at end of file diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index bd434bea39d..d830c16dfa2 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -11,6 +11,10 @@ - [ ] My PR's scope is as isolated as possible, it only solves 1 specific problem - [ ] I have requested a Greptile review by commenting `@greptileai` and received a **Confidence Score of at least 4/5** before requesting a maintainer review +## Delays in PR merge? + +If you're seeing a delay in your PR being merged, ping the LiteLLM Team on [Slack (#pr-review)](https://join.slack.com/t/litellmossslack/shared_invite/zt-3o7nkuyfr-p_kbNJj8taRfXGgQI1~YyA). + ## CI (LiteLLM team) > **CI status guideline:** diff --git a/.github/workflows/codspeed.yml b/.github/workflows/codspeed.yml new file mode 100644 index 00000000000..385b95fdaf5 --- /dev/null +++ b/.github/workflows/codspeed.yml @@ -0,0 +1,44 @@ +name: CodSpeed Benchmarks + +on: + push: + branches: + - main + pull_request: + branches: + - main + # Allow CodSpeed to trigger backtest performance analysis + # in order to generate initial data + workflow_dispatch: + +permissions: + contents: read + id-token: write + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + benchmarks: + runs-on: ubuntu-latest + timeout-minutes: 15 + + steps: + - uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Install dependencies + run: | + pip install -e "." + pip install pytest pytest-codspeed==4.3.0 + + - name: Run benchmarks + uses: CodSpeedHQ/action@v4 + with: + mode: simulation + run: pytest tests/benchmarks/ --codspeed diff --git a/.github/workflows/ghcr_deploy.yml b/.github/workflows/ghcr_deploy.yml index c317309d91a..344b0ec48ee 100644 --- a/.github/workflows/ghcr_deploy.yml +++ b/.github/workflows/ghcr_deploy.yml @@ -369,7 +369,8 @@ jobs: release: name: "New LiteLLM Release" needs: [docker-hub-deploy, build-and-push-image, build-and-push-image-database] - + permissions: + contents: write runs-on: "ubuntu-latest" steps: diff --git a/.github/workflows/test-linting.yml b/.github/workflows/test-linting.yml index e918a71373a..4cedb8b5bae 100644 --- a/.github/workflows/test-linting.yml +++ b/.github/workflows/test-linting.yml @@ -28,15 +28,18 @@ jobs: find . -type d -name "__pycache__" -exec rm -rf {} + || true find . -name "*.pyc" -delete || true + - name: Check poetry.lock is up to date + run: | + poetry check --lock || (echo "❌ poetry.lock is out of sync with pyproject.toml. Run 'poetry lock' locally and commit the result." && exit 1) + - name: Install dependencies run: | - poetry lock poetry install --with dev - - name: Run Black formatting + - name: Check Black formatting run: | cd litellm - poetry run black . + poetry run black --check --exclude '/enterprise/' . cd .. - name: Debug - Check file state diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 9396f323e45..2bc361bc48f 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -14,12 +14,12 @@ repos: types: [python] files: (litellm/|litellm_proxy_extras/|enterprise/).*\.py exclude: ^litellm/__init__.py$ - # - id: black - # name: black - # entry: poetry run black - # language: system - # types: [python] - # files: (litellm/|litellm_proxy_extras/|enterprise/).*\.py + - id: black + name: black + entry: poetry run black + language: system + types: [python] + files: (litellm/|litellm_proxy_extras/).*\.py - repo: https://github.com/pycqa/flake8 rev: 7.0.0 # The version of flake8 to use hooks: diff --git a/CLAUDE.md b/CLAUDE.md index 5d62d2cdcda..f0478120181 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -91,6 +91,10 @@ LiteLLM is a unified interface for 100+ LLM providers with two main components: - Async/await patterns throughout - Type hints required for all public APIs - **Avoid imports within methods** — place all imports at the top of the file (module-level). Inline imports inside functions/methods make dependencies harder to trace and hurt readability. The only exception is avoiding circular imports where absolutely necessary. +- **Use dict spread for immutable copies** — prefer `{**original, "key": new_value}` over `dict(obj)` + mutation. The spread produces the final dict in one step and makes intent clear. +- **Guard at resolution time** — when resolving an optional value through a fallback chain (`a or b or ""`), raise immediately if the resolved result being empty is an error. Don't pass empty strings or sentinel values downstream for the callee to deal with. +- **Extract complex comprehensions to named helpers** — a set/dict comprehension that calls into the DB or manager (e.g. "which of these server IDs are OAuth2?") belongs in a named helper function, not inline in the caller. +- **FastAPI parameter declarations** — mark required query/form params with `= Query(...)` / `= Form(...)` explicitly when other params in the same handler are optional. Mixing `str` (required) with `Optional[str] = None` in the same signature causes silent 422s when the required param is missing. ### Testing Strategy - Unit tests in `tests/test_litellm/` @@ -98,6 +102,8 @@ LiteLLM is a unified interface for 100+ LLM providers with two main components: - Proxy tests in `tests/proxy_unit_tests/` - Load tests in `tests/load_tests/` - **Always add tests when adding new entity types or features** — if the existing test file covers other entity types, add corresponding tests for the new one +- **Keep monkeypatch stubs in sync with real signatures** — when a function gains a new optional parameter, update every `fake_*` / `stub_*` in tests that patch it to also accept that kwarg (even as `**kwargs`). Stale stubs fail with `unexpected keyword argument` and mask real bugs. +- **Test all branches of name→ID resolution** — when adding server/resource lookup that resolves names to UUIDs, test: (1) name resolves and UUID is allowed, (2) name resolves but UUID is not allowed, (3) name does not resolve at all. The silent-fallback path is where access-control bugs hide. ### UI / Backend Consistency - When wiring a new UI entity type to an existing backend endpoint, verify the backend API contract (single value vs. array, required vs. optional params) and ensure the UI controls match — e.g., use a single-select dropdown when the backend accepts a single value, not a multi-select @@ -134,6 +140,11 @@ LiteLLM is a unified interface for 100+ LLM providers with two main components: - **Check index coverage.** For new or modified queries, check `schema.prisma` for a supporting index. Prefer extending an existing index (e.g. `@@index([a])` → `@@index([a, b])`) over adding a new one, unless it's a `@@unique`. Only add indexes for large/frequent queries. - **Keep schema files in sync.** Apply schema changes to all `schema.prisma` copies (`schema.prisma`, `litellm/proxy/`, `litellm-proxy-extras/`, `litellm-js/spend-logs/` for SpendLogs) with a migration under `litellm-proxy-extras/litellm_proxy_extras/migrations/`. +### Setup Wizard (`litellm/setup_wizard.py`) +- The wizard is implemented as a single `SetupWizard` class with `@staticmethod` methods — keep it that way. No module-level functions except `run_setup_wizard()` (the public entrypoint) and pure helpers (color, ANSI). +- Use `litellm.utils.check_valid_key(model, api_key)` for credential validation — never roll a custom completion call. +- Do not hardcode provider env-key names or model lists that already exist in the codebase. Add a `test_model` field to each provider entry to drive `check_valid_key`; set it to `None` for providers that can't be validated with a single API key (Azure, Bedrock, Ollama). + ### Enterprise Features - Enterprise-specific code in `enterprise/` directory - Optional features enabled via environment variables @@ -150,4 +161,4 @@ LiteLLM is a unified interface for 100+ LLM providers with two main components: **Fix options:** 1. **Create a Prisma migration** (permanent) — run `prisma migrate dev --name ` in the worktree. The generated file will be picked up by `prisma migrate deploy` on next startup. 2. **Apply manually for local dev** — `psql -d litellm -c "ALTER TABLE ... ADD COLUMN IF NOT EXISTS ..."` after each proxy start. Fine for dev, not for production. -3. **Update litellm-proxy-extras** — if the package is installed from PyPI, its migration directory must include the new file. Either update the package or run the migration manually until the next release ships it. \ No newline at end of file +3. **Update litellm-proxy-extras** — if the package is installed from PyPI, its migration directory must include the new file. Either update the package or run the migration manually until the next release ships it. diff --git a/Dockerfile b/Dockerfile index 75ccff29663..7bda32acf27 100644 --- a/Dockerfile +++ b/Dockerfile @@ -39,7 +39,7 @@ RUN pip wheel --no-cache-dir --wheel-dir=/wheels/ -r requirements.txt # ensure pyjwt is used, not jwt RUN pip uninstall jwt -y RUN pip uninstall PyJWT -y -RUN pip install PyJWT==2.9.0 --no-cache-dir +RUN pip install PyJWT==2.12.0 --no-cache-dir # Runtime stage FROM $LITELLM_RUNTIME_IMAGE AS runtime @@ -49,7 +49,7 @@ USER root # Install runtime dependencies (libsndfile needed for audio processing on ARM64) RUN apk add --no-cache bash openssl tzdata nodejs npm python3 py3-pip libsndfile && \ - npm install -g npm@latest tar@7.5.10 glob@11.1.0 @isaacs/brace-expansion@5.0.1 minimatch@10.2.4 diff@8.0.3 && \ + npm install -g npm@latest tar@7.5.11 glob@11.1.0 @isaacs/brace-expansion@5.0.1 minimatch@10.2.4 diff@8.0.3 && \ # SECURITY FIX: npm bundles tar, glob, and brace-expansion at multiple nested # levels inside its dependency tree. `npm install -g ` only creates a # SEPARATE global package, it does NOT replace npm's internal copies. diff --git a/README.md b/README.md index 3db827d5fdd..67f2f3a2048 100644 --- a/README.md +++ b/README.md @@ -28,6 +28,9 @@ Slack + + CodSpeed + Group 7154 (1) diff --git a/ci_cd/security_scans.sh b/ci_cd/security_scans.sh index 62440d13ebb..801b700f64f 100755 --- a/ci_cd/security_scans.sh +++ b/ci_cd/security_scans.sh @@ -11,7 +11,7 @@ echo "Starting security scans for LiteLLM..." install_trivy() { echo "Installing Trivy and required tools..." sudo apt-get update - sudo apt-get install -y wget apt-transport-https gnupg lsb-release jq curl + sudo apt-get install -y wget apt-transport-https gnupg lsb-release jq curl bsdmainutils wget -qO - https://aquasecurity.github.io/trivy-repo/deb/public.key | sudo apt-key add - echo "deb https://aquasecurity.github.io/trivy-repo/deb $(lsb_release -sc) main" | sudo tee -a /etc/apt/sources.list.d/trivy.list sudo apt-get update @@ -163,6 +163,9 @@ run_grype_scans() { "CVE-2026-25639" # axios - full fix requires 1.x major version bump; pinned to >=0.30.2 to clear other axios CVEs, upgrade to 1.x in follow-up "CVE-2026-2297" # Python 3.13 SourcelessFileLoader audit hook bypass - no fix available in base image "GHSA-qffp-2rhf-9h96" # tar hardlink path traversal - from nodejs_wheel bundled npm, not used in application runtime code + "CVE-2026-2673" # OpenSSL 3.6.1 TLS 1.3 key exchange group negotiation issue - no fix available yet + "CVE-2026-3644" # Python 3.13 vulnerability - no fix available in base image + "CVE-2026-4224" # Python 3.13 Expat parser stack overflow in ElementDeclHandler - no fix available in base image ) # Build JSON array of allowlisted CVE IDs for jq diff --git a/deploy/charts/litellm-helm/templates/deployment.yaml b/deploy/charts/litellm-helm/templates/deployment.yaml index 51af22b7a46..3040fb45d86 100644 --- a/deploy/charts/litellm-helm/templates/deployment.yaml +++ b/deploy/charts/litellm-helm/templates/deployment.yaml @@ -20,6 +20,9 @@ spec: selector: matchLabels: {{- include "litellm.selectorLabels" . | nindent 6 }} + {{- if .Values.deploymentMinReadySeconds }} + minReadySeconds: {{ .Values.deploymentMinReadySeconds }} + {{- end }} template: metadata: annotations: diff --git a/deploy/charts/litellm-helm/tests/deployment_tests.yaml b/deploy/charts/litellm-helm/tests/deployment_tests.yaml index 2e9c48043de..0d278f25693 100644 --- a/deploy/charts/litellm-helm/tests/deployment_tests.yaml +++ b/deploy/charts/litellm-helm/tests/deployment_tests.yaml @@ -306,3 +306,16 @@ tests: - equal: path: spec.template.spec.containers[0].resources value: {} + - it: should be able to set minReadySeconds + template: deployment.yaml + set: + deploymentMinReadySeconds: 5 + asserts: + - equal: + path: spec.minReadySeconds + value: 5 + - it: should have minReadySeconds absent when deploymentMinReadySeconds is not set + template: deployment.yaml + asserts: + - notExists: + path: spec.minReadySeconds diff --git a/deploy/charts/litellm-helm/values.yaml b/deploy/charts/litellm-helm/values.yaml index f8944bddd53..690ca69e730 100644 --- a/deploy/charts/litellm-helm/values.yaml +++ b/deploy/charts/litellm-helm/values.yaml @@ -31,6 +31,8 @@ serviceAccount: # annotations for litellm deployment deploymentAnnotations: {} deploymentLabels: {} +deploymentMinReadySeconds: 0 + # annotations for litellm pods podAnnotations: {} podLabels: {} diff --git a/docker/Dockerfile.custom_ui b/docker/Dockerfile.custom_ui index 4052c7a51bc..c1bd9a383fa 100644 --- a/docker/Dockerfile.custom_ui +++ b/docker/Dockerfile.custom_ui @@ -19,7 +19,7 @@ RUN apt-get update && apt-get upgrade -y \ libgnutls30 \ libc6 && \ apt-get install -y nodejs npm && \ - npm install -g npm@latest tar@7.5.10 glob@11.1.0 @isaacs/brace-expansion@5.0.1 minimatch@10.2.4 diff@8.0.3 && \ + npm install -g npm@latest tar@7.5.11 glob@11.1.0 @isaacs/brace-expansion@5.0.1 minimatch@10.2.4 diff@8.0.3 && \ GLOBAL="$(npm root -g)" && \ find "$GLOBAL/npm" -type d -name "tar" -path "*/node_modules/tar" | while read d; do \ rm -rf "$d" && cp -rL "$GLOBAL/tar" "$d"; \ diff --git a/docker/Dockerfile.database b/docker/Dockerfile.database index 962d129e57f..3e1c55a75a8 100644 --- a/docker/Dockerfile.database +++ b/docker/Dockerfile.database @@ -50,7 +50,7 @@ USER root # Install runtime dependencies RUN apk add --no-cache bash openssl tzdata nodejs npm python3 py3-pip libsndfile && \ - npm install -g npm@latest tar@7.5.10 glob@11.1.0 @isaacs/brace-expansion@5.0.1 minimatch@10.2.4 diff@8.0.3 && \ + npm install -g npm@latest tar@7.5.11 glob@11.1.0 @isaacs/brace-expansion@5.0.1 minimatch@10.2.4 diff@8.0.3 && \ GLOBAL="$(npm root -g)" && \ find "$GLOBAL/npm" -type d -name "tar" -path "*/node_modules/tar" | while read d; do \ rm -rf "$d" && cp -rL "$GLOBAL/tar" "$d"; \ @@ -112,7 +112,7 @@ RUN sed -i 's/\r$//' docker/install_auto_router.sh && chmod +x docker/install_au # ensure pyjwt is used, not jwt RUN pip uninstall jwt -y RUN pip uninstall PyJWT -y -RUN pip install PyJWT==2.9.0 --no-cache-dir +RUN pip install PyJWT==2.12.0 --no-cache-dir # Build Admin UI (runtime stage) # Convert Windows line endings to Unix and make executable diff --git a/docker/Dockerfile.dev b/docker/Dockerfile.dev index cfc4c646ba2..e3e7ac0e0d6 100644 --- a/docker/Dockerfile.dev +++ b/docker/Dockerfile.dev @@ -31,7 +31,7 @@ RUN --mount=type=cache,target=/root/.cache/pip \ # Fix JWT dependency conflicts early RUN pip uninstall jwt -y || true && \ pip uninstall PyJWT -y || true && \ - pip install PyJWT==2.9.0 --no-cache-dir + pip install PyJWT==2.12.0 --no-cache-dir # Copy only necessary files for build COPY pyproject.toml README.md schema.prisma poetry.lock ./ @@ -75,7 +75,7 @@ RUN apt-get update && apt-get upgrade -y \ nodejs \ npm \ && rm -rf /var/lib/apt/lists/* \ - && npm install -g npm@latest tar@7.5.10 glob@11.1.0 @isaacs/brace-expansion@5.0.1 minimatch@10.2.4 diff@8.0.3 \ + && npm install -g npm@latest tar@7.5.11 glob@11.1.0 @isaacs/brace-expansion@5.0.1 minimatch@10.2.4 diff@8.0.3 \ && GLOBAL="$(npm root -g)" \ && find "$GLOBAL/npm" -type d -name "tar" -path "*/node_modules/tar" | while read d; do \ rm -rf "$d" && cp -rL "$GLOBAL/tar" "$d"; \ diff --git a/docker/Dockerfile.non_root b/docker/Dockerfile.non_root index fbc16e4f876..db3981fb7e7 100644 --- a/docker/Dockerfile.non_root +++ b/docker/Dockerfile.non_root @@ -32,7 +32,7 @@ RUN for i in 1 2 3; do \ # Cache Python dependencies COPY requirements.txt . RUN pip wheel --no-cache-dir --wheel-dir=/wheels/ -r requirements.txt \ - && pip wheel --no-cache-dir --wheel-dir=/wheels/ "semantic_router==0.1.11" "aurelio-sdk==0.0.19" "PyJWT==2.9.0" + && pip wheel --no-cache-dir --wheel-dir=/wheels/ "semantic_router==0.1.11" "aurelio-sdk==0.0.19" "PyJWT==2.12.0" # Copy source after dependency layers COPY . . @@ -106,7 +106,7 @@ RUN for i in 1 2 3; do \ apk add --no-cache python3 py3-pip bash openssl tzdata nodejs npm supervisor && break || sleep 5; \ done \ && apk upgrade --no-cache nodejs \ - && npm install -g npm@latest tar@7.5.10 glob@11.1.0 @isaacs/brace-expansion@5.0.1 minimatch@10.2.4 diff@8.0.3 \ + && npm install -g npm@latest tar@7.5.11 glob@11.1.0 @isaacs/brace-expansion@5.0.1 minimatch@10.2.4 diff@8.0.3 \ && GLOBAL="$(npm root -g)" \ && find "$GLOBAL/npm" -type d -name "tar" -path "*/node_modules/tar" | while read d; do \ rm -rf "$d" && cp -rL "$GLOBAL/tar" "$d"; \ @@ -198,7 +198,7 @@ RUN sed -i 's/\r$//' docker/entrypoint.sh && \ chown -R nobody:nogroup /app /var/lib/litellm/ui /var/lib/litellm/assets /nonexistent /.npm && \ pip uninstall jwt -y || true && \ pip uninstall PyJWT -y || true && \ - pip install --no-index --find-links=/wheels/ PyJWT==2.10.1 --no-cache-dir && \ + pip install --no-index --find-links=/wheels/ PyJWT==2.12.0 --no-cache-dir && \ rm -rf /wheels && \ PRISMA_PATH=$(python -c "import os, prisma; print(os.path.dirname(prisma.__file__))") && \ chown -R nobody:nogroup $PRISMA_PATH && \ diff --git a/docs/my-website/blog/anthropic_opus_4_5_and_advanced_features/index.md b/docs/my-website/blog/anthropic_opus_4_5_and_advanced_features/index.md index 8a54426dfb0..21ba3d60790 100644 --- a/docs/my-website/blog/anthropic_opus_4_5_and_advanced_features/index.md +++ b/docs/my-website/blog/anthropic_opus_4_5_and_advanced_features/index.md @@ -3,18 +3,9 @@ slug: anthropic_advanced_features title: "Day 0 Support: Claude 4.5 Opus (+Advanced Features)" date: 2025-11-25T10:00:00 authors: - - name: Sameer Kankute - title: SWE @ LiteLLM (LLM Translation) - url: https://www.linkedin.com/in/sameer-kankute/ - image_url: https://pbs.twimg.com/profile_images/2001352686994907136/ONgNuSk5_400x400.jpg - - name: Krrish Dholakia - title: "CEO, LiteLLM" - url: https://www.linkedin.com/in/krish-d/ - image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg - - name: Ishaan Jaff - title: "CTO, LiteLLM" - url: https://www.linkedin.com/in/reffajnaahsi/ - image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg + - sameer + - krrish + - ishaan-alt description: "Guide to Claude Opus 4.5 and advanced features in LiteLLM: Tool Search, Programmatic Tool Calling, and Effort Parameter." tags: [anthropic, claude, tool search, programmatic tool calling, effort, advanced features] hide_table_of_contents: false @@ -25,6 +16,8 @@ import TabItem from '@theme/TabItem'; This guide covers Anthropic's latest model (Claude Opus 4.5) and its advanced features now available in LiteLLM: Tool Search, Programmatic Tool Calling, Tool Input Examples, and the Effort Parameter. +{/* truncate */} + --- | Feature | Supported Models | diff --git a/docs/my-website/blog/anthropic_wildcard_model_access_incident/index.md b/docs/my-website/blog/anthropic_wildcard_model_access_incident/index.md index f6172cd6744..8d58e18e580 100644 --- a/docs/my-website/blog/anthropic_wildcard_model_access_incident/index.md +++ b/docs/my-website/blog/anthropic_wildcard_model_access_incident/index.md @@ -3,18 +3,9 @@ slug: anthropic-wildcard-model-access-incident title: "Incident Report: Wildcard Blocking New Models After Cost Map Reload" date: 2026-02-23T10:00:00 authors: - - name: Sameer Kankute - title: SWE @ LiteLLM (LLM Translation) - url: https://www.linkedin.com/in/sameer-kankute/ - image_url: https://pbs.twimg.com/profile_images/2001352686994907136/ONgNuSk5_400x400.jpg - - name: Krrish Dholakia - title: "CEO, LiteLLM" - url: https://www.linkedin.com/in/krish-d/ - image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg - - name: Ishaan Jaff - title: "CTO, LiteLLM" - url: https://www.linkedin.com/in/reffajnaahsi/ - image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg + - sameer + - krrish + - ishaan-alt tags: [incident-report, proxy, auth, model-access] hide_table_of_contents: false --- diff --git a/docs/my-website/blog/authors.yml b/docs/my-website/blog/authors.yml index 2a49a736333..1b1ef4d34c4 100644 --- a/docs/my-website/blog/authors.yml +++ b/docs/my-website/blog/authors.yml @@ -4,6 +4,12 @@ litellm: url: https://github.com/BerriAI/litellm image_url: https://github.com/BerriAI.png +sameer: + name: Sameer Kankute + title: SWE @ LiteLLM (LLM Translation) + url: https://www.linkedin.com/in/sameer-kankute/ + image_url: https://pbs.twimg.com/profile_images/2001352686994907136/ONgNuSk5_400x400.jpg + krrish: name: Krrish Dholakia title: CEO, LiteLLM @@ -22,3 +28,21 @@ ishaan-alt: title: CTO, LiteLLM url: https://www.linkedin.com/in/reffajnaahsi/ image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg + +ryan: + name: Ryan Crabbe + title: Performance Engineer, LiteLLM + url: https://www.linkedin.com/in/ryan-crabbe-0b9687214 + image_url: https://media.licdn.com/dms/image/v2/D5603AQHt1t9Z4BJ6Gw/profile-displayphoto-shrink_400_400/profile-displayphoto-shrink_400_400/0/1724453682340?e=1772064000&v=beta&t=VXdmr13rsNB05wyA2F1TENOB5UuDHUZ0FCHTolNyR5M + +alexsander: + name: Alexsander Hamir + title: Performance Engineer, LiteLLM + url: https://www.linkedin.com/in/alexsander-baptista/ + image_url: https://github.com/AlexsanderHamir.png + +yuneng: + name: Yuneng Jiang + title: SWE @ LiteLLM (Full Stack) + url: https://www.linkedin.com/in/yuneng-david-jiang-455676139/ + image_url: https://avatars.githubusercontent.com/u/171294688?v=4 diff --git a/docs/my-website/blog/claude_code_beta_headers/index.md b/docs/my-website/blog/claude_code_beta_headers/index.md index 44567f616aa..ee07da79397 100644 --- a/docs/my-website/blog/claude_code_beta_headers/index.md +++ b/docs/my-website/blog/claude_code_beta_headers/index.md @@ -3,18 +3,9 @@ slug: claude-code-beta-headers-incident title: "Incident Report: Invalid beta headers with Claude Code" date: 2026-02-16T10:00:00 authors: - - name: Sameer Kankute - title: SWE @ LiteLLM (LLM Translation) - url: https://www.linkedin.com/in/sameer-kankute/ - image_url: https://pbs.twimg.com/profile_images/2001352686994907136/ONgNuSk5_400x400.jpg - - name: Ishaan Jaff - title: "CTO, LiteLLM" - url: https://www.linkedin.com/in/reffajnaahsi/ - image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg - - name: Krrish Dholakia - title: "CEO, LiteLLM" - url: https://www.linkedin.com/in/krish-d/ - image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg + - sameer + - ishaan-alt + - krrish tags: [incident-report, anthropic, stability] hide_table_of_contents: false --- @@ -173,5 +164,5 @@ curl -X POST "https://your-proxy-url/reload/anthropic_beta_headers" \ ## Related documentation -- [Managing Anthropic Beta Headers](../proxy/sync_anthropic_beta_headers.md) - Complete configuration guide +- [Managing Anthropic Beta Headers](../../docs/proxy/sync_anthropic_beta_headers) - Complete configuration guide - [`anthropic_beta_headers_config.json`](https://github.com/BerriAI/litellm/blob/main/litellm/anthropic_beta_headers_config.json) - Current configuration file diff --git a/docs/my-website/blog/claude_opus_4_6/index.md b/docs/my-website/blog/claude_opus_4_6/index.md index e44420bd570..eeb5d5ff2f8 100644 --- a/docs/my-website/blog/claude_opus_4_6/index.md +++ b/docs/my-website/blog/claude_opus_4_6/index.md @@ -3,18 +3,9 @@ slug: claude_opus_4_6 title: "Day 0 Support: Claude Opus 4.6" date: 2026-02-05T10:00:00 authors: - - name: Sameer Kankute - title: SWE @ LiteLLM (LLM Translation) - url: https://www.linkedin.com/in/sameer-kankute/ - image_url: https://pbs.twimg.com/profile_images/2001352686994907136/ONgNuSk5_400x400.jpg - - name: Ishaan Jaff - title: "CTO, LiteLLM" - url: https://www.linkedin.com/in/reffajnaahsi/ - image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg - - name: Krrish Dholakia - title: "CEO, LiteLLM" - url: https://www.linkedin.com/in/krish-d/ - image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg + - sameer + - ishaan-alt + - krrish description: "Day 0 support for Claude Opus 4.6 on LiteLLM AI Gateway - use across Anthropic, Azure, Vertex AI, and Bedrock." tags: [anthropic, claude, opus 4.6] hide_table_of_contents: false @@ -25,6 +16,8 @@ import TabItem from '@theme/TabItem'; LiteLLM now supports Claude Opus 4.6 on Day 0. Use it across Anthropic, Azure, Vertex AI, and Bedrock through the LiteLLM AI Gateway. +{/* truncate */} + ## Docker Image ```bash diff --git a/docs/my-website/blog/claude_sonnet_4_6/index.md b/docs/my-website/blog/claude_sonnet_4_6/index.md index df54fa09792..12446c82c60 100644 --- a/docs/my-website/blog/claude_sonnet_4_6/index.md +++ b/docs/my-website/blog/claude_sonnet_4_6/index.md @@ -3,14 +3,8 @@ slug: claude_sonnet_4_6 title: "Day 0 Support: Claude Sonnet 4.6" date: 2026-02-17T10:00:00 authors: - - name: Ishaan Jaff - title: "CTO, LiteLLM" - url: https://www.linkedin.com/in/reffajnaahsi/ - image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg - - name: Krrish Dholakia - title: "CEO, LiteLLM" - url: https://www.linkedin.com/in/krish-d/ - image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg + - ishaan-alt + - krrish description: "Day 0 support for Claude Sonnet 4.6 on LiteLLM AI Gateway - use across Anthropic, Azure, Vertex AI, and Bedrock." tags: [anthropic, claude, sonnet 4.6] hide_table_of_contents: false @@ -21,6 +15,8 @@ import TabItem from '@theme/TabItem'; LiteLLM now supports Claude Sonnet 4.6 on Day 0. Use it across Anthropic, Azure, Vertex AI, and Bedrock through the LiteLLM AI Gateway. +{/* truncate */} + ## Docker Image ```bash diff --git a/docs/my-website/blog/fastapi_middleware_performance/index.mdx b/docs/my-website/blog/fastapi_middleware_performance/index.mdx index b0c5ba13634..e373326c071 100644 --- a/docs/my-website/blog/fastapi_middleware_performance/index.mdx +++ b/docs/my-website/blog/fastapi_middleware_performance/index.mdx @@ -3,18 +3,9 @@ slug: fastapi-middleware-performance title: "Your Middleware Could Be a Bottleneck" date: 2026-02-07T10:00:00 authors: - - name: Krrish Dholakia - title: "CEO, LiteLLM" - url: https://www.linkedin.com/in/krish-d/ - image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg - - name: Ishaan Jaff - title: "CTO, LiteLLM" - url: https://www.linkedin.com/in/reffajnaahsi/ - image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg - - name: Ryan Crabbe - title: "Performance Engineer, LiteLLM" - url: https://www.linkedin.com/in/ryan-crabbe-0b9687214 - image_url: https://media.licdn.com/dms/image/v2/D5603AQHt1t9Z4BJ6Gw/profile-displayphoto-shrink_400_400/profile-displayphoto-shrink_400_400/0/1724453682340?e=1772064000&v=beta&t=VXdmr13rsNB05wyA2F1TENOB5UuDHUZ0FCHTolNyR5M + - krrish + - ishaan-alt + - ryan description: "How we improved LiteLLM proxy latency and throughput by replacing a single middleware base class" tags: [performance, fastapi, middleware] hide_table_of_contents: false diff --git a/docs/my-website/blog/gemin_3.1/index.md b/docs/my-website/blog/gemin_3.1/index.md index b81595e4bd5..0afccb49d98 100644 --- a/docs/my-website/blog/gemin_3.1/index.md +++ b/docs/my-website/blog/gemin_3.1/index.md @@ -3,18 +3,9 @@ slug: gemini_3_1_pro title: "DAY 0 Support: Gemini 3.1 Pro on LiteLLM" date: 2026-02-19T10:00:00 authors: - - name: Sameer Kankute - title: SWE @ LiteLLM (LLM Translation) - url: https://www.linkedin.com/in/sameer-kankute/ - image_url: https://pbs.twimg.com/profile_images/2001352686994907136/ONgNuSk5_400x400.jpg - - name: Krrish Dholakia - title: "CEO, LiteLLM" - url: https://www.linkedin.com/in/krish-d/ - image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg - - name: Ishaan Jaff - title: "CTO, LiteLLM" - url: https://www.linkedin.com/in/reffajnaahsi/ - image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg + - sameer + - krrish + - ishaan-alt description: "Guide to using Gemini 3.1 Pro on LiteLLM Proxy and SDK with day 0 support." tags: [gemini, day 0 support, llms] hide_table_of_contents: false @@ -28,6 +19,8 @@ import TabItem from '@theme/TabItem'; LiteLLM now supports `gemini-3.1-pro-preview` and all the new API changes along with it. +{/* truncate */} + ## Deploy this version @@ -67,7 +60,7 @@ LiteLLM provides **full end-to-end support** for Gemini 3.1 Pro on: - ✅ `/v1/chat/completions` - OpenAI-compatible chat completions endpoint - ✅ `/v1/responses` - OpenAI Responses API endpoint (streaming and non-streaming) - ✅ [`/v1/messages`](../../docs/anthropic_unified) - Anthropic-compatible messages endpoint -- ✅ `/v1/generateContent` – [Google Gemini API](../../docs/generateContent.md) compatible endpoint +- ✅ `/v1/generateContent` – [Google Gemini API](../../docs/generateContent) compatible endpoint All endpoints support: - Streaming and non-streaming responses @@ -147,4 +140,3 @@ curl -X POST http://localhost:4000/v1/chat/completions \ | `high` | `high` | | `disable` | `minimal` | | `none` | `minimal` | - diff --git a/docs/my-website/blog/gemini_3/index.md b/docs/my-website/blog/gemini_3/index.md index 7263acc12c9..a5b94382b6f 100644 --- a/docs/my-website/blog/gemini_3/index.md +++ b/docs/my-website/blog/gemini_3/index.md @@ -3,18 +3,9 @@ slug: gemini_3 title: "DAY 0 Support: Gemini 3 on LiteLLM" date: 2025-11-19T10:00:00 authors: - - name: Sameer Kankute - title: SWE @ LiteLLM (LLM Translation) - url: https://www.linkedin.com/in/sameer-kankute/ - image_url: https://pbs.twimg.com/profile_images/2001352686994907136/ONgNuSk5_400x400.jpg - - name: Krrish Dholakia - title: "CEO, LiteLLM" - url: https://www.linkedin.com/in/krish-d/ - image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg - - name: Ishaan Jaff - title: "CTO, LiteLLM" - url: https://www.linkedin.com/in/reffajnaahsi/ - image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg + - sameer + - krrish + - ishaan-alt description: "Common questions and best practices for using gemini-3-pro-preview with LiteLLM Proxy and SDK." tags: [gemini, day 0 support, llms] hide_table_of_contents: false @@ -29,6 +20,8 @@ This guide covers common questions and best practices for using `gemini-3-pro-pr ::: +{/* truncate */} + ## Quick Start @@ -976,8 +969,7 @@ messages.append(response.choices[0].message) # ✅ Includes thought signatures ## Additional Resources -- [Gemini Provider Documentation](../gemini.md) -- [Thought Signatures Guide](../gemini.md#thought-signatures) -- [Reasoning Content Documentation](../../reasoning_content.md) -- [Function Calling Guide](../../function_calling.md) - +- [Gemini Provider Documentation](../../docs/providers/gemini) +- [Thought Signatures Guide](../../docs/providers/gemini#thought-signatures) +- [Reasoning Content Documentation](../../docs/reasoning_content) +- [Function Calling Guide](../../docs/completion/function_call) diff --git a/docs/my-website/blog/gemini_3_1_flash_lite/index.md b/docs/my-website/blog/gemini_3_1_flash_lite/index.md index 9ef4bacb2ad..0ae79e5fa67 100644 --- a/docs/my-website/blog/gemini_3_1_flash_lite/index.md +++ b/docs/my-website/blog/gemini_3_1_flash_lite/index.md @@ -3,18 +3,9 @@ slug: gemini_3_1_flash_lite_preview title: "DAY 0 Support: Gemini 3.1 Flash Lite Preview on LiteLLM" date: 2026-03-03T08:00:00 authors: - - name: Sameer Kankute - title: SWE @ LiteLLM (LLM Translation) - url: https://www.linkedin.com/in/sameer-kankute/ - image_url: https://pbs.twimg.com/profile_images/2001352686994907136/ONgNuSk5_400x400.jpg - - name: Krrish Dholakia - title: "CEO, LiteLLM" - url: https://www.linkedin.com/in/krish-d/ - image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg - - name: Ishaan Jaff - title: "CTO, LiteLLM" - url: https://www.linkedin.com/in/reffajnaahsi/ - image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg + - sameer + - krrish + - ishaan-alt description: "Guide to using Gemini 3.1 Flash Lite Preview on LiteLLM Proxy and SDK with day 0 support." tags: [gemini, day 0 support, llms, supernova] hide_table_of_contents: false @@ -32,6 +23,8 @@ LiteLLM now supports `gemini-3.1-flash-lite-preview` with full day 0 support! If you only want cost tracking, you need no change in your current Litellm version. But if you want the support for new features introduced along with it like thinking levels, you will need to use v1.80.8-stable.1 or above. ::: +{/* truncate */} + ## Deploy this version @@ -150,7 +143,7 @@ LiteLLM provides **full end-to-end support** for Gemini 3.1 Flash Lite Preview o - ✅ `/v1/chat/completions` - OpenAI-compatible chat completions endpoint - ✅ `/v1/responses` - OpenAI Responses API endpoint (streaming and non-streaming) - ✅ [`/v1/messages`](../../docs/anthropic_unified) - Anthropic-compatible messages endpoint -- ✅ `/v1/generateContent` – [Google Gemini API](../../docs/generateContent.md) compatible endpoint +- ✅ `/v1/generateContent` – [Google Gemini API](../../docs/generateContent) compatible endpoint All endpoints support: - Streaming and non-streaming responses @@ -172,4 +165,4 @@ LiteLLM automatically maps OpenAI's `reasoning_effort` parameter to Gemini's `th | `medium` | `medium` | Balanced reasoning for moderate complexity | | `high` | `high` | Maximum reasoning depth, complex problems | | `disable` | `minimal` | Disable extended reasoning | -| `none` | `minimal` | No extended reasoning | \ No newline at end of file +| `none` | `minimal` | No extended reasoning | diff --git a/docs/my-website/blog/gemini_3_flash/index.md b/docs/my-website/blog/gemini_3_flash/index.md index 830c21e5f66..5e98d2136b4 100644 --- a/docs/my-website/blog/gemini_3_flash/index.md +++ b/docs/my-website/blog/gemini_3_flash/index.md @@ -3,18 +3,9 @@ slug: gemini_3_flash title: "DAY 0 Support: Gemini 3 Flash on LiteLLM" date: 2025-12-17T10:00:00 authors: - - name: Sameer Kankute - title: SWE @ LiteLLM (LLM Translation) - url: https://www.linkedin.com/in/sameer-kankute/ - image_url: https://pbs.twimg.com/profile_images/2001352686994907136/ONgNuSk5_400x400.jpg - - name: Krrish Dholakia - title: "CEO, LiteLLM" - url: https://www.linkedin.com/in/krish-d/ - image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg - - name: Ishaan Jaff - title: "CTO, LiteLLM" - url: https://www.linkedin.com/in/reffajnaahsi/ - image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg + - sameer + - krrish + - ishaan-alt description: "Guide to using Gemini 3 Flash on LiteLLM Proxy and SDK with day 0 support." tags: [gemini, day 0 support, llms] hide_table_of_contents: false @@ -32,6 +23,8 @@ LiteLLM now supports `gemini-3-flash-preview` and all the new API changes along If you only want cost tracking, you need no change in your current Litellm version. But if you want the support for new features introduced along with it like thinking levels, you will need to use v1.80.8-stable.1 or above. ::: +{/* truncate */} + ## Deploy this version @@ -80,7 +73,7 @@ LiteLLM provides **full end-to-end support** for Gemini 3 Flash on: - ✅ `/v1/chat/completions` - OpenAI-compatible chat completions endpoint - ✅ `/v1/responses` - OpenAI Responses API endpoint (streaming and non-streaming) - ✅ [`/v1/messages`](../../docs/anthropic_unified) - Anthropic-compatible messages endpoint -- ✅ `/v1/generateContent` – [Google Gemini API](../../docs/generateContent.md) compatible endpoint +- ✅ `/v1/generateContent` – [Google Gemini API](../../docs/generateContent) compatible endpoint All endpoints support: - Streaming and non-streaming responses - Function calling with thought signatures @@ -252,4 +245,3 @@ If using this model via vertex_ai, keep the location as global as this is the on | `high` | `high` | | `disable` | `minimal` | | `none` | `minimal` | - diff --git a/docs/my-website/blog/gemini_embedding_2_multimodal/index.md b/docs/my-website/blog/gemini_embedding_2_multimodal/index.md index 8c09432e3b6..d66de1b6c79 100644 --- a/docs/my-website/blog/gemini_embedding_2_multimodal/index.md +++ b/docs/my-website/blog/gemini_embedding_2_multimodal/index.md @@ -3,10 +3,7 @@ slug: gemini_embedding_2_multimodal title: "Gemini Embedding 2 Preview: Multimodal Embeddings on LiteLLM" date: 2025-03-11T10:00:00 authors: - - name: Sameer Kankute - title: SWE @ LiteLLM (LLM Translation) - url: https://www.linkedin.com/in/sameer-kankute/ - image_url: https://pbs.twimg.com/profile_images/2001352686994907136/ONgNuSk5_400x400.jpg + - sameer description: "Generate embeddings from text, images, audio, video, and PDFs with gemini-embedding-2-preview on LiteLLM via Gemini API and Vertex AI." tags: [gemini, embeddings, multimodal, vertex ai] hide_table_of_contents: false @@ -19,6 +16,8 @@ import TabItem from '@theme/TabItem'; LiteLLM now supports **multimodal embeddings** with `gemini-embedding-2-preview`—generating a single embedding from a mix of text, images, audio, video, and PDF content. Available via both the **Gemini API** (API key) and **Vertex AI** (GCP credentials). +{/* truncate */} + ## Supported Input Types | Modality | Supported Formats | diff --git a/docs/my-website/blog/gpt_5_3_codex/index.md b/docs/my-website/blog/gpt_5_3_codex/index.md index 850586538f6..1dfab1f7ac7 100644 --- a/docs/my-website/blog/gpt_5_3_codex/index.md +++ b/docs/my-website/blog/gpt_5_3_codex/index.md @@ -3,18 +3,9 @@ slug: gpt_5_3_codex title: "Day 0 Support: GPT-5.3-Codex" date: 2026-02-24T10:00:00 authors: - - name: Sameer Kankute - title: SWE @ LiteLLM (LLM Translation) - url: https://www.linkedin.com/in/sameer-kankute/ - image_url: https://pbs.twimg.com/profile_images/2001352686994907136/ONgNuSk5_400x400.jpg - - name: Krrish Dholakia - title: "CEO, LiteLLM" - url: https://www.linkedin.com/in/krish-d/ - image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg - - name: Ishaan Jaff - title: "CTO, LiteLLM" - url: https://www.linkedin.com/in/reffajnaahsi/ - image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg + - sameer + - krrish + - ishaan-alt description: "Day 0 support for GPT-5.3-Codex on LiteLLM, including phase parameter handling for Responses API." tags: [openai, gpt-5.3-codex, codex, day 0 support] hide_table_of_contents: false @@ -25,6 +16,8 @@ import TabItem from '@theme/TabItem'; LiteLLM now supports GPT-5.3-Codex on Day 0, including support for the new assistant `phase` metadata on Responses API output items. +{/* truncate */} + ## Why `phase` matters for GPT-5.3-Codex `phase` appears on assistant output items and helps distinguish preamble/commentary turns from final closeout responses. diff --git a/docs/my-website/blog/gpt_5_4/index.md b/docs/my-website/blog/gpt_5_4/index.md index de099736f00..4f7e4344157 100644 --- a/docs/my-website/blog/gpt_5_4/index.md +++ b/docs/my-website/blog/gpt_5_4/index.md @@ -3,18 +3,9 @@ slug: gpt_5_4 title: "Day 0 Support: GPT-5.4" date: 2026-03-05T10:00:00 authors: - - name: Sameer Kankute - title: SWE @ LiteLLM (LLM Translation) - url: https://www.linkedin.com/in/sameer-kankute/ - image_url: https://pbs.twimg.com/profile_images/2001352686994907136/ONgNuSk5_400x400.jpg - - name: Krrish Dholakia - title: "CEO, LiteLLM" - url: https://www.linkedin.com/in/krish-d/ - image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg - - name: Ishaan Jaff - title: "CTO, LiteLLM" - url: https://www.linkedin.com/in/reffajnaahsi/ - image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg + - sameer + - krrish + - ishaan-alt description: "GPT-5.4 model support in LiteLLM" tags: [openai, gpt-5.4, completion] hide_table_of_contents: false @@ -25,6 +16,8 @@ import TabItem from '@theme/TabItem'; LiteLLM now supports fully GPT-5.4! +{/* truncate */} + ## Docker Image ```bash diff --git a/docs/my-website/blog/guardrail_logging_secret_exposure_incident/index.md b/docs/my-website/blog/guardrail_logging_secret_exposure_incident/index.md new file mode 100644 index 00000000000..71f9e3da011 --- /dev/null +++ b/docs/my-website/blog/guardrail_logging_secret_exposure_incident/index.md @@ -0,0 +1,78 @@ +--- +slug: guardrail-logging-secret-exposure-incident +title: "Incident Report: Guardrail logging exposed secret headers in spend logs and traces" +date: 2026-03-18T10:00:00 +authors: + - litellm +tags: [incident-report, security, guardrails] +hide_table_of_contents: false +--- + +**Date:** March 18, 2026 +**Duration:** Unknown +**Severity:** High +**Status:** Resolved + +## Summary + +When a custom guardrail returned the full LiteLLM request/data dictionary, the guardrail response logged by LiteLLM could include `secret_fields.raw_headers`, including plaintext `Authorization` headers containing API keys or other credentials. + +This information could then propagate to logging and observability surfaces that consume guardrail metadata, including: + +- **Spend logs in the LiteLLM UI:** visible to admins with access to spend-log data +- **OpenTelemetry traces:** visible to anyone with access to the relevant telemetry backend + +LLM calls, proxy routing, and provider execution were not blocked by this bug. The impact was exposure of sensitive request headers in observability and logging paths. + +{/* truncate */} + +--- + +## Background + +LiteLLM keeps internal request data (including request headers) for use during the call. That data is not meant to be written to logs or telemetry. + +When custom guardrails run, their outcomes are logged so they can appear in spend logs, OpenTelemetry traces, and other observability backends. If a guardrail returned the full request payload instead of a minimal result, that internal request data could be included in what was logged. Before the fix, the guardrail logging path did not strip that data before sending it to those systems. + +```mermaid +flowchart TD + inboundRequest["1. Incoming proxy request"] --> storeSecrets["2. Store internal request data"] + storeSecrets --> guardrailRuns["3. Custom guardrail runs"] + guardrailRuns --> fullDataReturn["4. Guardrail returns full request payload"] + fullDataReturn --> loggingBuild["5. Build guardrail log payload"] + loggingBuild --> spendLogs["6a. Persist to spend logs / UI"] + loggingBuild --> otelTraces["6b. Attach to OTEL guardrail spans"] +``` + +--- + +## Root Cause + +The root cause was incomplete sanitization in the guardrail logging path. When building the payload that gets sent to spend logs and traces, LiteLLM prepared guardrail responses for logging but did not strip internal request data (such as headers) from them. If a guardrail returned a response that included that data, it was passed through to the logging and observability systems unchanged. + +--- + +## Impact + +This issue required all of the following: + +1. A custom guardrail returned the full LiteLLM request/data dictionary, or another response object containing `secret_fields`. +2. LiteLLM logged that guardrail response through the standard guardrail logging path. +3. An operator, admin, or telemetry consumer had access to the resulting logs or traces. + +When those conditions were met, sensitive values could become visible through: + +- **Spend logs / UI responses:** guardrail metadata could be included in spend-log payloads rendered in the admin UI. +- **OpenTelemetry traces:** `guardrail_response` could be written as a span attribute on guardrail spans. +- **Other downstream observability backends:** any integration consuming the same guardrail metadata could receive the leaked values. + +This was a logging and telemetry exposure bug. It did not let callers bypass auth, access other tenants directly, or change model behavior, but it could expose plaintext credentials to people with access to those observability systems. + +--- + +## Guidance For Users + +- Upgrade to LiteLLM 1.82.3+. +- If you operated custom guardrails that return the full request/data dict, review whether spend logs or telemetry traces were retained during the affected period. +- Rotate any credentials that may have appeared in `Authorization` or other forwarded request headers in those systems. +- Apply least-privilege access controls to spend-log views and telemetry backends that may contain request-derived metadata. diff --git a/docs/my-website/blog/httpx_cache_eviction_incident/index.md b/docs/my-website/blog/httpx_cache_eviction_incident/index.md index 9e6152d0e63..7fc3789a91d 100644 --- a/docs/my-website/blog/httpx_cache_eviction_incident/index.md +++ b/docs/my-website/blog/httpx_cache_eviction_incident/index.md @@ -3,17 +3,9 @@ slug: httpx-cache-eviction-incident title: "Incident Report: Cache Eviction Closes In-Use httpx Clients" date: 2026-02-27T10:00:00 authors: - - name: Ryan Crabbe - title: Performance Engineer, LiteLLM - url: https://www.linkedin.com/in/ryan-crabbe-0b9687214 - - name: Ishaan Jaff - title: "CTO, LiteLLM" - url: https://www.linkedin.com/in/reffajnaahsi/ - image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg - - name: Krrish Dholakia - title: "CEO, LiteLLM" - url: https://www.linkedin.com/in/krish-d/ - image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg + - ryan + - ishaan-alt + - krrish tags: [incident-report, caching, stability] hide_table_of_contents: false --- @@ -31,6 +23,8 @@ A change to improve Redis connection pool cleanup introduced a regression that c **Impact:** Any proxy instance that hit the cache TTL (default 10 minutes) or capacity limit (200 entries) would have its httpx clients closed out from under it, causing requests to LLM providers to fail with connection errors. +{/* truncate */} + --- ## Background diff --git a/docs/my-website/blog/litellm_observatory/index.md b/docs/my-website/blog/litellm_observatory/index.md index 4554f77fb85..36366e5de22 100644 --- a/docs/my-website/blog/litellm_observatory/index.md +++ b/docs/my-website/blog/litellm_observatory/index.md @@ -3,18 +3,9 @@ slug: litellm-observatory title: "Improve release stability with 24 hour load tests" date: 2026-02-06T10:00:00 authors: - - name: Alexsander Hamir - title: "Performance Engineer, LiteLLM" - url: https://www.linkedin.com/in/alexsander-baptista/ - image_url: https://github.com/AlexsanderHamir.png - - name: Krrish Dholakia - title: "CEO, LiteLLM" - url: https://www.linkedin.com/in/krish-d/ - image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg - - name: Ishaan Jaff - title: "CTO, LiteLLM" - url: https://www.linkedin.com/in/reffajnaahsi/ - image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg + - alexsander + - krrish + - ishaan-alt description: "How we built a long-running, release-validation system to catch regressions before they reach users." tags: [testing, observability, reliability, releases] hide_table_of_contents: false @@ -28,6 +19,8 @@ As LiteLLM adoption has grown, so have expectations around reliability, performa This post introduces **LiteLLM Observatory**, a long-running release-validation system we built to catch regressions before they reach users. +{/* truncate */} + --- ## Why We Built the Observatory @@ -133,4 +126,3 @@ Reliability is an ongoing investment. LiteLLM Observatory is one of several systems we’re building to continuously raise the bar on release quality and operational safety. As LiteLLM evolves, so will our validation tooling, informed by real-world usage and lessons learned. We’ll continue to share those improvements openly as we go. - diff --git a/docs/my-website/blog/minimax_m2_5/index.md b/docs/my-website/blog/minimax_m2_5/index.md index 50084fcc1e5..9bccbdf7979 100644 --- a/docs/my-website/blog/minimax_m2_5/index.md +++ b/docs/my-website/blog/minimax_m2_5/index.md @@ -3,18 +3,9 @@ slug: minimax_m2_5 title: "Day 0 Support: MiniMax-M2.5" date: 2026-02-12T10:00:00 authors: - - name: Sameer Kankute - title: SWE @ LiteLLM (LLM Translation) - url: https://www.linkedin.com/in/sameer-kankute/ - image_url: https://pbs.twimg.com/profile_images/2001352686994907136/ONgNuSk5_400x400.jpg - - name: Krrish Dholakia - title: "CEO, LiteLLM" - url: https://www.linkedin.com/in/krish-d/ - image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg - - name: Ishaan Jaff - title: "CTO, LiteLLM" - url: https://www.linkedin.com/in/reffajnaahsi/ - image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg + - sameer + - krrish + - ishaan-alt description: "Day 0 support for MiniMax-M2.5 on LiteLLM" tags: [minimax, M2.5, llm] hide_table_of_contents: false @@ -25,6 +16,8 @@ import TabItem from '@theme/TabItem'; LiteLLM now supports MiniMax-M2.5 on Day 0. Use it across OpenAI-compatible and Anthropic-compatible APIs through the LiteLLM AI Gateway. +{/* truncate */} + ## Supported Models LiteLLM supports the following MiniMax models: diff --git a/docs/my-website/blog/model_cost_map_incident/index.md b/docs/my-website/blog/model_cost_map_incident/index.md index b9ff20e4128..5b4499cc31c 100644 --- a/docs/my-website/blog/model_cost_map_incident/index.md +++ b/docs/my-website/blog/model_cost_map_incident/index.md @@ -3,10 +3,7 @@ slug: model-cost-map-incident title: "Incident Report: Invalid model cost map on main" date: 2026-02-10T10:00:00 authors: - - name: Ishaan Jaffer - title: "CTO, LiteLLM" - url: https://www.linkedin.com/in/ishaanjaffer/ - image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg + - ishaan tags: [incident-report, stability] hide_table_of_contents: false --- diff --git a/docs/my-website/blog/realtime_webrtc_http_endpoints/index.md b/docs/my-website/blog/realtime_webrtc_http_endpoints/index.md new file mode 100644 index 00000000000..70fc5b2c48e --- /dev/null +++ b/docs/my-website/blog/realtime_webrtc_http_endpoints/index.md @@ -0,0 +1,111 @@ +--- +slug: realtime_webrtc_http_endpoints +title: "Realtime WebRTC HTTP Endpoints" +date: 2026-03-12T10:00:00 +authors: + - sameer + - krrish + - ishaan-alt +description: "Use the LiteLLM proxy to route OpenAI-style WebRTC realtime via HTTP: client_secrets and SDP exchange." +tags: [realtime, webrtc, proxy, openai] +hide_table_of_contents: false +--- + +import WebRTCTester from '@site/src/components/WebRTCTester'; + +Connect to the Realtime API via WebRTC from browser/mobile clients. LiteLLM handles auth and key management. + +{/* truncate */} + +## How it works + +![WebRTC flow: Browser, LiteLLM Proxy, and OpenAI/Azure](../../img/webrtc_flow.png) + +**Flow of generating ephemeral token** + +![Ephemeral token flow: Browser requests token, LiteLLM gets real token from OpenAI, returns encrypted token](../../img/ephemeral_token.png) + + +## Proxy Setup + +```yaml +model_list: + - model_name: gpt-4o-realtime + litellm_params: + model: openai/gpt-4o-realtime-preview-2024-12-17 + api_key: os.environ/OPENAI_API_KEY + model_info: + mode: realtime +``` + +**Azure:** use `model: azure/gpt-4o-realtime-preview`, `api_key`, `api_base`. + +```bash +litellm --config /path/to/config.yaml +``` + +## Try it live + + + +## Client Usage + +**1. Get token** - `POST /v1/realtime/client_secrets` with LiteLLM API key and `{ model }`. + +**2. WebRTC handshake** - Create `RTCPeerConnection`, add mic track, create data channel `oai-events`, send SDP offer to `POST /v1/realtime/calls` with `Authorization: Bearer ` and `Content-Type: application/sdp`. + +**3. Events** - Use the data channel for `session.update` and other events. + +
+Full code example + +```javascript +// 1. Token +const r = await fetch("http://proxy:4000/v1/realtime/client_secrets", { + method: "POST", + headers: { "Authorization": "Bearer sk-litellm-key", "Content-Type": "application/json" }, + body: JSON.stringify({ model: "gpt-4o-realtime" }), +}); +const { client_secret } = await r.json(); +const token = client_secret.value; + +// 2. WebRTC +const pc = new RTCPeerConnection(); +const audio = document.createElement("audio"); +audio.autoplay = true; +pc.ontrack = (e) => (audio.srcObject = e.streams[0]); +const ms = await navigator.mediaDevices.getUserMedia({ audio: true }); +pc.addTrack(ms.getTracks()[0]); +const dc = pc.createDataChannel("oai-events"); +const offer = await pc.createOffer(); +await pc.setLocalDescription(offer); + +const sdpRes = await fetch("http://proxy:4000/v1/realtime/calls", { + method: "POST", + headers: { "Authorization": `Bearer ${token}`, "Content-Type": "application/sdp" }, + body: offer.sdp, +}); +await pc.setRemoteDescription({ type: "answer", sdp: await sdpRes.text() }); + +// 3. Events +dc.send(JSON.stringify({ type: "session.update", session: { instructions: "..." } })); +``` + +
+ +## FAQ + +**Q: What do I do if I get a 401 Token expired error?** +A: Tokens are short-lived. Get a fresh token right before creating the WebRTC offer. + +**Q: Which key should I use for `/v1/realtime/calls`?** +A: Use the **encrypted token** from `client_secrets`, not your raw API key. + +**Q: Should I pass the `model` parameter when making the call?** +A: No, the encrypted token already encodes all routing information including model. + +**Q: How do I resolve Azure `api-version` errors?** +A: Set the correct `api_version` in `litellm_params` (or via the `AZURE_API_VERSION` environment variable), along with the right `api_base` and deployment values. + +**Q: What if I get no audio?** +A: Make sure you grant microphone permission, ensure `pc.ontrack` assigns the audio element with `autoplay` enabled, check your network/firewall for WebRTC traffic, and inspect the browser console for ICE or SDP errors. diff --git a/docs/my-website/blog/responses_api_encrypted_content_incident/index.md b/docs/my-website/blog/responses_api_encrypted_content_incident/index.md index 19b55898caa..fd5a9e76c42 100644 --- a/docs/my-website/blog/responses_api_encrypted_content_incident/index.md +++ b/docs/my-website/blog/responses_api_encrypted_content_incident/index.md @@ -3,18 +3,9 @@ slug: responses-api-encrypted-content-incident title: "Incident Report: Encrypted Content Failures in Multi-Region Responses API Load Balancing" date: 2026-02-24T10:00:00 authors: - - name: Sameer Kankute - title: SWE @ LiteLLM (LLM Translation) - url: https://www.linkedin.com/in/sameer-kankute/ - image_url: https://pbs.twimg.com/profile_images/2001352686994907136/ONgNuSk5_400x400.jpg - - name: Krrish Dholakia - title: "CEO, LiteLLM" - url: https://www.linkedin.com/in/krish-d/ - image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg - - name: Ishaan Jaff - title: "CTO, LiteLLM" - url: https://www.linkedin.com/in/reffajnaahsi/ - image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg + - sameer + - krrish + - ishaan-alt tags: [incident-report, proxy, responses-api, load-balancing] hide_table_of_contents: false --- diff --git a/docs/my-website/blog/server_root_path/index.md b/docs/my-website/blog/server_root_path/index.md index d7925baf6b4..13b7365cc9e 100644 --- a/docs/my-website/blog/server_root_path/index.md +++ b/docs/my-website/blog/server_root_path/index.md @@ -3,17 +3,9 @@ slug: server-root-path-incident title: "Incident Report: SERVER_ROOT_PATH regression broke UI routing" date: 2026-02-21T10:00:00 authors: - - name: Yuneng Jiang - title: SWE @ LiteLLM (Full Stack) - url: https://www.linkedin.com/in/yunengjiang/ - - name: Ishaan Jaff - title: "CTO, LiteLLM" - url: https://www.linkedin.com/in/reffajnaahsi/ - image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg - - name: Krrish Dholakia - title: "CEO, LiteLLM" - url: https://www.linkedin.com/in/krish-d/ - image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg + - yuneng + - ishaan-alt + - krrish tags: [incident-report, ui, stability] hide_table_of_contents: false --- diff --git a/docs/my-website/blog/sub_millisecond_proxy_overhead/index.md b/docs/my-website/blog/sub_millisecond_proxy_overhead/index.md index 1857383363c..7f8ac086a21 100644 --- a/docs/my-website/blog/sub_millisecond_proxy_overhead/index.md +++ b/docs/my-website/blog/sub_millisecond_proxy_overhead/index.md @@ -3,18 +3,9 @@ slug: sub-millisecond-proxy-overhead title: "Achieving Sub-Millisecond Proxy Overhead" date: 2026-02-02T10:00:00 authors: - - name: Alexsander Hamir - title: "Performance Engineer, LiteLLM" - url: https://www.linkedin.com/in/alexsander-baptista/ - image_url: https://github.com/AlexsanderHamir.png - - name: Krrish Dholakia - title: "CEO, LiteLLM" - url: https://www.linkedin.com/in/krish-d/ - image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg - - name: Ishaan Jaff - title: "CTO, LiteLLM" - url: https://www.linkedin.com/in/reffajnaahsi/ - image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg + - alexsander + - krrish + - ishaan-alt description: "Our Q1 performance target and architectural direction for achieving sub-millisecond proxy overhead on modest hardware." tags: [performance, architecture] hide_table_of_contents: false @@ -32,6 +23,8 @@ Proxy overhead refers to the latency introduced by LiteLLM itself, independent o To measure it, we run the same workload directly against the provider and through LiteLLM at identical QPS (for example, 1,000 QPS) and compare the latency delta. To reduce noise, the load generator, LiteLLM, and a mock LLM endpoint all run on the same machine, ensuring the difference reflects proxy overhead rather than network latency. +{/* truncate */} + --- ## Where We're Coming From diff --git a/docs/my-website/blog/video_characters_litellm/index.md b/docs/my-website/blog/video_characters_litellm/index.md new file mode 100644 index 00000000000..a0f87385d81 --- /dev/null +++ b/docs/my-website/blog/video_characters_litellm/index.md @@ -0,0 +1,121 @@ +--- +slug: video_characters_api +title: "New Video Characters, Edit and Extension API support" +date: 2026-03-16T10:00:00 +authors: + - sameer + - krrish + - ishaan-alt +description: "LiteLLM now supports creating, retrieving, and managing reusable video characters across multiple video generations." +tags: [videos, characters, proxy, routing] +hide_table_of_contents: false +--- + +LiteLLM now supoports videos character, edit and extension apis. + +{/* truncate */} + +## What's New + +Four new endpoints for video character operations: +- **Create character** - Upload a video to create a reusable asset +- **Get character** - Retrieve character metadata +- **Edit video** - Modify generated videos +- **Extend video** - Continue clips with character consistency + +**Available from:** LiteLLM v1.83.0+ + +## Quick Example + +```python +import litellm + +# Create character from video +character = litellm.avideo_create_character( + name="Luna", + video=open("luna.mp4", "rb"), + custom_llm_provider="openai", + model="sora-2" +) +print(f"Character: {character.id}") + +# Use in generation +video = litellm.avideo( + model="sora-2", + prompt="Luna dances through a magical forest.", + characters=[{"id": character.id}], + seconds="8" +) + +# Get character info +fetched = litellm.avideo_get_character( + character_id=character.id, + custom_llm_provider="openai" +) + +# Edit with character preserved +edited = litellm.avideo_edit( + video_id=video.id, + prompt="Add warm golden lighting" +) + +# Extend sequence +extended = litellm.avideo_extension( + video_id=video.id, + prompt="Luna waves goodbye", + seconds="5" +) +``` + +## Via Proxy + +```bash +# Create character +curl -X POST "http://localhost:4000/v1/videos/characters" \ + -H "Authorization: Bearer sk-litellm-key" \ + -F "video=@luna.mp4" \ + -F "name=Luna" + +# Get character +curl -X GET "http://localhost:4000/v1/videos/characters/char_abc123def456" \ + -H "Authorization: Bearer sk-litellm-key" + +# Edit video +curl -X POST "http://localhost:4000/v1/videos/edits" \ + -H "Authorization: Bearer sk-litellm-key" \ + -H "Content-Type: application/json" \ + -d '{ + "video": {"id": "video_xyz789"}, + "prompt": "Add warm golden lighting and enhance colors" + }' + +# Extend video +curl -X POST "http://localhost:4000/v1/videos/extensions" \ + -H "Authorization: Bearer sk-litellm-key" \ + -H "Content-Type: application/json" \ + -d '{ + "video": {"id": "video_xyz789"}, + "prompt": "Luna waves goodbye and walks into the sunset", + "seconds": "5" + }' +``` + +## Managed Character IDs + +LiteLLM automatically encodes provider and model metadata into character IDs: + +**What happens:** +``` +Upload character "Luna" with model "sora-2" on OpenAI + ↓ +LiteLLM creates: char_abc123def456 (contains provider + model_id) + ↓ +When you reference it later, LiteLLM decodes automatically + ↓ +Router knows exactly which deployment to use +``` + +**Behind the scenes:** +- Character ID format: `character_` +- Metadata includes: provider, model_id, original_character_id +- Transparent to you - just use the ID, LiteLLM handles routing diff --git a/docs/my-website/blog/vllm_embeddings_incident/index.md b/docs/my-website/blog/vllm_embeddings_incident/index.md index a1ce8152857..26387e66dd0 100644 --- a/docs/my-website/blog/vllm_embeddings_incident/index.md +++ b/docs/my-website/blog/vllm_embeddings_incident/index.md @@ -3,18 +3,9 @@ slug: vllm-embeddings-incident title: "Incident Report: vLLM Embeddings Broken by encoding_format Parameter" date: 2026-02-18T10:00:00 authors: - - name: Sameer Kankute - title: SWE @ LiteLLM (LLM Translation) - url: https://www.linkedin.com/in/sameer-kankute/ - image_url: https://pbs.twimg.com/profile_images/2001352686994907136/ONgNuSk5_400x400.jpg - - name: Krrish Dholakia - title: "CEO, LiteLLM" - url: https://www.linkedin.com/in/krish-d/ - image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg - - name: Ishaan Jaff - title: "CTO, LiteLLM" - url: https://www.linkedin.com/in/reffajnaahsi/ - image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg + - sameer + - krrish + - ishaan-alt tags: [incident-report, embeddings, vllm] hide_table_of_contents: false --- diff --git a/docs/my-website/docs/files_endpoints.md b/docs/my-website/docs/files_endpoints.md index 30677c748a9..deb17931638 100644 --- a/docs/my-website/docs/files_endpoints.md +++ b/docs/my-website/docs/files_endpoints.md @@ -326,4 +326,10 @@ print("file content=", content.text) ### [Bedrock](./providers/bedrock_batches#4-retrieve-batch-results) +### [Anthropic](./providers/anthropic#files-api) + +:::note +Anthropic Files API has a different purpose than OpenAI's. It's **not** for Batches or Fine-tuning—it's for uploading files once and referencing them by `file_id` in multiple messages, avoiding re-uploads. File API operations are free — file content used in Messages requests is priced as input tokens. +::: + ## [Swagger API Reference](https://litellm-api.up.railway.app/#/files) diff --git a/docs/my-website/docs/guides/index.md b/docs/my-website/docs/guides/index.md new file mode 100644 index 00000000000..1641600dab2 --- /dev/null +++ b/docs/my-website/docs/guides/index.md @@ -0,0 +1,78 @@ +--- +title: Guides +sidebar_label: Overview +--- + +import NavigationCards from '@site/src/components/NavigationCards'; + +**Guides** are focused references organized by the job you are trying to do with LiteLLM: make requests, use tools, handle media, manage context, or operate the gateway safely. + +> New to LiteLLM or not sure whether you need the SDK or Gateway path first? Start at [Learn →](/docs/learn) + +--- + +## Build With LiteLLM + + + +--- + +## Operate & Extend + + diff --git a/docs/my-website/docs/index.md b/docs/my-website/docs/index.md index ba605e316d3..ca63c9e39ff 100644 --- a/docs/my-website/docs/index.md +++ b/docs/my-website/docs/index.md @@ -1,689 +1,462 @@ +--- +id: index +title: Getting Started +sidebar_label: Quickstart +--- + import Tabs from '@theme/Tabs'; import TabItem from '@theme/TabItem'; +import NavigationCards from '@site/src/components/NavigationCards'; +import Image from '@theme/IdealImage'; -# LiteLLM - Getting Started + -https://github.com/BerriAI/litellm +**LiteLLM** is an open-source library that gives you a single, unified interface to call 100+ LLMs — OpenAI, Anthropic, Vertex AI, Bedrock, and more — using the OpenAI format. -## **Call 100+ LLMs using the OpenAI Input/Output Format** +- Call any provider using the same `completion()` interface — no re-learning the API for each one +- Consistent output format regardless of which provider or model you use +- Built-in retry / fallback logic across multiple deployments via the [Router](./routing.md) +- Self-hosted [LLM Gateway (Proxy)](./simple_proxy) with virtual keys, cost tracking, and an admin UI -- Translate inputs to provider's endpoints (`/chat/completions`, `/responses`, `/embeddings`, `/images`, `/audio`, `/batches`, and more) -- [Consistent output](https://docs.litellm.ai/docs/supported_endpoints) - same response format regardless of which provider you use -- Retry/fallback logic across multiple deployments (e.g. Azure/OpenAI) - [Router](https://docs.litellm.ai/docs/routing) -- Track spend & set budgets per project [LiteLLM Proxy Server](https://docs.litellm.ai/docs/simple_proxy) +[![PyPI](https://img.shields.io/pypi/v/litellm.svg)](https://pypi.org/project/litellm/) +[![GitHub Stars](https://img.shields.io/github/stars/BerriAI/litellm?style=social)](https://github.com/BerriAI/litellm) -## How to use LiteLLM +--- -You can use LiteLLM through either the Proxy Server or Python SDK. Both gives you a unified interface to access multiple LLMs (100+ LLMs). Choose the option that best fits your needs: - - - - - - - - - - - - - - - - - - - - - - - - - - -
LiteLLM Proxy ServerLiteLLM Python SDK
Use CaseCentral service (LLM Gateway) to access multiple LLMsUse LiteLLM directly in your Python code
Who Uses It?Gen AI Enablement / ML Platform TeamsDevelopers building LLM projects
Key Features• Centralized API gateway with authentication & authorization
• Multi-tenant cost tracking and spend management per project/user
• Per-project customization (logging, guardrails, caching)
• Virtual keys for secure access control
• Admin dashboard UI for monitoring and management		• Direct Python library integration in your codebase
• Router with retry/fallback logic across multiple deployments (e.g. Azure/OpenAI) - Router
• Application-level load balancing and cost tracking
• Exception handling with OpenAI-compatible errors
• Observability callbacks (Lunary, MLflow, Langfuse, etc.)
- - -## **LiteLLM Python SDK** - -### Basic usage - - - Open In Colab - +## Installation ```shell pip install litellm ``` - - - -```python -from litellm import completion -import os - -## set ENV variables -os.environ["OPENAI_API_KEY"] = "your-api-key" - -response = completion( - model="openai/gpt-4o", - messages=[{ "content": "Hello, how are you?","role": "user"}] -) -``` - - - - -```python -from litellm import completion -import os - -## set ENV variables -os.environ["ANTHROPIC_API_KEY"] = "your-api-key" - -response = completion( - model="anthropic/claude-3-sonnet-20240229", - messages=[{ "content": "Hello, how are you?","role": "user"}] -) -``` - - - - -```python -from litellm import completion -import os - -## set ENV variables -os.environ["XAI_API_KEY"] = "your-api-key" - -response = completion( - model="xai/grok-2-latest", - messages=[{ "content": "Hello, how are you?","role": "user"}] -) -``` - - - -```python -from litellm import completion -import os - -# auth: run 'gcloud auth application-default' -os.environ["VERTEXAI_PROJECT"] = "hardy-device-386718" -os.environ["VERTEXAI_LOCATION"] = "us-central1" - -response = completion( - model="vertex_ai/gemini-1.5-pro", - messages=[{ "content": "Hello, how are you?","role": "user"}] -) -``` - - - - - -```python -from litellm import completion -import os - -## set ENV variables -os.environ["NVIDIA_NIM_API_KEY"] = "nvidia_api_key" -os.environ["NVIDIA_NIM_API_BASE"] = "nvidia_nim_endpoint_url" - -response = completion( - model="nvidia_nim/", - messages=[{ "content": "Hello, how are you?","role": "user"}] -) -``` - - - - - -```python -from litellm import completion -import os - -os.environ["HUGGINGFACE_API_KEY"] = "huggingface_api_key" - -# e.g. Call 'WizardLM/WizardCoder-Python-34B-V1.0' hosted on HF Inference endpoints -response = completion( - model="huggingface/WizardLM/WizardCoder-Python-34B-V1.0", - messages=[{ "content": "Hello, how are you?","role": "user"}], - api_base="https://my-endpoint.huggingface.cloud" -) - -print(response) -``` - - - - - -```python -from litellm import completion -import os - -## set ENV variables -os.environ["AZURE_API_KEY"] = "" -os.environ["AZURE_API_BASE"] = "" -os.environ["AZURE_API_VERSION"] = "" - -# azure call -response = completion( - "azure/", - messages = [{ "content": "Hello, how are you?","role": "user"}] -) -``` - - - - - -```python -from litellm import completion - -response = completion( - model="ollama/llama2", - messages = [{ "content": "Hello, how are you?","role": "user"}], - api_base="http://localhost:11434" -) -``` - - - - -```python -from litellm import completion -import os - -## set ENV variables -os.environ["OPENROUTER_API_KEY"] = "openrouter_api_key" - -response = completion( - model="openrouter/google/palm-2-chat-bison", - messages = [{ "content": "Hello, how are you?","role": "user"}], -) -``` - - - - -```python -from litellm import completion -import os - -## set ENV variables. Visit https://novita.ai/settings/key-management to get your API key -os.environ["NOVITA_API_KEY"] = "novita-api-key" - -response = completion( - model="novita/deepseek/deepseek-r1", - messages=[{ "content": "Hello, how are you?","role": "user"}] -) -``` - - - - - -```python -from litellm import completion -import os - -## set ENV variables. Visit https://vercel.com/docs/ai-gateway#using-the-ai-gateway-with-an-api-key for insturctions on obtaining a key -os.environ["VERCEL_AI_GATEWAY_API_KEY"] = "your-vercel-api-key" - -response = completion( - model="vercel_ai_gateway/openai/gpt-4o", - messages=[{ "content": "Hello, how are you?","role": "user"}] -) -``` - - - - - -### Response Format (OpenAI Chat Completions Format) - -```json -{ - "id": "chatcmpl-565d891b-a42e-4c39-8d14-82a1f5208885", - "created": 1734366691, - "model": "gpt-4o-2024-08-06", - "object": "chat.completion", - "system_fingerprint": null, - "choices": [ - { - "finish_reason": "stop", - "index": 0, - "message": { - "content": "Hello! As an AI language model, I don't have feelings, but I'm operating properly and ready to assist you with any questions or tasks you may have. How can I help you today?", - "role": "assistant", - "tool_calls": null, - "function_call": null - } - } - ], - "usage": { - "completion_tokens": 43, - "prompt_tokens": 13, - "total_tokens": 56, - "completion_tokens_details": null, - "prompt_tokens_details": { - "audio_tokens": null, - "cached_tokens": 0 - }, - "cache_creation_input_tokens": 0, - "cache_read_input_tokens": 0 - } -} -``` - -### Streaming -Set `stream=True` in the `completion` args. - - - - -```python -from litellm import completion -import os - -## set ENV variables -os.environ["OPENAI_API_KEY"] = "your-api-key" - -response = completion( - model="openai/gpt-4o", - messages=[{ "content": "Hello, how are you?","role": "user"}], - stream=True, -) -``` - - - - -```python -from litellm import completion -import os - -## set ENV variables -os.environ["ANTHROPIC_API_KEY"] = "your-api-key" - -response = completion( - model="anthropic/claude-3-sonnet-20240229", - messages=[{ "content": "Hello, how are you?","role": "user"}], - stream=True, -) -``` - - - - -```python -from litellm import completion -import os - -## set ENV variables -os.environ["XAI_API_KEY"] = "your-api-key" - -response = completion( - model="xai/grok-2-latest", - messages=[{ "content": "Hello, how are you?","role": "user"}], - stream=True, -) -``` - - - -```python -from litellm import completion -import os - -# auth: run 'gcloud auth application-default' -os.environ["VERTEX_PROJECT"] = "hardy-device-386718" -os.environ["VERTEX_LOCATION"] = "us-central1" - -response = completion( - model="vertex_ai/gemini-1.5-pro", - messages=[{ "content": "Hello, how are you?","role": "user"}], - stream=True, -) -``` - - - - - -```python -from litellm import completion -import os - -## set ENV variables -os.environ["NVIDIA_NIM_API_KEY"] = "nvidia_api_key" -os.environ["NVIDIA_NIM_API_BASE"] = "nvidia_nim_endpoint_url" - -response = completion( - model="nvidia_nim/", - messages=[{ "content": "Hello, how are you?","role": "user"}] - stream=True, -) -``` - - - - -```python -from litellm import completion -import os - -os.environ["HUGGINGFACE_API_KEY"] = "huggingface_api_key" - -# e.g. Call 'WizardLM/WizardCoder-Python-34B-V1.0' hosted on HF Inference endpoints -response = completion( - model="huggingface/WizardLM/WizardCoder-Python-34B-V1.0", - messages=[{ "content": "Hello, how are you?","role": "user"}], - api_base="https://my-endpoint.huggingface.cloud", - stream=True, -) - -print(response) -``` - - - - - -```python -from litellm import completion -import os - -## set ENV variables -os.environ["AZURE_API_KEY"] = "" -os.environ["AZURE_API_BASE"] = "" -os.environ["AZURE_API_VERSION"] = "" - -# azure call -response = completion( - "azure/", - messages = [{ "content": "Hello, how are you?","role": "user"}], - stream=True, -) -``` - - - - - -```python -from litellm import completion - -response = completion( - model="ollama/llama2", - messages = [{ "content": "Hello, how are you?","role": "user"}], - api_base="http://localhost:11434", - stream=True, -) -``` - - - - -```python -from litellm import completion -import os - -## set ENV variables -os.environ["OPENROUTER_API_KEY"] = "openrouter_api_key" - -response = completion( - model="openrouter/google/palm-2-chat-bison", - messages = [{ "content": "Hello, how are you?","role": "user"}], - stream=True, -) -``` - - - - -```python -from litellm import completion -import os - -## set ENV variables. Visit https://novita.ai/settings/key-management to get your API key -os.environ["NOVITA_API_KEY"] = "novita_api_key" - -response = completion( - model="novita/deepseek/deepseek-r1", - messages = [{ "content": "Hello, how are you?","role": "user"}], - stream=True, -) -``` - - - - - -```python -from litellm import completion -import os - -## set ENV variables. Visit https://vercel.com/docs/ai-gateway#using-the-ai-gateway-with-an-api-key for insturctions on obtaining a key -os.environ["VERCEL_AI_GATEWAY_API_KEY"] = "your-vercel-api-key" - -response = completion( - model="vercel_ai_gateway/openai/gpt-4o", - messages = [{ "content": "Hello, how are you?","role": "user"}], - stream=True, -) -``` - - - - - -### Streaming Response Format (OpenAI Format) - -```json -{ - "id": "chatcmpl-2be06597-eb60-4c70-9ec5-8cd2ab1b4697", - "created": 1734366925, - "model": "claude-3-sonnet-20240229", - "object": "chat.completion.chunk", - "system_fingerprint": null, - "choices": [ - { - "finish_reason": null, - "index": 0, - "delta": { - "content": "Hello", - "role": "assistant", - "function_call": null, - "tool_calls": null, - "audio": null - }, - "logprobs": null - } - ] -} -``` - -### Exception handling - -LiteLLM maps exceptions across all supported providers to the OpenAI exceptions. All our exceptions inherit from OpenAI's exception types, so any error-handling you have for that, should work out of the box with LiteLLM. - -```python -import litellm -from litellm import completion -import os - -os.environ["ANTHROPIC_API_KEY"] = "bad-key" -try: - completion(model="anthropic/claude-instant-1", messages=[{"role": "user", "content": "Hey, how's it going?"}]) -except litellm.AuthenticationError as e: - # Thrown when the API key is invalid - print(f"Authentication failed: {e}") -except litellm.RateLimitError as e: - # Thrown when you've exceeded your rate limit - print(f"Rate limited: {e}") -except litellm.APIError as e: - # Thrown for general API errors - print(f"API error: {e}") -``` -### See How LiteLLM Transforms Your Requests - -Want to understand how LiteLLM parses and normalizes your LLM API requests? Use the `/utils/transform_request` endpoint to see exactly how your request is transformed internally. - -You can try it out now directly on our Demo App! -Go to the [LiteLLM API docs for transform_request](https://litellm-api.up.railway.app/#/llm%20utils/transform_request_utils_transform_request_post) - -LiteLLM will show you the normalized, provider-agnostic version of your request. This is useful for debugging, learning, and understanding how LiteLLM handles different providers and options. - - -### Logging Observability - Log LLM Input/Output ([Docs](https://docs.litellm.ai/docs/observability/callbacks)) -LiteLLM exposes pre defined callbacks to send data to Lunary, MLflow, Langfuse, Helicone, Promptlayer, Traceloop, Slack - -```python -from litellm import completion - -## set env variables for logging tools (API key set up is not required when using MLflow) -os.environ["LUNARY_PUBLIC_KEY"] = "your-lunary-public-key" # get your public key at https://app.lunary.ai/settings -os.environ["HELICONE_API_KEY"] = "your-helicone-key" -os.environ["LANGFUSE_PUBLIC_KEY"] = "" -os.environ["LANGFUSE_SECRET_KEY"] = "" - -os.environ["OPENAI_API_KEY"] - -# set callbacks -litellm.success_callback = ["lunary", "mlflow", "langfuse", "helicone"] # log input/output to lunary, mlflow, langfuse, helicone - -#openai call -response = completion(model="gpt-3.5-turbo", messages=[{"role": "user", "content": "Hi 👋 - i'm openai"}]) -``` - -### Track Costs, Usage, Latency for streaming -Use a callback function for this - more info on custom callbacks: https://docs.litellm.ai/docs/observability/custom_callback - -```python -import litellm - -# track_cost_callback -def track_cost_callback( - kwargs, # kwargs to completion - completion_response, # response from completion - start_time, end_time # start/end time -): - try: - response_cost = kwargs.get("response_cost", 0) - print("streaming response_cost", response_cost) - except: - pass -# set callback -litellm.success_callback = [track_cost_callback] # set custom callback function - -# litellm.completion() call -response = completion( - model="gpt-3.5-turbo", - messages=[ - { - "role": "user", - "content": "Hi 👋 - i'm openai" - } - ], - stream=True -) -``` - -## **LiteLLM Proxy Server (LLM Gateway)** - -Track spend across multiple projects/people - -![ui_3](https://github.com/BerriAI/litellm/assets/29436595/47c97d5e-b9be-4839-b28c-43d7f4f10033) - -The proxy provides: - -1. [Hooks for auth](https://docs.litellm.ai/docs/proxy/virtual_keys#custom-auth) -2. [Hooks for logging](https://docs.litellm.ai/docs/proxy/logging#step-1---create-your-custom-litellm-callback-class) -3. [Cost tracking](https://docs.litellm.ai/docs/proxy/virtual_keys#tracking-spend) -4. [Rate Limiting](https://docs.litellm.ai/docs/proxy/users#set-rate-limits) - -### 📖 Proxy Endpoints - [Swagger Docs](https://litellm-api.up.railway.app/) - -Go here for a complete tutorial with keys + rate limits - [**here**](./proxy/docker_quick_start.md) - -### Quick Start Proxy - CLI +To run the full Proxy Server (LLM Gateway): ```shell pip install 'litellm[proxy]' ``` -#### Step 1: Start litellm proxy +--- + +## Quick Start + +Make your first LLM call using the provider of your choice: + - +```python +from litellm import completion +import os -```shell -$ litellm --model huggingface/bigcode/starcoder +os.environ["OPENAI_API_KEY"] = "your-api-key" -#INFO: Proxy running on http://0.0.0.0:4000 +response = completion( + model="openai/gpt-4o", + messages=[{"role": "user", "content": "Hello, how are you?"}] +) +print(response.choices[0].message.content) ``` + - +```python +from litellm import completion +import os +os.environ["ANTHROPIC_API_KEY"] = "your-api-key" -Step 1. CREATE config.yaml +response = completion( + model="anthropic/claude-3-5-sonnet-20241022", + messages=[{"role": "user", "content": "Hello, how are you?"}] +) +print(response.choices[0].message.content) +``` -Example `litellm_config.yaml` + + -```yaml +```python +from litellm import completion +import os + +# auth: run 'gcloud auth application-default login' +os.environ["VERTEXAI_PROJECT"] = "your-project-id" +os.environ["VERTEXAI_LOCATION"] = "us-central1" + +response = completion( + model="vertex_ai/gemini-1.5-pro", + messages=[{"role": "user", "content": "Hello, how are you?"}] +) +print(response.choices[0].message.content) +``` + + + + +```python +from litellm import completion +import os + +os.environ["AWS_ACCESS_KEY_ID"] = "your-key" +os.environ["AWS_SECRET_ACCESS_KEY"] = "your-secret" +os.environ["AWS_REGION_NAME"] = "us-east-1" + +response = completion( + model="bedrock/anthropic.claude-3-5-sonnet-20241022-v2:0", + messages=[{"role": "user", "content": "Hello, how are you?"}] +) +print(response.choices[0].message.content) +``` + + + + +```python +from litellm import completion + +response = completion( + model="ollama/llama3", + messages=[{"role": "user", "content": "Hello, how are you?"}], + api_base="http://localhost:11434" +) +print(response.choices[0].message.content) +``` + + + + +```python +from litellm import completion +import os + +os.environ["AZURE_API_KEY"] = "your-key" +os.environ["AZURE_API_BASE"] = "https://your-resource.openai.azure.com" +os.environ["AZURE_API_VERSION"] = "2024-02-01" + +response = completion( + model="azure/your-deployment-name", + messages=[{"role": "user", "content": "Hello, how are you?"}] +) +print(response.choices[0].message.content) +``` + + + + +Every response follows the OpenAI Chat Completions format, regardless of provider. ✅ + +### Response Format + +Non-streaming responses return a `ModelResponse` object: + +```json +{ + "id": "chatcmpl-abc123", + "object": "chat.completion", + "created": 1677858242, + "model": "gpt-4o", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": "Hello! I'm doing well, thanks for asking." + }, + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 13, + "completion_tokens": 12, + "total_tokens": 25 + } +} +``` + +Streaming responses (`stream=True`) yield `ModelResponseStream` chunks: + +```json +{ + "id": "chatcmpl-abc123", + "object": "chat.completion.chunk", + "created": 1677858242, + "model": "gpt-4o", + "choices": [ + { + "index": 0, + "delta": { + "role": "assistant", + "content": "Hello" + }, + "finish_reason": null + } + ] +} +``` + +📖 [Full output format reference →](./completion/output) + +:::tip Open in Colab + +Open In Colab + +::: + +--- + +## New to LiteLLM? + +**Want to get started fast?** Head to [Tutorials](/docs/tutorials) for step-by-step walkthroughs — AI coding tools, agent SDKs, proxy setup, and more. + +**Need to understand a specific feature?** Check [Guides](/docs/guides) for streaming, function calling, prompt caching, and other how-tos. + +--- + +## Choose Your Path + + + +--- + +## LiteLLM Python SDK + +### Streaming + +Add `stream=True` to receive chunks as they are generated: + +```python +from litellm import completion +import os + +os.environ["OPENAI_API_KEY"] = "your-api-key" + +for chunk in completion( + model="openai/gpt-4o", + messages=[{"role": "user", "content": "Write a short poem"}], + stream=True, +): + print(chunk.choices[0].delta.content or "", end="") +``` + +### Exception Handling + +LiteLLM maps every provider's errors to the OpenAI exception types — your existing error handling works out of the box: + +```python +import litellm + +try: + litellm.completion( + model="anthropic/claude-instant-1", + messages=[{"role": "user", "content": "Hey!"}] + ) +except litellm.AuthenticationError as e: + print(f"Bad API key: {e}") +except litellm.RateLimitError as e: + print(f"Rate limited: {e}") +except litellm.APIError as e: + print(f"API error: {e}") +``` + +### Logging & Observability + +Send input/output to Langfuse, MLflow, Helicone, Lunary, and more with a single line: + +```python +import litellm + +litellm.success_callback = ["langfuse", "mlflow", "helicone"] + +response = litellm.completion( + model="gpt-4o", + messages=[{"role": "user", "content": "Hi!"}] +) +``` + +📖 [See all observability integrations →](/docs/observability/agentops_integration) + +### Track Costs & Usage + +Use a callback to capture cost per response: + +```python +import litellm + +def track_cost(kwargs, completion_response, start_time, end_time): + print("Cost:", kwargs.get("response_cost", 0)) + +litellm.success_callback = [track_cost] + +litellm.completion( + model="gpt-4o", + messages=[{"role": "user", "content": "Hello!"}], + stream=True +) +``` + +📖 [Custom callback docs →](./observability/custom_callback) + +--- + +## LiteLLM Proxy Server (LLM Gateway) + +The proxy is a self-hosted OpenAI-compatible gateway. Any client that works with OpenAI works with the proxy — no code changes needed. + +![LiteLLM Proxy Dashboard](https://github.com/BerriAI/litellm/assets/29436595/47c97d5e-b9be-4839-b28c-43d7f4f10033) + +#### Step 1 — Start the proxy + + + + +```shell +litellm --model huggingface/bigcode/starcoder +# Proxy running on http://0.0.0.0:4000 +``` + + + + +```yaml title="litellm_config.yaml" model_list: - model_name: gpt-3.5-turbo litellm_params: - model: azure/ - api_base: os.environ/AZURE_API_BASE # runs os.getenv("AZURE_API_BASE") - api_key: os.environ/AZURE_API_KEY # runs os.getenv("AZURE_API_KEY") + model: azure/your-deployment + api_base: os.environ/AZURE_API_BASE + api_key: os.environ/AZURE_API_KEY api_version: "2023-07-01-preview" ``` -Step 2. RUN Docker Image - ```shell docker run \ - -v $(pwd)/litellm_config.yaml:/app/config.yaml \ - -e AZURE_API_KEY=d6*********** \ - -e AZURE_API_BASE=https://openai-***********/ \ - -p 4000:4000 \ - docker.litellm.ai/berriai/litellm:main-latest \ - --config /app/config.yaml --detailed_debug + -v $(pwd)/litellm_config.yaml:/app/config.yaml \ + -e AZURE_API_KEY=your-key \ + -e AZURE_API_BASE=https://your-resource.openai.azure.com/ \ + -p 4000:4000 \ + docker.litellm.ai/berriai/litellm:main-latest \ + --config /app/config.yaml --detailed_debug ``` - -#### Step 2: Make ChatCompletions Request to Proxy +#### Step 2 — Call it with the OpenAI client ```python -import openai # openai v1.0.0+ -client = openai.OpenAI(api_key="anything",base_url="http://0.0.0.0:4000") # set proxy to base_url -# request sent to model set on litellm proxy, `litellm --model` -response = client.chat.completions.create(model="gpt-3.5-turbo", messages = [ - { - "role": "user", - "content": "this is a test request, write a short poem" - } -]) +import openai -print(response) +client = openai.OpenAI(api_key="anything", base_url="http://0.0.0.0:4000") + +response = client.chat.completions.create( + model="gpt-3.5-turbo", + messages=[{"role": "user", "content": "Write a short poem"}] +) +print(response.choices[0].message.content) ``` -## More details +👉 [Full proxy quickstart with Docker →](./proxy/docker_quick_start) -- [exception mapping](./exception_mapping.md) -- [retries + model fallbacks for completion()](./completion/reliable_completions.md) -- [proxy virtual keys & spend management](./proxy/virtual_keys.md) -- [E2E Tutorial for LiteLLM Proxy Server](./proxy/docker_quick_start.md) +:::tip Debugging tool +Use [**`/utils/transform_request`**](./utils/transform_request) to inspect exactly what LiteLLM sends to any provider — useful for debugging prompt formatting, header issues, and provider-specific parameters. +::: + +🔗 [Interactive API explorer (Swagger) →](https://litellm-api.up.railway.app/) + +--- + +## Agent & MCP Gateway + +LiteLLM is a unified gateway for **LLMs, agents, and MCP** — you don't need a separate agent or MCP gateway. One endpoint for 100+ models, A2A agents, and MCP tools. + + + +--- + +## What to Explore Next + + diff --git a/docs/my-website/docs/integrations/index.md b/docs/my-website/docs/integrations/index.md index 95c922cce89..0ad934d5b41 100644 --- a/docs/my-website/docs/integrations/index.md +++ b/docs/my-website/docs/integrations/index.md @@ -1,18 +1,336 @@ -# Integrations +--- +title: Integrations +sidebar_label: Overview +--- + +import NavigationCards from '@site/src/components/NavigationCards'; This section covers integrations with various tools and services that can be used with LiteLLM (either Proxy or SDK). -## AI Agent Frameworks -- **[Letta](./letta.md)** - Build stateful LLM agents with persistent memory using LiteLLM Proxy +--- -## Development Tools -- **[OpenWebUI](../tutorials/openweb_ui.md)** - Self-hosted ChatGPT-style interface +## Observability -## Observability & Monitoring -- **[Langfuse](../observability/langfuse_integration.md)** - LLM observability and analytics -- **[Prometheus](../proxy/prometheus.md)** - Metrics collection and monitoring -- **[PagerDuty](../proxy/pagerduty.md)** - Incident response and alerting -- **[Datadog](../observability/datadog.md)** +Track, debug, and analyze LLM calls with observability platforms. + -Click into each section to learn more about the integrations. \ No newline at end of file +[View all observability integrations →](/docs/integrations/observability_integrations) + +--- + +## Alerting & Monitoring + +Set up alerts, metrics collection, and infrastructure monitoring. + + + +--- + +## Guardrail Providers + +Add safety and content filtering to LLM calls. + + + +[View all guardrail providers →](/docs/guardrail_providers) + +--- + +## Policies + +Define and enforce usage policies across your LLM deployment. + + + +--- + +## AI Tools + +Connect LiteLLM to AI-powered coding and productivity tools. + + + +--- + +## Agent SDKs + +Use LiteLLM with agent frameworks and SDKs. + + + +--- + +## Prompt Management + +Manage, version, and deploy prompts. + + + +--- + +## Manage with AI Agents + +Use AI agents to manage your LiteLLM deployment — create users, teams, keys, models, and more via natural language. + + diff --git a/docs/my-website/docs/integrations/letta.md b/docs/my-website/docs/integrations/letta.md index 2afb82542f2..9711999df5e 100644 --- a/docs/my-website/docs/integrations/letta.md +++ b/docs/my-website/docs/integrations/letta.md @@ -920,9 +920,9 @@ for model in models: ## Resources -- [Letta Documentation](https://docs.letta.ai/) -- [LiteLLM Proxy Documentation](../proxy/quick_start.md) -- [LiteLLM SDK Documentation](../completion/input.md) -- [Function Calling Guide](../completion/function_call.md) -- [Observability Setup](../observability/langfuse_integration.md) -- [Router Configuration](../routing.md) \ No newline at end of file +- [Letta Documentation](https://docs.letta.com/) +- [LiteLLM Proxy Documentation](/docs/simple_proxy) +- [LiteLLM SDK Documentation](/docs/#litellm-python-sdk) +- [Function Calling Guide](/docs/completion/function_call) +- [Observability Setup](/docs/integrations/observability_integrations) +- [Router Configuration](/docs/routing) \ No newline at end of file diff --git a/docs/my-website/docs/integrations/observability_index.md b/docs/my-website/docs/integrations/observability_index.md new file mode 100644 index 00000000000..8ab83950cdc --- /dev/null +++ b/docs/my-website/docs/integrations/observability_index.md @@ -0,0 +1,28 @@ +--- +title: Observability +sidebar_label: Overview +slug: observability_integrations +--- + +Track, debug, and analyze LLM calls with observability platforms. + +import NavigationCards from '@site/src/components/NavigationCards'; + +## Observability Integrations + + + +[View all observability integrations →](/docs/observability/callbacks) diff --git a/docs/my-website/docs/integrations/websearch_interception.md b/docs/my-website/docs/integrations/websearch_interception.md index 0c5d8927013..bc5e8ec0b39 100644 --- a/docs/my-website/docs/integrations/websearch_interception.md +++ b/docs/my-website/docs/integrations/websearch_interception.md @@ -375,7 +375,7 @@ search_tools: - [Search Providers](../search/index.md) - Detailed search provider setup - [Claude Code WebSearch](../tutorials/claude_code_websearch.md) - Using with Claude Code - [Tool Calling](../completion/function_call.md) - General tool calling documentation -- [Callbacks](./custom_callback.md) - Custom callback documentation +- [Callbacks](../observability/custom_callback.md) - Custom callback documentation ## Technical Details diff --git a/docs/my-website/docs/learn/gateway_quickstart.md b/docs/my-website/docs/learn/gateway_quickstart.md new file mode 100644 index 00000000000..acec259758c --- /dev/null +++ b/docs/my-website/docs/learn/gateway_quickstart.md @@ -0,0 +1,174 @@ +--- +title: Gateway Quickstart +sidebar_label: Gateway Quickstart +description: Start LiteLLM Gateway, add models and keys, then connect applications and SDKs to one shared endpoint. +--- + +import NavigationCards from '@site/src/components/NavigationCards'; + +Use this path if you need one shared OpenAI-compatible endpoint for a team or platform. + +If you need a Docker or database-first setup, use the [Docker + Database tutorial](/docs/proxy/docker_quick_start). Otherwise, use the steps below to get to a working request fast. + +## 1. Install The Gateway + +```bash +pip install 'litellm[proxy]' +``` + +## 2. Set One Provider Key + +```bash +export OPENAI_API_KEY="your-api-key" +``` + +## 3. Create `config.yaml` + +```yaml +model_list: + - model_name: gpt-4o-mini + litellm_params: + model: openai/gpt-4o-mini + api_key: os.environ/OPENAI_API_KEY + +general_settings: + master_key: sk-1234 +``` + +## 4. Start The Gateway + +```bash +litellm --config config.yaml +``` + +You should see the proxy start on `http://0.0.0.0:4000`. + +## 5. Send Your First Request + +```bash +curl -X POST 'http://0.0.0.0:4000/chat/completions' \ + -H 'Content-Type: application/json' \ + -H 'Authorization: Bearer sk-1234' \ + -d '{ + "model": "gpt-4o-mini", + "messages": [ + {"role": "user", "content": "Hello from LiteLLM Gateway"} + ] + }' +``` + +## 6. Check The Response + +If the request succeeds, the proxy returns `200 OK` with an OpenAI-style response. + +The assistant text will be in: + +```json +choices[0].message.content +``` + +If your gateway is routing to OpenAI, a real response can look like this: + +```json +{ + "id": "chatcmpl-abc123", + "created": 1677858242, + "model": "gpt-4o-mini-2024-07-18", + "object": "chat.completion", + "system_fingerprint": "fp_406d6473f8", + "choices": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "role": "assistant", + "content": "Hello! How can I assist you today?", + "tool_calls": null, + "function_call": null, + "annotations": [] + } + } + ], + "usage": { + "completion_tokens": 9, + "prompt_tokens": 13, + "total_tokens": 22, + "completion_tokens_details": { + "accepted_prediction_tokens": 0, + "audio_tokens": 0, + "reasoning_tokens": 0, + "rejected_prediction_tokens": 0 + }, + "prompt_tokens_details": { + "audio_tokens": 0, + "cached_tokens": 0 + } + }, + "service_tier": "default" +} +``` + +`id`, `created`, the resolved model version, token counts, and message text will vary by request. Other providers may return a smaller or slightly different set of fields, but `choices[0].message.content` is the main field to read. + +## 7. Add Keys And The UI + +If you need virtual keys, spend tracking, or the admin UI, add a database next. + +- Add `database_url` under `general_settings` +- Use [Virtual keys](/docs/proxy/virtual_keys) for key creation and budgets +- Use [Admin UI](/docs/proxy/ui) to manage models and keys +- Use the [Docker + Database tutorial](/docs/proxy/docker_quick_start) if you want a fuller setup + +## 8. Pick Your Next Step + + + +## When To Use The SDK Path Instead + +If you only need to call models from one application and do not need centralized auth or shared infrastructure, start with the [SDK Quickstart](/docs/learn/sdk_quickstart) instead. diff --git a/docs/my-website/docs/learn/index.md b/docs/my-website/docs/learn/index.md new file mode 100644 index 00000000000..018aec5af00 --- /dev/null +++ b/docs/my-website/docs/learn/index.md @@ -0,0 +1,117 @@ +--- +title: Learn LiteLLM +sidebar_label: Learn +slug: /learn +--- + +import NavigationCards from '@site/src/components/NavigationCards'; + +LiteLLM gives you one OpenAI-compatible interface for 100+ LLM providers. Start with the path that matches your setup. + +--- + +## Start Here + +Pick one path first. + + + +--- + +## Common Tasks + +Jump to a specific task. + + + +--- + +## Docs Map + +Use these when you already know the type of doc you want. + + + +Not sure where to start? Use [SDK Quickstart](/docs/learn/sdk_quickstart) for app code or [Gateway Quickstart](/docs/learn/gateway_quickstart) for shared infrastructure. diff --git a/docs/my-website/docs/learn/sdk_quickstart.md b/docs/my-website/docs/learn/sdk_quickstart.md new file mode 100644 index 00000000000..bdf7b63eb5d --- /dev/null +++ b/docs/my-website/docs/learn/sdk_quickstart.md @@ -0,0 +1,174 @@ +--- +title: SDK Quickstart +sidebar_label: SDK Quickstart +description: Make your first LiteLLM SDK call, then jump to the right docs for the next feature you need. +--- + +import NavigationCards from '@site/src/components/NavigationCards'; + +Use this path if you are integrating LiteLLM directly into application code. + +## 1. Install LiteLLM + +```bash +pip install litellm +``` + +## 2. Set Provider Credentials + +Start with one provider and set its environment variables. + +- OpenAI: `OPENAI_API_KEY` +- Anthropic: `ANTHROPIC_API_KEY` +- Azure OpenAI: `AZURE_API_KEY`, `AZURE_API_BASE`, `AZURE_API_VERSION` +- Bedrock: standard AWS credentials +- Vertex AI: `VERTEXAI_PROJECT`, `VERTEXAI_LOCATION` + +If you have not picked a provider yet, browse [all supported providers](/docs/providers). + +## 3. Make Your First Call + +```python +from litellm import completion +import os + +os.environ["OPENAI_API_KEY"] = "your-api-key" + +response = completion( + model="openai/gpt-4o", + messages=[{"role": "user", "content": "Hello, how are you?"}], +) + +print(response.choices[0].message.content) +``` + +## 4. Check The Response + +The line below: + +```python +print(response.choices[0].message.content) +``` + +prints the assistant text, for example: + +```text +Hello! I'm doing well, thanks for asking. +``` + +If you print the full object with: + +```python +print(response) +``` + +you will see a Python `ModelResponse(...)` object. For an OpenAI-backed model, it can look like this: + +```python +ModelResponse( + id='chatcmpl-abc123', + created=1773782130, + model='gpt-4o-2024-08-06', + object='chat.completion', + system_fingerprint='fp_4ff89bf575', + choices=[ + Choices( + finish_reason='stop', + index=0, + message=Message( + content="Hello! I'm just a program, but I'm here to help you. How can I assist you today?", + role='assistant', + tool_calls=None, + function_call=None, + provider_specific_fields={'refusal': None}, + annotations=[] + ), + provider_specific_fields={} + ) + ], + usage=Usage( + completion_tokens=21, + prompt_tokens=13, + total_tokens=34, + completion_tokens_details=CompletionTokensDetailsWrapper(...), + prompt_tokens_details=PromptTokensDetailsWrapper(...) + ), + service_tier='default' +) +``` + +The same response follows an OpenAI-style shape. Conceptually, it looks like this: + +```json +{ + "id": "chatcmpl-abc123", + "object": "chat.completion", + "created": 1677858242, + "model": "gpt-4o", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": "Hello! I'm doing well, thanks for asking." + }, + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 13, + "completion_tokens": 12, + "total_tokens": 25 + } +} +``` + +`id`, `created`, token counts, and message text will vary by request. + +If you call an OpenAI-backed model, you may also see extra fields such as `system_fingerprint`, `service_tier`, `tool_calls`, `function_call`, `annotations`, `provider_specific_fields`, and detailed token usage. For the full output reference, see [completion output](/docs/completion/output). + +Need more provider examples? See the main [Getting Started](/docs/#quick-start) page. + +## 5. Pick Your Next Step + + + +## When To Use Gateway Instead + +Use LiteLLM Gateway if you need centralized auth, virtual keys, spend tracking, shared logging, or one OpenAI-compatible endpoint for multiple apps. + +[Go to Gateway Quickstart →](/docs/learn/gateway_quickstart) diff --git a/docs/my-website/docs/load_test_advanced.md b/docs/my-website/docs/load_test_advanced.md index d35b5f74784..d7bc35e74e0 100644 --- a/docs/my-website/docs/load_test_advanced.md +++ b/docs/my-website/docs/load_test_advanced.md @@ -11,7 +11,7 @@ Tutorial on how to get to 1K+ RPS with LiteLLM Proxy on locust - [Github releases](https://github.com/BerriAI/litellm/releases) - [litellm docker containers](https://github.com/BerriAI/litellm/pkgs/container/litellm) - [litellm database docker container](https://github.com/BerriAI/litellm/pkgs/container/litellm-database) -- [ ] Ensure you're following **ALL** [best practices for production](./proxy/production_setup.md) +- [ ] Ensure you're following **ALL** [best practices for production](./proxy/prod.md) - [ ] Locust - Ensure you're Locust instance can create 1K+ requests per second - 👉 You can use our **[maintained locust instance here](https://locust-load-tester-production.up.railway.app/)** - If you're self hosting locust @@ -222,4 +222,4 @@ class MyUser(HttpUser): def on_start(self): self.api_key = os.getenv('API_KEY', 'sk-1234') self.client.headers.update({'Authorization': f'Bearer {self.api_key}'}) -``` \ No newline at end of file +``` diff --git a/docs/my-website/docs/mcp_zero_trust.md b/docs/my-website/docs/mcp_zero_trust.md new file mode 100644 index 00000000000..8f431523cb8 --- /dev/null +++ b/docs/my-website/docs/mcp_zero_trust.md @@ -0,0 +1,294 @@ +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +# MCP Zero Trust Auth (JWT Signer) + +![Zero Trust MCP Gateway](/img/mcp_zero_trust_gateway.png) + +MCP servers have no built-in way to verify that a request actually came through LiteLLM. Without this guardrail, any client that can reach your MCP server directly can call tools — bypassing your access controls entirely. + +`MCPJWTSigner` fixes this. It signs every outbound tool call with a short-lived RS256 JWT. Your MCP server verifies the signature against LiteLLM's public key. Requests that didn't go through LiteLLM have no valid signature and are rejected. + +--- + +## Basic setup + +Add the guardrail to your config and point your MCP server at LiteLLM's JWKS endpoint. Every tool call gets a signed JWT automatically — no changes needed on the client side. + +```yaml title="config.yaml" +mcp_servers: + - server_name: weather + url: http://localhost:8000/mcp + transport: http + +guardrails: + - guardrail_name: mcp-jwt-signer + litellm_params: + guardrail: mcp_jwt_signer + mode: pre_mcp_call + default_on: true + issuer: "https://my-litellm.example.com" # defaults to request base URL + audience: "mcp" # default: "mcp" + ttl_seconds: 300 # default: 300 +``` + +**Bring your own signing key** — recommended for production. Auto-generated keys are lost on restart. + +```bash +export MCP_JWT_SIGNING_KEY="-----BEGIN RSA PRIVATE KEY-----\n..." +# or point to a file +export MCP_JWT_SIGNING_KEY="file:///secrets/mcp-signing-key.pem" +``` + +**Build a verified MCP server with [FastMCP](https://gofastmcp.com):** + +```python title="weather_server.py" +from fastmcp import FastMCP, Context +from fastmcp.server.auth.providers.jwt import JWTVerifier + +auth = JWTVerifier( + jwks_uri="https://my-litellm.example.com/.well-known/jwks.json", + issuer="https://my-litellm.example.com", + audience="mcp", + algorithm="RS256", +) + +mcp = FastMCP("weather-server", auth=auth) + +@mcp.tool() +async def get_weather(city: str, ctx: Context) -> str: + caller = ctx.client_id # JWT `sub` — the verified user identity + return f"Weather in {city}: sunny, 72°F (requested by {caller})" + +if __name__ == "__main__": + mcp.run(transport="http", host="0.0.0.0", port=8000) +``` + +FastMCP fetches the JWKS automatically and re-fetches when the signing key changes. + +LiteLLM publishes OIDC discovery so MCP servers find the key without any manual configuration: + +``` +GET /.well-known/openid-configuration → { "jwks_uri": "https:///.well-known/jwks.json" } +GET /.well-known/jwks.json → { "keys": [{ "kty": "RSA", "alg": "RS256", ... }] } +``` + +> **Read further only if you need to:** thread a corporate IdP identity into the JWT, enforce specific claims on callers, add custom metadata, use AWS Bedrock AgentCore Gateway, or debug JWT rejections. + +--- + +## Thread IdP identity into MCP JWTs + +By default the outbound JWT `sub` is LiteLLM's internal `user_id`. If your users authenticate with Okta, Azure AD, or another IdP, the MCP server sees a LiteLLM-internal ID — not the user's email or employee ID. + +With verify+re-sign, LiteLLM validates the incoming IdP token first, then builds the outbound JWT using the real identity claims from that token. The MCP server gets the user's actual identity without ever having to trust the original IdP directly. + +```yaml title="config.yaml" +guardrails: + - guardrail_name: mcp-jwt-signer + litellm_params: + guardrail: mcp_jwt_signer + mode: pre_mcp_call + default_on: true + issuer: "https://my-litellm.example.com" + + # Validate the incoming Bearer token against the IdP + access_token_discovery_uri: "https://login.microsoftonline.com/{tenant}/v2.0/.well-known/openid-configuration" + verify_issuer: "https://login.microsoftonline.com/{tenant}/v2.0" + verify_audience: "api://my-app" + + # Which claim to use for `sub` in the outbound JWT — first non-empty value wins + end_user_claim_sources: + - "token:sub" # from the verified incoming JWT + - "token:email" # fallback to email + - "litellm:user_id" # last resort: LiteLLM's internal user_id +``` + +If the incoming token is **opaque** (not a JWT — some IdPs issue these), add an introspection endpoint. LiteLLM will POST the token to it (RFC 7662) and use the returned claims: + +```yaml + token_introspection_endpoint: "https://idp.example.com/oauth2/introspect" +``` + +**Supported `end_user_claim_sources` values:** + +| Source | Resolves to | +|--------|-------------| +| `token:` | Any claim from the verified incoming JWT (e.g. `token:sub`, `token:email`, `token:oid`) | +| `litellm:user_id` | LiteLLM's internal user ID | +| `litellm:email` | User email from LiteLLM auth context | +| `litellm:end_user_id` | End-user ID if set separately | +| `litellm:team_id` | Team ID from LiteLLM auth context | + +--- + +## Block callers missing required attributes + +Some MCP servers expose sensitive operations that should only be reachable by verified employees — not service accounts, not external API keys. You can enforce this at the LiteLLM layer so the MCP server never receives the request at all. + +`required_claims` rejects with `403` if the incoming token is missing any listed claim. `optional_claims` forwards claims that are useful but not mandatory. + +```yaml title="config.yaml" +guardrails: + - guardrail_name: mcp-jwt-signer + litellm_params: + guardrail: mcp_jwt_signer + mode: pre_mcp_call + default_on: true + + access_token_discovery_uri: "https://idp.example.com/.well-known/openid-configuration" + + # Service accounts without `employee_id` are blocked before the tool runs + required_claims: + - "sub" + - "employee_id" + + # Forward these into the outbound JWT when present — skipped silently if absent + optional_claims: + - "groups" + - "department" +``` + +**What the client sees when blocked:** +```json +HTTP 403 +{ "error": "MCPJWTSigner: incoming token is missing required claims: ['employee_id']. Configure the IdP to include these claims." } +``` + +--- + +## Add custom metadata to every JWT + +Your MCP server may need context that LiteLLM doesn't carry natively — which deployment sent the request, a tenant ID, an environment tag. Use claim operations to inject, override, or strip claims from the outbound JWT. + +```yaml title="config.yaml" +guardrails: + - guardrail_name: mcp-jwt-signer + litellm_params: + guardrail: mcp_jwt_signer + mode: pre_mcp_call + default_on: true + + # add: insert only when the key is not already in the JWT + add_claims: + deployment_id: "prod-us-east-1" + tenant_id: "acme-corp" + + # set: always override — even if the claim came from the incoming token + set_claims: + env: "production" + + # remove: strip claims the MCP server shouldn't see + remove_claims: + - "nbf" # some validators reject nbf; remove it if yours does +``` + +Operations run in order — `add_claims` → `set_claims` → `remove_claims`. `set_claims` always wins over `add_claims`; `remove_claims` beats both. + +--- + +## AWS Bedrock AgentCore Gateway + +Bedrock AgentCore Gateway uses two separate JWTs: one to authenticate the transport connection and another to authorize tool calls. They need different `aud` values and TTLs — a single JWT won't work for both. + +LiteLLM can issue both in one hook and inject them into separate headers: + +```yaml title="config.yaml" +guardrails: + - guardrail_name: mcp-jwt-signer + litellm_params: + guardrail: mcp_jwt_signer + mode: pre_mcp_call + default_on: true + issuer: "https://my-litellm.example.com" + audience: "mcp-resource" # for the MCP resource layer + ttl_seconds: 300 + + # Second JWT for the transport channel — same sub/act/scope, different aud + TTL + channel_token_audience: "bedrock-agentcore-gateway" + channel_token_ttl: 60 # transport tokens should be short-lived +``` + +LiteLLM injects two headers on every tool call: +- `Authorization: Bearer ` — audience `mcp-resource`, TTL 300s +- `x-mcp-channel-token: Bearer ` — audience `bedrock-agentcore-gateway`, TTL 60s + +Both tokens are signed with the same LiteLLM key, so your MCP server only needs to trust one JWKS endpoint. + +--- + +## Control which scopes go into the JWT + +By default LiteLLM generates least-privilege scopes per request: +- Tool call → `mcp:tools/call mcp:tools/{name}:call` +- List tools → `mcp:tools/call mcp:tools/list` + +If your MCP server does its own scope enforcement and needs a specific format, set `allowed_scopes` to replace auto-generation entirely: + +```yaml title="config.yaml" +guardrails: + - guardrail_name: mcp-jwt-signer + litellm_params: + guardrail: mcp_jwt_signer + mode: pre_mcp_call + default_on: true + + allowed_scopes: + - "mcp:tools/call" + - "mcp:tools/list" + - "mcp:admin" +``` + +Every JWT carries exactly those scopes regardless of which tool is being called. + +--- + +## Debug JWT rejections + +Your MCP server is returning 401 and you're not sure what's in the JWT. Enable `debug_headers` and LiteLLM adds a `x-litellm-mcp-debug` response header with the key claims that were signed: + +```yaml title="config.yaml" +guardrails: + - guardrail_name: mcp-jwt-signer + litellm_params: + guardrail: mcp_jwt_signer + mode: pre_mcp_call + default_on: true + debug_headers: true +``` + +Response header: +``` +x-litellm-mcp-debug: v=1; kid=a3f1b2c4d5e6f708; sub=alice@corp.com; iss=https://my-litellm.example.com; exp=1712345678; scope=mcp:tools/call mcp:tools/get_weather:call +``` + +Check that `kid` matches what the MCP server fetched from JWKS, `iss`/`aud` match your server's expected values, and `exp` hasn't passed. Disable in production — the header leaks claim metadata. + +--- + +## JWT claims reference + +| Claim | Value | +|-------|-------| +| `iss` | `issuer` config value (or request base URL) | +| `aud` | `audience` config value (default: `"mcp"`) | +| `sub` | Resolved via `end_user_claim_sources` (default: `user_id` → api-key hash → `"litellm-proxy"`) | +| `act.sub` | `team_id` → `org_id` → `"litellm-proxy"` (RFC 8693 delegation) | +| `email` | `user_email` from LiteLLM auth context (when available) | +| `scope` | Auto-generated per tool call, or `allowed_scopes` when set | +| `iat`, `exp`, `nbf` | Standard timing claims (RFC 7519) | + +--- + +## Limitations + +- **OpenAPI-backed MCP servers** (`spec_path` set) do not support JWT injection. LiteLLM logs a warning and skips the header. Use SSE/HTTP transport servers to get full JWT injection. +- The keypair is **in-memory by default** and rotated on each restart unless `MCP_JWT_SIGNING_KEY` is set. FastMCP's `JWTVerifier` handles key rotation transparently via JWKS key ID matching. + +--- + +## Related + +- [MCP Guardrails](./mcp_guardrail) — PII masking and blocking for MCP calls +- [MCP OAuth](./mcp_oauth) — upstream OAuth2 for MCP server access +- [MCP AWS SigV4](./mcp_aws_sigv4) — AWS-signed requests to MCP servers diff --git a/docs/my-website/docs/observability/langfuse_otel_integration.md b/docs/my-website/docs/observability/langfuse_otel_integration.md index b4c9a2bd1ad..79ad2f6f75d 100644 --- a/docs/my-website/docs/observability/langfuse_otel_integration.md +++ b/docs/my-website/docs/observability/langfuse_otel_integration.md @@ -83,6 +83,9 @@ os.environ["LANGFUSE_OTEL_HOST"] = "https://cloud.langfuse.com" # EU region # Or use self-hosted instance # os.environ["LANGFUSE_OTEL_HOST"] = "https://my-langfuse.company.com" +# Optional: Ignore otel context propagation to prevent parent-child relationships with spans from other providers +# os.environ["OTEL_IGNORE_CONTEXT_PROPAGATION"] = "true" + litellm.callbacks = ["langfuse_otel"] ``` @@ -124,6 +127,9 @@ export LANGFUSE_PUBLIC_KEY="pk-lf-..." export LANGFUSE_SECRET_KEY="sk-lf-..." export LANGFUSE_OTEL_HOST="https://us.cloud.langfuse.com" # Default US region # export LANGFUSE_OTEL_HOST="https://otel.my-langfuse.company.com" # custom OTEL endpoint + +# Optional: Ignore otel context propagation to prevent parent-child relationships with spans from other providers +# export OTEL_IGNORE_CONTEXT_PROPAGATION="true" ``` 2. Setup config.yaml diff --git a/docs/my-website/docs/observability/vantage.md b/docs/my-website/docs/observability/vantage.md new file mode 100644 index 00000000000..31b43a76c32 --- /dev/null +++ b/docs/my-website/docs/observability/vantage.md @@ -0,0 +1,148 @@ +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +# Vantage Integration + +LiteLLM can export proxy spend data to [Vantage](https://vantage.sh) as [FOCUS 1.2](https://focus.finops.org/) formatted cost reports. This lets you visualize LLM spend alongside your cloud infrastructure costs in the Vantage dashboard. + +## Overview + +| Property | Details | +|----------|---------| +| Destination | Export LiteLLM usage data to Vantage Custom Provider | +| Data format | FOCUS CSV (automatically transformed from LiteLLM spend data) | +| Supported operations | Manual export, automatic scheduled export (hourly/daily/interval) | +| Authentication | Vantage API key + Custom Provider token | + +## Prerequisites + +You need two credentials from the [Vantage console](https://console.vantage.sh): + +1. **API Key** — Go to **Settings → API Access Tokens** → Create a token with **Write** scope. The token looks like `vntg_tkn_...`. +2. **Custom Provider Token** — Go to **Settings → Integrations** → Create a **Custom Provider** integration → Copy the Provider ID (looks like `accss_crdntl_...`). + +## Setup via API + +The recommended setup uses the proxy admin endpoints. No config file changes needed. + +### 1. Initialize credentials + +```bash +curl -X POST http://localhost:4000/vantage/init \ + -H "Authorization: Bearer $LITELLM_ADMIN_KEY" \ + -H "Content-Type: application/json" \ + -d '{ + "api_key": "vntg_tkn_YOUR_VANTAGE_API_KEY", + "integration_token": "accss_crdntl_YOUR_PROVIDER_TOKEN" + }' +``` + +Credentials are encrypted and stored in the proxy database. + +### 2. Preview data (dry run) + +```bash +curl -X POST http://localhost:4000/vantage/dry-run \ + -H "Authorization: Bearer $LITELLM_ADMIN_KEY" \ + -H "Content-Type: application/json" \ + -d '{"limit": 10}' +``` + +This returns FOCUS-transformed data without sending anything to Vantage. Use it to verify the pipeline works and inspect the data mapping. + +### 3. Export to Vantage + +```bash +curl -X POST http://localhost:4000/vantage/export \ + -H "Authorization: Bearer $LITELLM_ADMIN_KEY" \ + -H "Content-Type: application/json" \ + -d '{}' +``` + +Optional parameters: +- `limit` — Max number of records to export +- `start_time_utc` / `end_time_utc` — Filter by time range (must be provided together) + +### 4. Verify in Vantage + +Go to **Settings → Integrations → your Custom Provider → Import Costs** tab to see uploaded CSVs. Once the status changes from "Importing and Processing" to "Stable", costs appear in **Cost Reporting → All Resources**. + +## Setup via Environment Variables + +For automatic scheduled exports, configure via environment variables and proxy config: + +### Environment variables + +| Variable | Required | Description | +|----------|----------|-------------| +| `VANTAGE_API_KEY` | Yes | Vantage API access token | +| `VANTAGE_INTEGRATION_TOKEN` | Yes | Custom Provider token from Vantage dashboard | +| `VANTAGE_BASE_URL` | No | API URL override (default: `https://api.vantage.sh`) | +| `VANTAGE_EXPORT_FREQUENCY` | No | `hourly` (default), `daily`, or `interval` | +| `VANTAGE_EXPORT_INTERVAL_SECONDS` | No | Seconds between exports when frequency is `interval` | + +### Proxy config + +```yaml +model_list: + - model_name: gpt-4o + litellm_params: + model: openai/gpt-4o + api_key: sk-your-key + +litellm_settings: + callbacks: ["vantage"] +``` + +```bash +export VANTAGE_API_KEY="vntg_tkn_..." +export VANTAGE_INTEGRATION_TOKEN="accss_crdntl_..." +litellm --config /path/to/config.yaml +``` + +The proxy registers a background job that exports data on the configured schedule. + +## API Endpoints + +All endpoints require admin authentication. + +| Method | Endpoint | Description | +|--------|----------|-------------| +| `POST` | `/vantage/init` | Store Vantage credentials (encrypted) | +| `GET` | `/vantage/settings` | View current config (credentials masked) | +| `PUT` | `/vantage/settings` | Update credentials or base URL | +| `POST` | `/vantage/dry-run` | Preview FOCUS data without uploading | +| `POST` | `/vantage/export` | Upload cost data to Vantage | +| `DELETE` | `/vantage/delete` | Remove credentials and stop scheduled exports | + +## FOCUS Field Mapping + +LiteLLM spend data is transformed into the FOCUS 1.2 schema: + +| LiteLLM Field | FOCUS Column | Description | +|---------------|-------------|-------------| +| `spend` | BilledCost, EffectiveCost | Cost of the usage | +| `model` | ChargeDescription, ResourceId | Model identifier | +| `model_group` | ServiceName | Model group / deployment | +| `custom_llm_provider` | ProviderName, PublisherName | Provider (openai, anthropic, etc.) | +| `api_key` | BillingAccountId | Hashed API key | +| `api_key_alias` | BillingAccountName | Human-readable key alias | +| `team_id` | SubAccountId | Team identifier | +| `team_alias` | SubAccountName | Team name | + +Additional metadata (user_id, model_group, etc.) is included in the `Tags` column as JSON. + +## Upload Limits + +Vantage enforces per-upload limits. LiteLLM handles these automatically: + +- **10,000 rows** per upload — large exports are split into batches +- **2 MB** per upload — oversized batches are further split by size +- **Unsupported columns** are stripped before upload + +## Related Links + +- [Vantage](https://vantage.sh) +- [Vantage Custom Providers](https://docs.vantage.sh/connecting_custom_providers) +- [FOCUS Specification](https://focus.finops.org/) +- [Focus Export (S3/Parquet)](./focus.md) diff --git a/docs/my-website/docs/providers/anthropic.md b/docs/my-website/docs/providers/anthropic.md index aa77ee7c268..50b964bd936 100644 --- a/docs/my-website/docs/providers/anthropic.md +++ b/docs/my-website/docs/providers/anthropic.md @@ -1965,6 +1965,98 @@ curl -L -X POST 'http://0.0.0.0:4000/v1/chat/completions' \ +## Files API + +Upload files once and reference them by `file_id` in multiple requests—no need to re-upload content each time. + +:::info +The `file_id` obtained from Anthropic only works with Anthropic Claude models. You cannot use it with other providers (OpenAI, Bedrock, etc.). +::: + +- **Max file size:** 500 MB | **Total storage:** 100 GB per org +- **Pricing:** File API operations are free. File content used in Messages requests is priced as input tokens. + +**Supported models by file type:** +- **Images:** All Claude 3+ models +- **PDFs:** All Claude 3.5+ models +- **Other file types** (for code execution): Claude 3.5 Haiku + all Claude 3.7+ models + +### Quick Start + +```python +import litellm +import os + +os.environ["ANTHROPIC_API_KEY"] = "sk-ant-..." + +# 1. Upload a file once +file = litellm.create_file( + file=open("document.pdf", "rb"), + purpose="messages", + custom_llm_provider="anthropic", +) + +# 2. Use file_id in messages (no re-upload needed) +response = litellm.completion( + model="anthropic/claude-sonnet-4-5-20250929", + messages=[{ + "role": "user", + "content": [ + {"type": "text", "text": "Summarize this document"}, + {"type": "file", "file": {"file_id": file.id, "format": "application/pdf"}} + ] + }] +) +``` + +### File Operations + +| Operation | Function | +|-----------|----------| +| Upload | `litellm.create_file(file, purpose="messages", custom_llm_provider="anthropic")` | +| List | `litellm.file_list(custom_llm_provider="anthropic")` | +| Retrieve | `litellm.file_retrieve(file_id, custom_llm_provider="anthropic")` | +| Delete | `litellm.file_delete(file_id, custom_llm_provider="anthropic")` | +| Download | `litellm.file_content(file_id, custom_llm_provider="anthropic")` | + +:::note +Download only works for files created by the [code execution tool](https://docs.anthropic.com/en/docs/agents-and-tools/tool-use/code-execution-tool), not uploaded files. +::: + +### Supported Formats + +| File Type | Format Value | +|-----------|-------------| +| PDF | `application/pdf` | +| Plain text | `text/plain` | +| JPEG | `image/jpeg` | +| PNG | `image/png` | +| GIF | `image/gif` | +| WebP | `image/webp` | + +### Using Images + +```python +# Upload image +image = litellm.create_file( + file=open("photo.jpg", "rb"), + purpose="messages", + custom_llm_provider="anthropic", +) + +# Use in message +response = litellm.completion( + model="anthropic/claude-sonnet-4-5-20250929", + messages=[{ + "role": "user", + "content": [ + {"type": "text", "text": "What's in this image?"}, + {"type": "file", "file": {"file_id": image.id, "format": "image/jpeg"}} + ] + }] +) +``` + ## Usage - passing 'user_id' to Anthropic LiteLLM translates the OpenAI `user` param to Anthropic's `metadata[user_id]` param. diff --git a/docs/my-website/docs/providers/aws_sagemaker.md b/docs/my-website/docs/providers/aws_sagemaker.md index bab475e7305..a2440c73d7d 100644 --- a/docs/my-website/docs/providers/aws_sagemaker.md +++ b/docs/my-website/docs/providers/aws_sagemaker.md @@ -526,3 +526,98 @@ print(f"response: {response}") ``` + +## Nova Models on SageMaker + +LiteLLM supports Amazon Nova models (Nova Micro, Nova Lite, Nova 2 Lite) deployed on SageMaker Inference real-time endpoints. These custom/fine-tuned Nova models use an OpenAI-compatible API format. + +**Reference:** [AWS Blog - Amazon SageMaker Inference for Custom Amazon Nova Models](https://aws.amazon.com/blogs/aws/announcing-amazon-sagemaker-inference-for-custom-amazon-nova-models/) + +### Usage + +Use the `sagemaker_nova/` prefix with your SageMaker endpoint name: + +```python +import litellm +import os + +os.environ["AWS_ACCESS_KEY_ID"] = "" +os.environ["AWS_SECRET_ACCESS_KEY"] = "" +os.environ["AWS_REGION_NAME"] = "us-east-1" + +# Basic chat completion +response = litellm.completion( + model="sagemaker_nova/my-nova-endpoint", + messages=[{"role": "user", "content": "Hello, how are you?"}], + temperature=0.7, + max_tokens=512, +) +print(response.choices[0].message.content) +``` + +### Streaming + +```python +response = litellm.completion( + model="sagemaker_nova/my-nova-endpoint", + messages=[{"role": "user", "content": "Write a short poem"}], + stream=True, + stream_options={"include_usage": True}, +) +for chunk in response: + if chunk.choices[0].delta.content: + print(chunk.choices[0].delta.content, end="") +``` + +### Multimodal (Images) + +Nova models on SageMaker support image inputs using base64 data URIs: + +```python +response = litellm.completion( + model="sagemaker_nova/my-nova-endpoint", + messages=[ + { + "role": "user", + "content": [ + {"type": "text", "text": "What's in this image?"}, + {"type": "image_url", "image_url": {"url": "data:image/jpeg;base64,..."}} + ] + } + ], +) +``` + +### Proxy Config + +```yaml +model_list: + - model_name: nova-micro + litellm_params: + model: sagemaker_nova/my-nova-micro-endpoint + aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID + aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY + aws_region_name: us-east-1 +``` + +### Supported Parameters + +All standard OpenAI parameters are supported, plus these Nova-specific parameters: + +| Parameter | Type | Description | +|-----------|------|-------------| +| `top_k` | integer | Limits token selection to top K most likely tokens | +| `reasoning_effort` | `"low"` \| `"high"` | Reasoning effort level (Nova 2 Lite custom models only) | +| `allowed_token_ids` | array[int] | Restrict output to specified token IDs | +| `truncate_prompt_tokens` | integer | Truncate prompt to N tokens if it exceeds limit | + +```python +response = litellm.completion( + model="sagemaker_nova/my-nova-endpoint", + messages=[{"role": "user", "content": "Think step by step: what is 2+2?"}], + top_k=40, + reasoning_effort="low", + logprobs=True, + top_logprobs=2, +) +``` diff --git a/docs/my-website/docs/providers/azure/azure.md b/docs/my-website/docs/providers/azure/azure.md index 12ddc1bd98e..682f263c108 100644 --- a/docs/my-website/docs/providers/azure/azure.md +++ b/docs/my-website/docs/providers/azure/azure.md @@ -1032,7 +1032,7 @@ print("list_batches_response=", list_batches_response) -### [Health Check Azure Batch models](./proxy/health.md#batch-models-azure-only) +### [Health Check Azure Batch models](../../proxy/health.md#batch-models-azure-only) ### [BETA] Loadbalance Multiple Azure Deployments diff --git a/docs/my-website/docs/providers/azure/azure_anthropic.md b/docs/my-website/docs/providers/azure/azure_anthropic.md index 4c722b30397..e7cd8fffbf0 100644 --- a/docs/my-website/docs/providers/azure/azure_anthropic.md +++ b/docs/my-website/docs/providers/azure/azure_anthropic.md @@ -372,7 +372,6 @@ response = completion( ## Related Documentation -- [Anthropic Provider Documentation](./anthropic.md) - For standard Anthropic API usage +- [Anthropic Provider Documentation](../anthropic.md) - For standard Anthropic API usage - [Azure OpenAI Documentation](./azure.md) - For Azure OpenAI models -- [Azure Authentication Guide](../secret_managers/azure_key_vault.md) - For Azure AD token setup - +- [Azure Authentication Guide](../../secret_managers/azure_key_vault.md) - For Azure AD token setup diff --git a/docs/my-website/docs/providers/openai.md b/docs/my-website/docs/providers/openai.md index 9d557303ef2..2907cdf9f47 100644 --- a/docs/my-website/docs/providers/openai.md +++ b/docs/my-website/docs/providers/openai.md @@ -638,7 +638,9 @@ This is useful when you want to use [Responses API](https://platform.openai.com/ :::tip gpt-5.4 + reasoning_effort + function tools -OpenAI does not support `reasoning_effort` with function tools for `gpt-5.4` in `/v1/chat/completions`. Use the responses bridge instead: +LiteLLM drops `reasoning_effort` from `gpt-5.4` requests to `litellm.completion()` that include tools, since that combination is supported in the Responses API. + +If you need reasoning **and** tools together, use the responses bridge instead: ```python response = litellm.completion( @@ -1151,4 +1153,4 @@ response = completion( LiteLLM supports OpenAI's video generation models including Sora. -For detailed documentation on video generation, see [OpenAI Video Generation →](./openai/video_generation.md) +For detailed documentation on video generation, see [OpenAI Video Generation →](./openai/videos.md) diff --git a/docs/my-website/docs/providers/openai/videos.md b/docs/my-website/docs/providers/openai/videos.md index 202c79c2446..b67800092a4 100644 --- a/docs/my-website/docs/providers/openai/videos.md +++ b/docs/my-website/docs/providers/openai/videos.md @@ -135,6 +135,81 @@ curl --location --request POST 'http://localhost:4000/v1/videos/video_id/remix' }' ``` +### Character, Edit, and Extension Routes + +OpenAI video routes supported by LiteLLM proxy: + +- `POST /v1/videos/characters` +- `GET /v1/videos/characters/{character_id}` +- `POST /v1/videos/edits` +- `POST /v1/videos/extensions` + +#### `target_model_names` support on character creation + +`POST /v1/videos/characters` supports `target_model_names` for model-based routing (same behavior as video create). + +```bash +curl --location 'http://localhost:4000/v1/videos/characters' \ +--header 'Authorization: Bearer sk-1234' \ +-F 'name=hero' \ +-F 'target_model_names=gpt-4' \ +-F 'video=@/path/to/character.mp4' +``` + +When `target_model_names` is used, LiteLLM returns an encoded character ID: + +```json +{ + "id": "character_...", + "object": "character", + "created_at": 1712697600, + "name": "hero" +} +``` + +Use that encoded ID directly on get: + +```bash +curl --location 'http://localhost:4000/v1/videos/characters/character_...' \ +--header 'Authorization: Bearer sk-1234' +``` + +#### Encoded and non-encoded video IDs for edit/extension + +Both routes accept either plain or encoded `video.id`: + +- `POST /v1/videos/edits` +- `POST /v1/videos/extensions` + +```bash +curl --location 'http://localhost:4000/v1/videos/edits' \ +--header 'Authorization: Bearer sk-1234' \ +--header 'Content-Type: application/json' \ +--data '{ + "prompt": "Make this brighter", + "video": { "id": "video_..." } +}' +``` + +```bash +curl --location 'http://localhost:4000/v1/videos/extensions' \ +--header 'Authorization: Bearer sk-1234' \ +--header 'Content-Type: application/json' \ +--data '{ + "prompt": "Continue this scene", + "seconds": "4", + "video": { "id": "video_..." } +}' +``` + +#### `custom_llm_provider` input sources + +For these routes, `custom_llm_provider` may be supplied via: + +- header: `custom-llm-provider` +- query: `?custom_llm_provider=...` +- body: `custom_llm_provider` (and `extra_body.custom_llm_provider` where supported) + Test OpenAI video generation request ```bash diff --git a/docs/my-website/docs/providers/vertex_partner.md b/docs/my-website/docs/providers/vertex_partner.md index 48a116eb7a8..75ec3b93087 100644 --- a/docs/my-website/docs/providers/vertex_partner.md +++ b/docs/my-website/docs/providers/vertex_partner.md @@ -11,6 +11,7 @@ import TabItem from '@theme/TabItem'; |----------|---------------|---------------| | Anthropic (Claude) | `vertex_ai/claude-*` | [Vertex AI - Anthropic Models](https://cloud.google.com/vertex-ai/generative-ai/docs/partner-models/use-claude) | | DeepSeek | `vertex_ai/deepseek-ai/{MODEL}` | [Vertex AI - DeepSeek Models](https://cloud.google.com/vertex-ai/generative-ai/docs/maas/deepseek) | +| ZAI (GLM) | `vertex_ai/zai-org/{MODEL}` | [Vertex AI - GLM Models](https://cloud.google.com/vertex-ai/generative-ai/docs/maas/zaiorg/glm-47) | | Meta/Llama | `vertex_ai/meta/{MODEL}` | [Vertex AI - Meta Models](https://cloud.google.com/vertex-ai/generative-ai/docs/partner-models/llama) | | Mistral | `vertex_ai/mistral-*` | [Vertex AI - Mistral Models](https://cloud.google.com/vertex-ai/generative-ai/docs/partner-models/mistral) | | AI21 (Jamba) | `vertex_ai/jamba-*` | [Vertex AI - AI21 Models](https://cloud.google.com/vertex-ai/generative-ai/docs/partner-models/ai21) | @@ -226,6 +227,79 @@ ModelResponse( |------------------|------------------------------| | vertex_ai/deepseek-ai/deepseek-r1-0528-maas | `completion('vertex_ai/deepseek-ai/deepseek-r1-0528-maas', messages)` | +## VertexAI ZAI (GLM) + +| Property | Details | +|----------|---------| +| Provider Route | `vertex_ai/zai-org/{MODEL}` | +| Vertex Documentation | [Vertex AI - GLM Models](https://cloud.google.com/vertex-ai/generative-ai/docs/maas/zaiorg/glm-47) | + +**LiteLLM Supports all Vertex AI GLM Models.** Ensure you use the `vertex_ai/zai-org/` prefix for all Vertex AI GLM models. + +| Model Name | Usage | +|------------|-------| +| vertex_ai/zai-org/glm-4.7-maas | `completion('vertex_ai/zai-org/glm-4.7-maas', messages)` | + +#### Usage + + + + +```python +from litellm import completion +import os + +os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = "" + +response = completion( + model="vertex_ai/zai-org/glm-4.7-maas", + messages=[{"role": "user", "content": "hi"}], + vertex_project="your-vertex-project", + # vertex_location routes to "global" +) +print("\nModel Response", response) +``` + + + +**1. Add to config** + +```yaml +model_list: + - model_name: glm-4.7 + litellm_params: + model: vertex_ai/zai-org/glm-4.7-maas + vertex_project: "my-project" + # vertex_location routes to "global" +``` + +**2. Start proxy** + +```bash +litellm --config /path/to/config.yaml + +# RUNNING at http://0.0.0.0:4000 +``` + +**3. Test it!** + +```bash +curl --location 'http://0.0.0.0:4000/chat/completions' \ + --header 'Authorization: Bearer sk-1234' \ + --header 'Content-Type: application/json' \ + --data '{ + "model": "glm-4.7", + "messages": [ + { + "role": "user", + "content": "what llm are you" + } + ], + }' +``` + + + ## VertexAI Meta/Llama API diff --git a/docs/my-website/docs/proxy/call_hooks.md b/docs/my-website/docs/proxy/call_hooks.md index 17354725fd5..5935a29c50b 100644 --- a/docs/my-website/docs/proxy/call_hooks.md +++ b/docs/my-website/docs/proxy/call_hooks.md @@ -7,7 +7,7 @@ import Image from '@theme/IdealImage'; - Enforce 'user' param for all openai endpoint calls :::tip -**Understanding Callback Hooks?** Check out our [Callback Management Guide](../observability/callback_management.md) to understand the differences between proxy-specific hooks like `async_pre_call_hook` and general logging hooks like `async_log_success_event`. +**Understanding Callback Hooks?** Check out our [Callback Guide](../observability/callbacks.md) to understand the differences between proxy-specific hooks like `async_pre_call_hook` and general logging hooks like `async_log_success_event`. ::: ## Which Hook Should I Use? diff --git a/docs/my-website/docs/proxy/config_settings.md b/docs/my-website/docs/proxy/config_settings.md index 90d71fb95c8..d7542fc2c3d 100644 --- a/docs/my-website/docs/proxy/config_settings.md +++ b/docs/my-website/docs/proxy/config_settings.md @@ -364,7 +364,7 @@ router_settings: | optional_pre_call_checks | List[str] | List of pre-call checks to add to the router. Supported: `router_budget_limiting`, `prompt_caching`, `responses_api_deployment_check`, `encrypted_content_affinity`, `deployment_affinity`, `session_affinity`, `forward_client_headers_by_model_group` | | deployment_affinity_ttl_seconds | int | TTL (seconds) for user-key → deployment affinity mapping when `deployment_affinity` is enabled (configured at Router init / proxy startup). Defaults to `3600` (1 hour). | | ignore_invalid_deployments | boolean | If true, ignores invalid deployments. Default for proxy is True - to prevent invalid models from blocking other models from being loaded. | -| search_tools | List[SearchToolTypedDict] | List of search tool configurations for Search API integration. Each tool specifies a search_tool_name and litellm_params with search_provider, api_key, api_base, etc. [Further Docs](../search.md) | +| search_tools | List[SearchToolTypedDict] | List of search tool configurations for Search API integration. Each tool specifies a search_tool_name and litellm_params with search_provider, api_key, api_base, etc. [Further Docs](../search/index.md) | | guardrail_list | List[GuardrailTypedDict] | List of guardrail configurations for guardrail load balancing. Enables load balancing across multiple guardrail deployments with the same guardrail_name. [Further Docs](./guardrails/guardrail_load_balancing.md) | @@ -401,8 +401,10 @@ router_settings: | AUTH_STRATEGY | Strategy used for authentication (e.g., OAuth, API key) | AUTO_REDIRECT_UI_LOGIN_TO_SSO | Flag to enable automatic redirect of UI login page to SSO when SSO is configured. Default is **false** | AUDIO_SPEECH_CHUNK_SIZE | Chunk size for audio speech processing. Default is 1024 -| ANTHROPIC_API_KEY | API key for Anthropic service +| ANTHROPIC_API_KEY | API key for Anthropic service. Uses `x-api-key` header for authentication. +| ANTHROPIC_AUTH_TOKEN | Alternative auth token for Anthropic service. Uses `Authorization: Bearer` header instead of `x-api-key`. Used as fallback when `ANTHROPIC_API_KEY` is not set. | ANTHROPIC_API_BASE | Base URL for Anthropic API. Default is https://api.anthropic.com +| ANTHROPIC_BASE_URL | Alternative to `ANTHROPIC_API_BASE` for setting the Anthropic API base URL. Used as fallback when `ANTHROPIC_API_BASE` is not set. | ANTHROPIC_TOKEN_COUNTING_BETA_VERSION | Beta version header for Anthropic token counting API. Default is `token-counting-2024-11-01` | AWS_ACCESS_KEY_ID | Access Key ID for AWS services | AWS_BATCH_ROLE_ARN | ARN of the AWS IAM role for batch operations @@ -778,6 +780,7 @@ router_settings: | LITELLM_GLOBAL_MAX_PARALLEL_REQUEST_RETRIES | Maximum retries for parallel requests in LiteLLM | LITELLM_GLOBAL_MAX_PARALLEL_REQUEST_RETRY_TIMEOUT | Timeout for retries of parallel requests in LiteLLM | LITELLM_DISABLE_LAZY_LOADING | When set to "1", "true", "yes", or "on", disables lazy loading of attributes (currently only affects encoding/tiktoken). This ensures encoding is initialized before VCR starts recording HTTP requests, fixing VCR cassette creation issues. See [issue #18659](https://github.com/BerriAI/litellm/issues/18659) +| LITELLM_DISABLE_REDACT_SECRETS | When set to "true", disables automatic redaction of secrets (API keys, tokens, credentials) from proxy log output. Secret redaction is enabled by default. | LITELLM_MIGRATION_DIR | Custom migrations directory for prisma migrations, used for baselining db in read-only file systems. | LITELLM_HOSTED_UI | URL of the hosted UI for LiteLLM | LITELLM_UI_API_DOC_BASE_URL | Optional override for the API Reference base URL (used in sample code/docs) when the admin UI runs on a different host than the proxy. Defaults to `PROXY_BASE_URL` when unset. @@ -901,6 +904,7 @@ router_settings: | OTEL_SERVICE_NAME | Service name identifier for OpenTelemetry | OTEL_TRACER_NAME | Tracer name for OpenTelemetry tracing | OTEL_LOGS_EXPORTER | Exporter type for OpenTelemetry logs (e.g., console) +| OTEL_IGNORE_CONTEXT_PROPAGATION | When true, ignore parent span context propagation in OpenTelemetry callbacks | PAGERDUTY_API_KEY | API key for PagerDuty Alerting | PANW_PRISMA_AIRS_API_KEY | API key for PANW Prisma AIRS service | PANW_PRISMA_AIRS_API_BASE | Base URL for PANW Prisma AIRS service @@ -910,6 +914,7 @@ router_settings: | PILLAR_API_BASE | Base URL for Pillar API Guardrails | PILLAR_API_KEY | API key for Pillar API Guardrails | PILLAR_ON_FLAGGED_ACTION | Action to take when content is flagged ('block' or 'monitor') +| PKCE_STRICT_CACHE_MISS | When set to `true`, the SSO callback will return a 401 error if the PKCE code_verifier is not found in the cache (e.g. due to a cache miss across pods). When `false` (default), it logs a warning and continues without the code_verifier. | POD_NAME | Pod name for the server, this will be [emitted to `datadog` logs](https://docs.litellm.ai/docs/proxy/logging#datadog) as `POD_NAME` | POSTHOG_API_KEY | API key for PostHog analytics integration | POSTHOG_API_URL | Base URL for PostHog API (defaults to https://us.i.posthog.com) @@ -934,6 +939,9 @@ router_settings: | PROXY_BASE_URL | Base URL for proxy service | PROXY_BATCH_WRITE_AT | Time in seconds to wait before batch writing spend logs to the database. Default is 10 | PROXY_BATCH_POLLING_INTERVAL | Time in seconds to wait before polling a batch, to check if it's completed. Default is 6000s (1 hour) +| PROXY_BATCH_POLLING_ENABLED | Set to `false` to disable the `CheckBatchCost` and `CheckResponsesCost` background polling jobs entirely. Useful for emergency mitigation on installs with large numbers of stale managed objects. Default is `true` +| MAX_OBJECTS_PER_POLL_CYCLE | Maximum number of managed objects (batches / responses) fetched per polling cycle. Prevents OOM on installs with many stale rows. Default is `50` +| MANAGED_OBJECT_STALENESS_CUTOFF_DAYS | Managed objects older than this many days in a non-terminal state are marked `stale_expired` at the start of each poll cycle and skipped. Default is `7` | PROXY_BUDGET_RESCHEDULER_MAX_TIME | Maximum time in seconds to wait before checking database for budget resets. Default is 605 | PROXY_BUDGET_RESCHEDULER_MIN_TIME | Minimum time in seconds to wait before checking database for budget resets. Default is 597 | PYTHON_GC_THRESHOLD | GC thresholds ('gen0,gen1,gen2', e.g. '1000,50,50'); defaults to Python’s values. @@ -1017,6 +1025,11 @@ router_settings: | UPSTREAM_LANGFUSE_SECRET_KEY | Secret key for upstream Langfuse authentication | USE_AWS_KMS | Flag to enable AWS Key Management Service for encryption | USE_PRISMA_MIGRATE | Flag to use prisma migrate instead of prisma db push. Recommended for production environments. +| VANTAGE_API_KEY | API key for Vantage cost-import integration +| VANTAGE_BASE_URL | Base URL for Vantage API. Default is `https://api.vantage.sh` +| VANTAGE_EXPORT_FREQUENCY | Export frequency for Vantage — `hourly` (default), `daily`, or `interval` +| VANTAGE_EXPORT_INTERVAL_SECONDS | Interval in seconds when VANTAGE_EXPORT_FREQUENCY is `interval` +| VANTAGE_INTEGRATION_TOKEN | Vantage integration token for the cost-import endpoint | WANDB_API_KEY | API key for Weights & Biases (W&B) logging integration | WANDB_HOST | Host URL for Weights & Biases (W&B) service | WANDB_PROJECT_ID | Project ID for Weights & Biases (W&B) logging integration diff --git a/docs/my-website/docs/proxy/configs.md b/docs/my-website/docs/proxy/configs.md index 56a8b9566db..84a6fac1210 100644 --- a/docs/my-website/docs/proxy/configs.md +++ b/docs/my-website/docs/proxy/configs.md @@ -602,6 +602,22 @@ Since you shouldn't use 12.5, round down to **10** to leave a safety buffer. Thi - Total maximum connections: 8 workers × 10 connections = 80 connections - This stays safely under your database's 100 connection limit +## LiteLLM License Key (Enterprise) + +To enable [LiteLLM Enterprise features](https://docs.litellm.ai/docs/proxy/enterprise), set your license key as an environment variable: + +```bash +export LITELLM_LICENSE="eyJ..." +``` + +The license key is a JWT token provided when you purchase a LiteLLM Enterprise license. Once set, LiteLLM will automatically detect and activate enterprise features. + +You can also add it to your `.env` file: + +```env +LITELLM_LICENSE="eyJ..." +``` + ## Extras diff --git a/docs/my-website/docs/proxy/docker_quick_start.md b/docs/my-website/docs/proxy/docker_quick_start.md index efdc73de43e..58a56604751 100644 --- a/docs/my-website/docs/proxy/docker_quick_start.md +++ b/docs/my-website/docs/proxy/docker_quick_start.md @@ -1,25 +1,90 @@ - import Tabs from '@theme/Tabs'; import TabItem from '@theme/TabItem'; +import Image from '@theme/IdealImage'; # Getting Started Tutorial End-to-End tutorial for LiteLLM Proxy to: -- Add an Azure OpenAI model -- Make a successful /chat/completion call -- Generate a virtual key -- Set RPM limit on virtual key +- Add an Azure OpenAI model +- Make a successful /chat/completion call +- Generate a virtual key +- Set RPM limit on virtual key +## Quick Install (Recommended for local / beginners) + +New to LiteLLM? This is the easiest way to get started locally. One command installs LiteLLM and walks you through setup interactively — no config files to write by hand. + +### 1. Install + +```bash +curl -fsSL https://raw.githubusercontent.com/BerriAI/litellm/main/scripts/install.sh | sh +``` + +This detects your OS, installs `litellm[proxy]`, and drops you straight into the setup wizard. + +### 2. Follow the wizard + +``` +$ litellm --setup + + Welcome to LiteLLM + + Choose your LLM providers + ○ 1. OpenAI GPT-4o, GPT-4o-mini, o1 + ○ 2. Anthropic Claude Opus, Sonnet, Haiku + ○ 3. Azure OpenAI GPT-4o via Azure + ○ 4. Google Gemini Gemini 2.0 Flash, 1.5 Pro + ○ 5. AWS Bedrock Claude, Llama via AWS + ○ 6. Ollama Local models + + ❯ Provider(s): 1,2 + + ❯ OpenAI API key: sk-... + ❯ Anthropic API key: sk-ant-... + + ❯ Port [4000]: + ❯ Master key [auto-generate]: + + ✔ Config saved → ./litellm_config.yaml + + ❯ Start the proxy now? (Y/n): +``` + +The wizard walks you through: +1. Pick your LLM providers (OpenAI, Anthropic, Azure, Bedrock, Gemini, Ollama) +2. Enter API keys for each provider +3. Set a port and master key (or accept the defaults) +4. Config is saved to `./litellm_config.yaml` and the proxy starts immediately + +### 3. Make a call + +Your proxy is running on `http://0.0.0.0:4000`. Test it: + +```bash +curl -X POST 'http://0.0.0.0:4000/chat/completions' \ +-H 'Content-Type: application/json' \ +-H 'Authorization: Bearer ' \ +-d '{ + "model": "gpt-4o", + "messages": [{"role": "user", "content": "Hello!"}] +}' +``` + +:::tip Already have pip installed? +You can skip the curl install and run `litellm --setup` directly after `pip install 'litellm[proxy]'`. +::: + +--- ## Pre-Requisites -- Install LiteLLM Docker Image **OR** LiteLLM CLI (pip package) +Choose your install method. **Docker Compose** users complete their full setup inside the tab and are done. **Docker** and **pip** users continue with the steps below the tabs. -``` +```bash docker pull docker.litellm.ai/berriai/litellm:main-latest ``` @@ -37,7 +102,25 @@ $ pip install 'litellm[proxy]' -Use this docker compose to spin up the proxy with a postgres database running locally. +Docker Compose bundles LiteLLM with a Postgres database. Follow the steps below — the proxy will be fully running by the end. + +### Step 1 — Pull the LiteLLM database image + +LiteLLM provides a dedicated `litellm-database` image for proxy deployments that connect to Postgres. + +```bash +docker pull ghcr.io/berriai/litellm-database:main-latest +``` + +See all available tags on the [GitHub Container Registry](https://github.com/BerriAI/litellm/pkgs/container/litellm-database). + +--- + +### Step 2 — Set up a database + +Complete all three config files **before** running `docker compose up`. The proxy server will not start correctly if any of these are missing. + +#### 2.1 — Get `docker-compose.yml` and create `.env` ```bash # Get the docker compose file @@ -46,26 +129,154 @@ curl -O https://raw.githubusercontent.com/BerriAI/litellm/main/docker-compose.ym # Add the master key - you can change this after setup echo 'LITELLM_MASTER_KEY="sk-1234"' > .env -# Add the litellm salt key - you cannot change this after adding a model -# It is used to encrypt / decrypt your LLM API Key credentials -# We recommend - https://1password.com/password-generator/ -# password generator to get a random hash for litellm salt key +# Add the litellm salt key — cannot be changed after adding a model +# Used to encrypt/decrypt your LLM API key credentials +# Generate a strong random value: https://1password.com/password-generator/ echo 'LITELLM_SALT_KEY="sk-1234"' >> .env -# Start +# Add your model credentials +echo 'AZURE_API_BASE="https://openai-***********/"' >> .env +echo 'AZURE_API_KEY="your-azure-api-key"' >> .env +``` + +#### 2.2 — Create `config.yaml` + +The default `docker-compose.yml` starts a Postgres container at `db:5432`. Your `config.yaml` must include `database_url` pointing to it: + +```yaml +model_list: + - model_name: gpt-4o + litellm_params: + model: azure/my_azure_deployment + api_base: os.environ/AZURE_API_BASE + api_key: os.environ/AZURE_API_KEY + api_version: "2025-01-01-preview" + +general_settings: + master_key: sk-1234 # 🔑 your proxy admin key (must start with sk-) + database_url: "postgresql://llmproxy:dbpassword9090@db:5432/litellm" +``` + +:::tip +`database_url` enables virtual keys, spend tracking, and the UI. Replace it with your [Supabase](https://supabase.com/) or [Neon](https://neon.tech/) connection string if you prefer a managed database. +::: + +#### 2.3 — Create `prometheus.yml` + +This file **must exist as a file** before `docker compose up`. If it is missing, Docker auto-creates it as an empty directory and the Prometheus container fails to start. + +```yaml +global: + scrape_interval: 15s + evaluation_interval: 15s + +scrape_configs: + - job_name: "litellm" + static_configs: + - targets: ["litellm:4000"] +``` + +Also verify that the `config.yaml` volume mount and `--config` flag are **not commented out** in `docker-compose.yml`: + +```yaml +services: + litellm: + volumes: + - ./config.yaml:/app/config.yaml # ✅ must be uncommented + command: + - "--config=/app/config.yaml" # ✅ must be uncommented +``` + +:::warning +All three files (`.env`, `config.yaml`, `prometheus.yml`) must be present before running `docker compose up`. See [Troubleshooting](#troubleshooting) if you run into issues. +::: + +--- + +### Step 3 — Start the proxy server and test it + +After `config.yaml`, `prometheus.yml`, and `.env` are complete, start the proxy: + +```bash docker compose up ``` +Once running, test it with a curl request: + +```bash +curl -X POST 'http://0.0.0.0:4000/chat/completions' \ + -H 'Content-Type: application/json' \ + -H 'Authorization: Bearer sk-1234' \ + -d '{ + "model": "gpt-4o", + "messages": [{"role": "user", "content": "Hello!"}] + }' +``` + +**Expected response:** + +```json +{ + "id": "chatcmpl-abcd", + "created": 1773817678, + "model": "gpt-4o", + "object": "chat.completion", + "system_fingerprint": "fp_6b1ef07cda", + "choices": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "Hello! How can I assist you today?", + "role": "assistant", + "annotations": [] + } + } + ], + "usage": { + "completion_tokens": 9, + "prompt_tokens": 9, + "total_tokens": 18, + "completion_tokens_details": { + "accepted_prediction_tokens": 0, + "audio_tokens": 0, + "reasoning_tokens": 0, + "rejected_prediction_tokens": 0 + }, + "prompt_tokens_details": { + "audio_tokens": 0, + "cached_tokens": 0 + } + }, + "service_tier": "default" +} +``` + +--- + +### Optional — Navigate to the LiteLLM UI and generate a virtual key + +Open [http://localhost:4000/ui](http://localhost:4000/ui) in your browser and log in with your master key (`sk-1234`). + +Navigate to **Virtual Keys** and click **+ Create New Key**: + +LiteLLM UI — Create Virtual Key + +Virtual keys let you track spend, set rate limits, and control model access per user or team. + + -## 1. Add a model +:::note Docker Compose users +Your setup is complete — the steps below are for **Docker** and **pip** users only. +::: -Control LiteLLM Proxy with a config.yaml file. +--- -Setup your config.yaml with your azure model. +## Step 1 — Add a model -Note: When using the proxy with a database, you can also **just add models via UI** (UI is available on `/ui` route). +Control LiteLLM Proxy with a `config.yaml` file. Create one with your Azure model: ```yaml model_list: @@ -89,8 +300,6 @@ You can read more about how model resolution works in the [Model Configuration]( - **`api_base`** (`str`) - The API base for your azure deployment. - **`api_version`** (`str`) - The API Version to use when calling Azure's OpenAI API. Get the latest Inference API version [here](https://learn.microsoft.com/en-us/azure/ai-services/openai/api-version-deprecation?source=recommendations#latest-preview-api-releases). ---- - --- @@ -138,19 +347,19 @@ $ litellm --config /app/config.yaml --detailed_debug +Confirm your config was loaded correctly — you should see this in the logs: -Confirm your config.yaml got mounted correctly - -```bash +``` Loaded config YAML (api_key and environment_variables are not shown): { -"model_list": [ -{ -"model_name ... + "model_list": [ + { + "model_name": ... ``` ### 2.2 Make Call +LiteLLM Proxy is 100% OpenAI-compatible. Test your model via `/chat/completions`: ```bash curl -X POST 'http://0.0.0.0:4000/chat/completions' \ @@ -244,15 +453,17 @@ curl -X POST 'http://0.0.0.0:4000/chat/completions' \ - [Other/Non-Chat Completion Endpoints](../embedding/supported_embedding.md) - [Pass-through for VertexAI, Bedrock, etc.](../pass_through/vertex_ai.md) -## 3. Generate a virtual key +## Optional: Generate a virtual key -Track Spend, and control model access via virtual keys for the proxy +Track spend and control model access via virtual keys for the proxy. -### 3.1 Set up a Database +### Prerequisite — Set up a database -**Requirements** -- Need a postgres database (e.g. [Supabase](https://supabase.com/), [Neon](https://neon.tech/), etc) +:::note Docker Compose users +Your Postgres container is already running — skip ahead to [Create Key w/ RPM Limit](#create-key-w-rpm-limit) below. +::: +**Docker / pip users** — you need a Postgres database (e.g. [Supabase](https://supabase.com/), [Neon](https://neon.tech/), or self-hosted). Add `general_settings` to your `config.yaml`: ```yaml model_list: @@ -268,7 +479,9 @@ general_settings: database_url: "postgresql://:@:/" # 👈 KEY CHANGE ``` -Save config.yaml as `litellm_config.yaml` (used in 3.2). +Save config.yaml as `litellm_config.yaml` before continuing. + +You must finish this setup before starting the proxy server. --- @@ -294,7 +507,7 @@ See All General Settings [here](http://localhost:3000/docs/proxy/configs#all-set `database_url: "postgresql://..."` - Set `DATABASE_URL=postgresql://:@:/` in your env -### 3.2 Start Proxy +### Start Proxy ```bash docker run \ @@ -302,12 +515,11 @@ docker run \ -e AZURE_API_KEY=d6*********** \ -e AZURE_API_BASE=https://openai-***********/ \ -p 4000:4000 \ - docker.litellm.ai/berriai/litellm:main-latest \ + ghcr.io/berriai/litellm-database:main-latest \ --config /app/config.yaml --detailed_debug ``` - -### 3.3 Create Key w/ RPM Limit +### Create Key w/ RPM Limit Create a key with `rpm_limit: 1`. This will only allow 1 request per minute for calls to proxy with this key. @@ -330,9 +542,9 @@ curl -L -X POST 'http://0.0.0.0:4000/key/generate' \ } ``` -### 3.4 Test it! +### Test it! -**Use your virtual key from step 3.3** +**Use the virtual key you just created.** 1st call - Expect to work! @@ -546,6 +758,24 @@ model_list: ## Troubleshooting +### `prometheus.yml` mount error — "not a directory" + +If you see: + +```bash +Error: cannot create subdirectories in ".../prometheus.yml": not a directory +``` + +Docker created `prometheus.yml` as an **empty directory** instead of a file. This happens when the file is missing at `docker compose up` time. + +Fix it: +Then create the file (see [Step 2.3 — Create `prometheus.yml`](#23--create-prometheusyml)) and run `docker compose up` again. +```bash +rm -rf prometheus.yml +``` + +Then create the file (see [Step 2.4](#step-24--create-prometheusyml)) and run `docker compose up` again. + ### Non-root docker image? If you need to run the docker image as a non-root user, use [this](https://github.com/BerriAI/litellm/pkgs/container/litellm-non_root). @@ -645,6 +875,3 @@ LiteLLM Proxy uses the [LiteLLM Python SDK](https://docs.litellm.ai/docs/routing - Our emails ✉️ ishaan@berri.ai / krrish@berri.ai [![Chat on WhatsApp](https://img.shields.io/static/v1?label=Chat%20on&message=WhatsApp&color=success&logo=WhatsApp&style=flat-square)](https://wa.link/huol9n) [![Chat on Discord](https://img.shields.io/static/v1?label=Chat%20on&message=Discord&color=blue&logo=Discord&style=flat-square)](https://discord.gg/wuPM9dRgDw) - - - diff --git a/docs/my-website/docs/proxy/endpoint_activity.md b/docs/my-website/docs/proxy/endpoint_activity.md index a66c0f7a5e5..d06727ce4b4 100644 --- a/docs/my-website/docs/proxy/endpoint_activity.md +++ b/docs/my-website/docs/proxy/endpoint_activity.md @@ -114,4 +114,4 @@ Understand spend distribution across endpoints: - [Customer Usage](./customer_usage.md) - Track spend and usage for individual customers - [Cost Tracking](./cost_tracking.md) - Comprehensive cost tracking and analytics -- [Spend Logs](./spend_logs.md) - Detailed request-level spend logs +- [Spend Logs](./cost_tracking.md#-spend-logs-api---individual-transaction-logs) - Detailed request-level spend logs diff --git a/docs/my-website/docs/proxy/keys_teams_router_settings.md b/docs/my-website/docs/proxy/keys_teams_router_settings.md index ec59e8f271b..6a1744ca951 100644 --- a/docs/my-website/docs/proxy/keys_teams_router_settings.md +++ b/docs/my-website/docs/proxy/keys_teams_router_settings.md @@ -146,5 +146,5 @@ Test new router settings on specific keys or teams before applying globally: - [Router Settings Reference](./config_settings.md#router_settings---reference) - Complete reference of all router settings - [Load Balancing](./load_balancing.md) - Learn about routing strategies and load balancing - [Reliability](./reliability.md) - Configure fallbacks, retries, and error handling -- [Keys](./keys.md) - Manage API keys and their settings -- [Teams](./teams.md) - Organize keys into teams +- [Keys](./virtual_keys.md) - Manage API keys and their settings +- [Teams](./multi_tenant_architecture.md) - Organize keys into teams diff --git a/docs/my-website/docs/proxy/model_compare_ui.md b/docs/my-website/docs/proxy/model_compare_ui.md index bd6f5414224..ee0376ed2fa 100644 --- a/docs/my-website/docs/proxy/model_compare_ui.md +++ b/docs/my-website/docs/proxy/model_compare_ui.md @@ -187,7 +187,7 @@ Use tags and multiple comparisons to run structured A/B tests: ## Related Features -- [Playground Chat UI](./playground.md) - Single model testing interface +- [Playground Chat UI](./ui.md) - Single model testing interface - [Model Management](./model_management.md) - Configure and manage models -- [Guardrails](./guardrails.md) - Set up safety filters +- [Guardrails](./guardrails/quick_start.md) - Set up safety filters - [AI Hub](./ai_hub.md) - Share models and agents with your organization diff --git a/docs/my-website/docs/proxy/realtime_webrtc.md b/docs/my-website/docs/proxy/realtime_webrtc.md new file mode 100644 index 00000000000..694f293652b --- /dev/null +++ b/docs/my-website/docs/proxy/realtime_webrtc.md @@ -0,0 +1,84 @@ +# /realtime - WebRTC Support + +Connect to the Realtime API via WebRTC from browser/mobile clients. LiteLLM handles auth; audio streams directly to OpenAI/Azure. + +**Providers:** OpenAI · Azure + +:::info **WebRTC vs WebSocket** +- **WebSocket** (`/v1/realtime`) — server-to-server +- **WebRTC** (`/v1/realtime/client_secrets` + `/v1/realtime/calls`) — browser/mobile, lower latency +::: + +## How it works + +LiteLLM issues tokens and relays SDP; audio never passes through the proxy. + +``` +Browser LiteLLM Proxy OpenAI/Azure + | | | + |-- POST client_secrets --->|-- POST sessions -------->| + |<-- encrypted_token -------|<-- ek_... ---------------| + |-- POST calls [SDP+token] ->|-- POST calls ----------->| + |<-- SDP answer ------------|<-- SDP answer -----------| + |===== audio P2P direct ===============================>| +``` + +## Proxy Setup + +```yaml +model_list: + - model_name: gpt-4o-realtime + litellm_params: + model: openai/gpt-4o-realtime-preview-2024-12-17 + api_key: os.environ/OPENAI_API_KEY + model_info: + mode: realtime +``` + +**Azure:** `model: azure/gpt-4o-realtime-preview`, `api_key`, `api_base`. + +```bash +litellm --config /path/to/config.yaml +``` + +## Client Usage + +1. **Token** — `POST /v1/realtime/client_secrets` with LiteLLM key and `{ model }`. +2. **WebRTC** — Create `RTCPeerConnection`, add mic, data channel `oai-events`, send SDP offer to `POST /v1/realtime/calls` with `Authorization: Bearer `, `Content-Type: application/sdp`. +3. **Events** — Use data channel for `session.update` and other events. + +```javascript +const r = await fetch("http://proxy:4000/v1/realtime/client_secrets", { + method: "POST", + headers: { "Authorization": "Bearer sk-litellm-key", "Content-Type": "application/json" }, + body: JSON.stringify({ model: "gpt-4o-realtime" }), +}); +const token = (await r.json()).client_secret.value; + +const pc = new RTCPeerConnection(); +const audio = document.createElement("audio"); +audio.autoplay = true; +pc.ontrack = (e) => (audio.srcObject = e.streams[0]); +const ms = await navigator.mediaDevices.getUserMedia({ audio: true }); +pc.addTrack(ms.getTracks()[0]); +const dc = pc.createDataChannel("oai-events"); +const offer = await pc.createOffer(); +await pc.setLocalDescription(offer); + +const sdpRes = await fetch("http://proxy:4000/v1/realtime/calls", { + method: "POST", + headers: { "Authorization": `Bearer ${token}`, "Content-Type": "application/sdp" }, + body: offer.sdp, +}); +await pc.setRemoteDescription({ type: "answer", sdp: await sdpRes.text() }); + +dc.send(JSON.stringify({ type: "session.update", session: { instructions: "..." } })); +``` + +## FAQ + +- **401 Token expired** — Get a fresh token right before creating the WebRTC offer. +- **Which key for `/calls`?** — Encrypted token from `client_secrets`, not raw key. +- **Pass `model`?** — No. Token encodes routing. +- **Azure `api-version`** — Set `api_version` in `litellm_params` and correct `api_base`. +- **No audio** — Grant mic; ensure `pc.ontrack` sets autoplay audio; check firewall/WebRTC; inspect console. \ No newline at end of file diff --git a/docs/my-website/docs/proxy/sync_anthropic_beta_headers.md b/docs/my-website/docs/proxy/sync_anthropic_beta_headers.md index e1645082d97..0373d20c879 100644 --- a/docs/my-website/docs/proxy/sync_anthropic_beta_headers.md +++ b/docs/my-website/docs/proxy/sync_anthropic_beta_headers.md @@ -125,4 +125,4 @@ curl -X DELETE "https://your-proxy-url/schedule/anthropic_beta_headers_reload" \ ## Related - [Model Cost Map Sync](./sync_models_github.md) - Auto-sync model pricing data -- [Anthropic Beta Headers](../completion/anthropic.md#beta-features) - Using Anthropic beta features +- [Anthropic Beta Headers](../providers/anthropic.md) - Using Anthropic beta features diff --git a/docs/my-website/docs/proxy/tag_routing.md b/docs/my-website/docs/proxy/tag_routing.md index 399c43d2c0f..a1ae52e5e45 100644 --- a/docs/my-website/docs/proxy/tag_routing.md +++ b/docs/my-website/docs/proxy/tag_routing.md @@ -209,6 +209,106 @@ Expect to see the following response header when this works x-litellm-model-id: default-model ``` +## Regex-based tag routing (`tag_regex`) + +Use `tag_regex` to route requests based on regex patterns matched against request headers, without requiring clients to pass a tag explicitly. This is useful when clients already send a recognisable header, such as `User-Agent`. + +**Use case: route all Claude Code traffic to dedicated AWS accounts** + +Claude Code always sends `User-Agent: claude-code/`. With `tag_regex` you can route that traffic to a dedicated deployment automatically — no per-developer configuration needed. + +### 1. Config + +```yaml +model_list: + # Claude Code traffic → dedicated deployment, matched by User-Agent + - model_name: claude-sonnet + litellm_params: + model: bedrock/converse/anthropic-claude-sonnet-4-6 + aws_region_name: us-east-1 + aws_role_name: arn:aws:iam::111122223333:role/LiteLLMClaudeCode + tag_regex: + - "^User-Agent: claude-code\\/" # matches claude-code/1.x, 2.x, etc. + model_info: + id: claude-code-deployment + + # All other traffic falls back to the default deployment + - model_name: claude-sonnet + litellm_params: + model: bedrock/converse/anthropic-claude-sonnet-4-6 + aws_region_name: us-east-1 + aws_role_name: arn:aws:iam::444455556666:role/LiteLLMDefault + tags: + - default + model_info: + id: regular-deployment + +router_settings: + enable_tag_filtering: true + tag_filtering_match_any: true + +general_settings: + master_key: sk-1234 +``` + +### 2. Verify routing + +Claude Code sets `User-Agent: claude-code/` automatically — no client config needed: + +```shell +# Claude Code request (User-Agent set automatically by Claude Code) +curl http://localhost:4000/v1/chat/completions \ + -H "Authorization: Bearer sk-1234" \ + -H "User-Agent: claude-code/1.2.3" \ + -d '{"model": "claude-sonnet", "messages": [{"role": "user", "content": "hi"}]}' +# → x-litellm-model-id: claude-code-deployment + +# Any other client (no matching User-Agent) → default deployment +curl http://localhost:4000/v1/chat/completions \ + -H "Authorization: Bearer sk-1234" \ + -d '{"model": "claude-sonnet", "messages": [{"role": "user", "content": "hi"}]}' +# → x-litellm-model-id: regular-deployment +``` + +### How matching works + +| Priority | Condition | Result | +|----------|-----------|--------| +| 1 | Request has `tags` AND deployment has `tags` | Exact tag match (respects `match_any` setting) | +| 2 | Deployment has `tag_regex` AND request has a `User-Agent` | Regex match (always OR logic — any pattern match suffices) | +| 3 | Deployment has `tags: [default]` | Default fallback | +| 4 | No default set | All healthy deployments returned | + +`tag_regex` always uses OR semantics — `tag_filtering_match_any=False` applies only to exact tag matching, not to regex patterns. + +### Observability + +When a regex matches, `tag_routing` is written into request metadata and flows to SpendLogs: + +```json +{ + "tag_routing": { + "matched_via": "tag_regex", + "matched_value": "^User-Agent: claude-code\\/", + "user_agent": "claude-code/1.2.3", + "request_tags": [] + } +} +``` + +### Security note + +:::caution + +**`User-Agent` is a client-supplied header and can be set to any value.** Any API consumer can send `User-Agent: claude-code/1.0` regardless of whether they are actually using Claude Code. + +Do not rely on `tag_regex` routing to enforce access controls or spend limits — use [team/key-based routing](./users) for that. `tag_regex` is a **traffic classification hint** (useful for billing visibility, capacity planning, and routing convenience), not a security boundary. + +::: + + +--- + ## ✨ Team based tag routing (Enterprise) LiteLLM Proxy supports team-based tag routing, allowing you to associate specific tags with teams and route requests accordingly. Example **Team A can access gpt-4 deployment A, Team B can access gpt-4 deployment B** (LLM Access Control For Teams) diff --git a/docs/my-website/docs/proxy/team_budgets.md b/docs/my-website/docs/proxy/team_budgets.md index 01b07f23a33..6c38e0b1b93 100644 --- a/docs/my-website/docs/proxy/team_budgets.md +++ b/docs/my-website/docs/proxy/team_budgets.md @@ -177,3 +177,7 @@ Expect to see this metric on prometheus to track the Remaining Budget for the te ```shell litellm_remaining_team_budget_metric{team_alias="QA Prod Bot",team_id="de35b29e-6ca8-4f47-b804-2b79d07aa99a"} 9.699999999999992e-06 ``` + +## See Also + +- [Per-model TPM/RPM for teams](./users.md#per-team-model) - Set rate limits per model for all keys in a team diff --git a/docs/my-website/docs/proxy/ui/ui_edit_logo.md b/docs/my-website/docs/proxy/ui/ui_edit_logo.md new file mode 100644 index 00000000000..c62a39c0050 --- /dev/null +++ b/docs/my-website/docs/proxy/ui/ui_edit_logo.md @@ -0,0 +1,138 @@ +import Image from '@theme/IdealImage'; + +# Customize UI Logo + +Personalize your LiteLLM dashboard by replacing the default logo with your own company branding. You can set a custom logo via the UI or the API. + +## Via the UI + +### 1. Navigate to Settings + +Click the **Settings** icon in the sidebar. + +![Navigate to Settings](https://colony-recorder.s3.amazonaws.com/files/2026-03-13/57a15404-51f7-481e-9db2-cea94566d3ce/ascreenshot_7a348567c839448bb806fd71cf4abca0_text_export.jpeg) + +### 2. Open UI Theme Settings + +Click **UI Theme** from the settings menu. + +![Open UI Theme](https://colony-recorder.s3.amazonaws.com/files/2026-03-13/30663fe1-9f78-4496-96d4-c53513cbaf82/ascreenshot_ac1eb59eda0e423fbd0e7d3a6cabd4c7_text_export.jpeg) + +### 3. Click the Logo URL Field + +Click the **Logo URL** text field to start editing. + +![Click Logo URL Field](https://colony-recorder.s3.amazonaws.com/files/2026-03-13/069e8412-8ec1-4d36-ba38-6b2e2858a45a/ascreenshot_8fc7fb4a3af74815bc1b69a8554bc110_text_export.jpeg) + +### 4. Find Your Logo Image + +Open a new browser tab and find the logo image you want to use (e.g., search Google Images for your company logo). + +![Find Logo Image](https://colony-recorder.s3.amazonaws.com/files/2026-03-13/d9b55dac-bc4e-4728-b422-4afbc21f9034/ascreenshot_2a805f39c83d4b5e95f43495a6ea4e79_text_export.jpeg) + +### 5. Right-Click on the Logo Image + +Right-click the image you want to use as your logo. + +![Right-Click Image](https://colony-recorder.s3.amazonaws.com/files/2026-03-13/9d42d13e-6028-4710-acb2-c6af04a855c7/ascreenshot_0f21f29ba0e44132afe483a4b88e8b70_text_export.jpeg) + +### 6. Copy the Image Address + +Select **Copy Image Address** from the context menu to copy the URL. + +![Copy Image Address](https://colony-recorder.s3.amazonaws.com/files/2026-03-13/c25637be-383a-498b-ad11-eb1761d52757/ascreenshot_b237ee800979462189a02c1e1942ebf1_text_export.jpeg) + +### 7. Switch Back to LiteLLM + +Navigate back to the LiteLLM UI tab (e.g., press **Cmd + Left** or click the tab). + +![Switch Back](https://colony-recorder.s3.amazonaws.com/files/2026-03-13/f0647856-679c-4591-9ff7-7fd3cfbc70b4/ascreenshot_3ce46dae64c94891ac0983f5ed8f085a_text_export.jpeg) + +### 8. Paste the Logo URL + +Paste the copied image URL into the **Logo URL** field with **Cmd + V**. + +![Paste URL](https://colony-recorder.s3.amazonaws.com/files/2026-03-13/54dd30d9-7a88-41e8-a580-a6acf707c7fa/ascreenshot_8a772218ac0743d9ae8ffd3311eccd5a_text_export.jpeg) + +### 9. Save Changes + +Click **Save Changes** to apply your new logo. + +![Save Changes](https://colony-recorder.s3.amazonaws.com/files/2026-03-13/4baf6494-d146-4600-b6f2-ef667338d580/ascreenshot_722cbcd568ec4267af5122b3958bb248_text_export.jpeg) + +Your custom logo will now appear in the LiteLLM dashboard sidebar and login page. + +## Via the API + +### Set a Custom Logo + +```bash +curl -X PATCH 'http://localhost:4000/settings/update/ui_theme_settings' \ + -H 'Authorization: Bearer ' \ + -H 'Content-Type: application/json' \ + -d '{ + "logo_url": "https://example.com/your-company-logo.png" + }' +``` + +### Set a Custom Favicon + +You can also customize the browser tab favicon: + +```bash +curl -X PATCH 'http://localhost:4000/settings/update/ui_theme_settings' \ + -H 'Authorization: Bearer ' \ + -H 'Content-Type: application/json' \ + -d '{ + "logo_url": "https://example.com/your-company-logo.png", + "favicon_url": "https://example.com/your-favicon.ico" + }' +``` + +### Get Current Theme Settings + +```bash +curl -X GET 'http://localhost:4000/settings/get/ui_theme_settings' +``` + +### Reset to Default Logo + +Send an empty `logo_url` to restore the default LiteLLM logo: + +```bash +curl -X PATCH 'http://localhost:4000/settings/update/ui_theme_settings' \ + -H 'Authorization: Bearer ' \ + -H 'Content-Type: application/json' \ + -d '{ + "logo_url": "" + }' +``` + +## Via `proxy_config.yaml` + +You can also set the logo URL in your proxy configuration file: + +```yaml +litellm_settings: + ui_theme_config: + logo_url: "https://example.com/your-company-logo.png" + favicon_url: "https://example.com/your-favicon.ico" # optional +``` + +Or set it as an environment variable: + +```yaml +environment_variables: + UI_LOGO_PATH: "https://example.com/your-company-logo.png" +``` + +## Supported Logo Formats + +| Format | Supported | +|--------|-----------| +| JPEG / JPG | Yes | +| PNG | Yes | +| SVG | Yes | +| ICO (favicon only) | Yes | +| HTTP/HTTPS URL | Yes | +| Local file path | Yes | diff --git a/docs/my-website/docs/proxy/ui_store_model_db_setting.md b/docs/my-website/docs/proxy/ui_store_model_db_setting.md index 5f860137d0f..4b0bc690f9a 100644 --- a/docs/my-website/docs/proxy/ui_store_model_db_setting.md +++ b/docs/my-website/docs/proxy/ui_store_model_db_setting.md @@ -87,6 +87,6 @@ Change the setting from the UI and have it take effect immediately—perfect for ## Related Documentation -- [Admin UI Overview](./ui_overview.md) – General guide to the LiteLLM Admin UI -- [Models and Endpoints](./models_and_endpoints.md) – Managing models and API endpoints +- [Admin UI Overview](./ui.md) – General guide to the LiteLLM Admin UI +- [Models and Endpoints](./model_management.md) – Managing models and API endpoints - [Config Settings](./config_settings.md) – `store_model_in_db` in `general_settings` diff --git a/docs/my-website/docs/proxy/user_onboarding.md b/docs/my-website/docs/proxy/user_onboarding.md index baa241d6cdf..ecbdc11db43 100644 --- a/docs/my-website/docs/proxy/user_onboarding.md +++ b/docs/my-website/docs/proxy/user_onboarding.md @@ -79,4 +79,4 @@ curl -X POST http://localhost:4000/v1/chat/completions \ ## See Also - [Proxy Quick Start](./quick_start.md) - [User Management](./users.md) -- [Key Management](./key_management.md) +- [Key Management](./virtual_keys.md) diff --git a/docs/my-website/docs/proxy/users.md b/docs/my-website/docs/proxy/users.md index 58813eaf49e..88a7a0f1e07 100644 --- a/docs/my-website/docs/proxy/users.md +++ b/docs/my-website/docs/proxy/users.md @@ -641,7 +641,7 @@ You can set: - tpm limits (tokens per minute) - rpm limits (requests per minute) - max parallel requests -- rpm / tpm limits per model for a given key +- rpm / tpm limits per model for a given key or team ### TPM Rate Limit Type (Input/Output/Total) @@ -689,6 +689,62 @@ curl --location 'http://0.0.0.0:4000/team/new' \ } ``` + + + +**Set rate limits per model for a team** + +Use `model_rpm_limit` and `model_tpm_limit` to set rate limits per model for all keys belonging to a team. These limits apply across all keys in the team and are inherited by keys unless overridden at the key level. + +Use `/team/new` or `/team/update` with `model_rpm_limit` and `model_tpm_limit` as dictionaries mapping model names to their limits: + +```shell +curl --location 'http://0.0.0.0:4000/team/new' \ +--header 'Authorization: Bearer sk-1234' \ +--header 'Content-Type: application/json' \ +--data '{ + "team_id": "my-prod-team", + "model_rpm_limit": {"gpt-4": 100, "gpt-3.5-turbo": 200}, + "model_tpm_limit": {"gpt-4": 10000, "gpt-3.5-turbo": 20000} +}' +``` + +**Update existing team with per-model limits:** + +```shell +curl --location 'http://0.0.0.0:4000/team/update' \ +--header 'Authorization: Bearer sk-1234' \ +--header 'Content-Type: application/json' \ +--data '{ + "team_id": "my-prod-team", + "model_rpm_limit": {"gpt-4": 100, "gpt-3.5-turbo": 200}, + "model_tpm_limit": {"gpt-4": 10000, "gpt-3.5-turbo": 20000} +}' +``` + +**Alternative: Use metadata** + +You can also pass per-model limits via the `metadata` field: + +```shell +curl --location 'http://0.0.0.0:4000/team/update' \ +--header 'Authorization: Bearer sk-1234' \ +--header 'Content-Type: application/json' \ +--data '{ + "team_id": "my-prod-team", + "metadata": { + "model_rpm_limit": {"gpt-4": 100, "gpt-3.5-turbo": 200}, + "model_tpm_limit": {"gpt-4": 10000, "gpt-3.5-turbo": 20000} + } +}' +``` + +**Resolution order:** When a key belongs to a team, rate limits are resolved as: **Key metadata > Key model_max_budget > Team metadata**. Keys can override team-level per-model limits with their own `model_rpm_limit` or `model_tpm_limit`. + +**Verify:** Make a `/chat/completions` request and check response headers `x-litellm-key-remaining-requests-{model}` and `x-litellm-key-remaining-tokens-{model}` for the model-specific limits. + +[**See Swagger**](https://litellm-api.up.railway.app/#/team%20management/new_team_team_new_post) + diff --git a/docs/my-website/docs/reasoning_content.md b/docs/my-website/docs/reasoning_content.md index 5dd40122c71..8bf59f66a33 100644 --- a/docs/my-website/docs/reasoning_content.md +++ b/docs/my-website/docs/reasoning_content.md @@ -594,7 +594,9 @@ Expected Response :::tip gpt-5.4: reasoning_effort + function tools -OpenAI does not support `reasoning_effort` with function tools for `gpt-5.4` in `/v1/chat/completions`. Use `openai/responses/gpt-5.4` to route through the Responses API instead. See [Responses API Bridge](/docs/providers/openai#openai-chat-completion-to-responses-api-bridge) for details. +LiteLLM drops `reasoning_effort` from `gpt-5.4` requests to `litellm.completion()` that include tools, since that combination is supported in the Responses API. + +If you need reasoning **and** tools together, use `openai/responses/gpt-5.4` to route through the Responses API instead. See [Responses API Bridge](/docs/providers/openai#openai-chat-completion-to-responses-api-bridge) for details. ::: diff --git a/docs/my-website/docs/tutorials/claude_code_skills.md b/docs/my-website/docs/tutorials/claude_code_skills.md new file mode 100644 index 00000000000..0c6344f9561 --- /dev/null +++ b/docs/my-website/docs/tutorials/claude_code_skills.md @@ -0,0 +1,99 @@ +# LiteLLM Skills + +[litellm-skills](https://github.com/BerriAI/litellm-skills) is a collection of [Agent Skills](https://agentskills.io) for managing a live LiteLLM proxy. Install them once and any agent that supports the Agent Skills standard (Claude Code, OpenCode, OpenClaw, etc.) can create users, teams, keys, models, MCP servers, agents, and query usage — all by running `curl` commands against your proxy. + +## Install + +```bash +curl -fsSL https://raw.githubusercontent.com/BerriAI/litellm-skills/main/install.sh | sh +``` + +## Requirements + +- `curl` installed +- A running LiteLLM proxy (local or remote) +- A proxy admin key — not a virtual key scoped to `llm_api_routes` + +## Available Skills + +### Users + +| Skill | What it does | +|-------|-------------| +| `/add-user` | Create a user — email, role, budget, model access | +| `/update-user` | Update budget, role, or models for an existing user | +| `/delete-user` | Delete one or more users | + +### Teams + +| Skill | What it does | +|-------|-------------| +| `/add-team` | Create a team with budget and model limits | +| `/update-team` | Update budget, models, or rate limits | +| `/delete-team` | Delete one or more teams | + +### API Keys + +| Skill | What it does | +|-------|-------------| +| `/add-key` | Generate a key scoped to a user, team, budget, and expiry | +| `/update-key` | Update budget, models, or expiry | +| `/delete-key` | Delete by key value or alias | + +### Organizations + +| Skill | What it does | +|-------|-------------| +| `/add-org` | Create an org with budget and model access | +| `/delete-org` | Delete one or more orgs | + +### Models + +| Skill | What it does | +|-------|-------------| +| `/add-model` | Add any provider (OpenAI, Azure, Anthropic, Bedrock, Ollama…) and test it | +| `/update-model` | Rotate credentials or swap the underlying deployment | +| `/delete-model` | Remove a model | + +### MCP Servers + +| Skill | What it does | +|-------|-------------| +| `/add-mcp` | Register an MCP server (SSE, HTTP, or stdio) | +| `/update-mcp` | Update URL, credentials, or allowed tools | +| `/delete-mcp` | Remove an MCP server | + +### Agents + +| Skill | What it does | +|-------|-------------| +| `/add-agent` | Create an agent backed by a model and optional MCP servers | +| `/update-agent` | Swap the model or update description and limits | +| `/delete-agent` | Remove an agent | + +### Usage + +| Skill | What it does | +|-------|-------------| +| `/view-usage` | Daily spend and token activity — by user, team, org, or model | + +## How it works + +When you invoke a skill, the agent asks for your `LITELLM_BASE_URL` and admin key, collects the fields needed for that operation, runs the `curl`, and shows the result. For example: + +``` +/add-model +``` +→ Agent asks: provider, public name, credentials. Adds the model, runs a test completion, reports pass/fail. + +``` +/view-usage +``` +→ Agent asks: date range (defaults to current month), optional team/model filter. Prints a table of daily requests, tokens, and spend. + +## Related + +- [litellm-skills on GitHub](https://github.com/BerriAI/litellm-skills) +- [Virtual Keys](../proxy/virtual_keys.md) — managing API keys on the proxy +- [Team-based routing](../proxy/team_based_routing.md) — setting up teams +- [Model Management](../proxy/model_management.md) — adding models via config or API diff --git a/docs/my-website/docs/tutorials/index.md b/docs/my-website/docs/tutorials/index.md new file mode 100644 index 00000000000..7f80cc760ee --- /dev/null +++ b/docs/my-website/docs/tutorials/index.md @@ -0,0 +1,98 @@ +--- +title: Tutorials +sidebar_label: Overview +--- + +import NavigationCards from '@site/src/components/NavigationCards'; + +**Tutorials** are step-by-step walkthroughs for integrating LiteLLM with external tools, frameworks, and services — or building complete end-to-end workflows. + +> Need help choosing the right path before you start? See [Learn →](/docs/learn) + +--- + +## Getting Started + + + +--- + +## Integrations + + + +--- + +## Proxy + + + +--- + +## Observability & Evaluation + + diff --git a/docs/my-website/docs/tutorials/installation.md b/docs/my-website/docs/tutorials/installation.md index ecaed0bec9d..cf39c55bee6 100644 --- a/docs/my-website/docs/tutorials/installation.md +++ b/docs/my-website/docs/tutorials/installation.md @@ -1,7 +1,3 @@ ---- -displayed_sidebar: tutorialSidebar ---- - # Set up environment Let's get the necessary keys to set up our demo environment. @@ -11,7 +7,5 @@ Every LLM provider needs API keys (e.g. `OPENAI_API_KEY`). You can get API keys Let's get them for our demo! **OpenAI**: https://platform.openai.com/account/api-keys -**Cohere**: https://dashboard.cohere.com/welcome/login?redirect_uri=%2Fapi-keys (no credit card required) +**Cohere**: https://dashboard.cohere.com/welcome/login?redirect_uri=%2Fapi-keys (no credit card required) **AI21**: https://studio.ai21.com/account/api-key (no credit card required) - - diff --git a/docs/my-website/docs/vertex_batch_passthrough.md b/docs/my-website/docs/vertex_batch_passthrough.md index 3203d7d792a..17ffc1e6dbc 100644 --- a/docs/my-website/docs/vertex_batch_passthrough.md +++ b/docs/my-website/docs/vertex_batch_passthrough.md @@ -155,6 +155,6 @@ Common error scenarios and their solutions: ## Related Documentation -- [Vertex AI Provider Documentation](./vertex.md) -- [General Batches API Documentation](../batches.md) -- [Cost Tracking and Monitoring](../observability/telemetry.md) +- [Vertex AI Provider Documentation](./providers/vertex.md) +- [General Batches API Documentation](./batches.md) +- [Cost Tracking and Monitoring](./observability/telemetry.md) diff --git a/docs/my-website/docs/videos.md b/docs/my-website/docs/videos.md index 0c284aa3c42..846e551435a 100644 --- a/docs/my-website/docs/videos.md +++ b/docs/my-website/docs/videos.md @@ -290,6 +290,82 @@ curl --location 'http://localhost:4000/v1/videos' \ --header 'custom-llm-provider: azure' ``` +### Character, Edit, and Extension Endpoints + +LiteLLM proxy also supports these OpenAI-compatible video routes: + +- `POST /v1/videos/characters` +- `GET /v1/videos/characters/{character_id}` +- `POST /v1/videos/edits` +- `POST /v1/videos/extensions` + +#### Routing Behavior (`target_model_names`, encoded IDs, and provider overrides) + +- `POST /v1/videos/characters` supports `target_model_names` like `POST /v1/videos`. +- When `target_model_names` is provided on character creation, LiteLLM encodes the returned `character_id` with routing metadata. +- `GET /v1/videos/characters/{character_id}` accepts encoded character IDs directly. LiteLLM decodes the ID internally and routes with the correct model/provider metadata. +- `POST /v1/videos/edits` and `POST /v1/videos/extensions` support both: + - plain `video.id` + - encoded `video.id` values returned by LiteLLM +- `custom_llm_provider` can be supplied using the same patterns as other proxy endpoints: + - header: `custom-llm-provider` + - query: `?custom_llm_provider=...` + - body: `custom_llm_provider` (or `extra_body.custom_llm_provider` where applicable) + +#### Character create with `target_model_names` + +```bash +curl --location 'http://localhost:4000/v1/videos/characters' \ +--header 'Authorization: Bearer sk-1234' \ +-F 'name=hero' \ +-F 'target_model_names=gpt-4' \ +-F 'video=@/path/to/character.mp4' +``` + +Example response (encoded `id`): + +```json +{ + "id": "character_...", + "object": "character", + "created_at": 1712697600, + "name": "hero" +} +``` + +#### Get character using encoded `character_id` + +```bash +curl --location 'http://localhost:4000/v1/videos/characters/character_...' \ +--header 'Authorization: Bearer sk-1234' +``` + +#### Video edit with encoded `video.id` + +```bash +curl --location 'http://localhost:4000/v1/videos/edits' \ +--header 'Authorization: Bearer sk-1234' \ +--header 'Content-Type: application/json' \ +--data '{ + "prompt": "Make this brighter", + "video": { "id": "video_..." } +}' +``` + +#### Video extension with provider override from `extra_body` + +```bash +curl --location 'http://localhost:4000/v1/videos/extensions' \ +--header 'Authorization: Bearer sk-1234' \ +--header 'Content-Type: application/json' \ +--data '{ + "prompt": "Continue this scene", + "seconds": "4", + "video": { "id": "video_..." }, + "extra_body": { "custom_llm_provider": "openai" } +}' +``` + Test Azure video generation request ```bash diff --git a/docs/my-website/docusaurus.config.js b/docs/my-website/docusaurus.config.js index 32d5d800b71..9e1fae6a34f 100644 --- a/docs/my-website/docusaurus.config.js +++ b/docs/my-website/docusaurus.config.js @@ -2,9 +2,9 @@ // Note: type annotations allow type checking and IDEs autocompletion // @ts-ignore -const lightCodeTheme = require('prism-react-renderer/themes/github'); +const lightCodeTheme = require('prism-react-renderer/themes/vsLight'); // @ts-ignore -const darkCodeTheme = require('prism-react-renderer/themes/dracula'); +const darkCodeTheme = require('prism-react-renderer/themes/nightOwl'); const inkeepConfig = { baseSettings: { @@ -87,18 +87,88 @@ const config = { }, ], [ - '@docusaurus/plugin-content-blog', + '@docusaurus/plugin-content-docs', { - id: 'release_notes', + id: 'release-notes', path: './release_notes', routeBasePath: 'release_notes', - blogTitle: 'Release Notes', - blogSidebarTitle: 'Releases', - blogSidebarCount: 'ALL', - postsPerPage: 'ALL', - showReadingTime: false, - sortPosts: 'descending', - include: ['**/*.{md,mdx}'], + sidebarPath: require.resolve('./sidebars-release-notes.js'), + async sidebarItemsGenerator({defaultSidebarItemsGenerator, docs, ...args}) { + const items = await defaultSidebarItemsGenerator({docs, ...args}); + + // Build map of doc id -> year from frontmatter date + const docYearMap = {}; + for (const doc of docs) { + const date = doc.frontMatter && doc.frontMatter.date; + if (date) { + const year = new Date(date).getFullYear(); + docYearMap[doc.id] = year; + } + } + + function parseVersion(str) { + const match = (str || '').match(/v?(\d+)\.(\d+)\.(\d+)/); + if (!match) return [0, 0, 0]; + return [parseInt(match[1]), parseInt(match[2]), parseInt(match[3])]; + } + function compareVersionsDesc(a, b) { + const [aMaj, aMin, aPatch] = parseVersion(a.label || a.id || ''); + const [bMaj, bMin, bPatch] = parseVersion(b.label || b.id || ''); + if (bMaj !== aMaj) return bMaj - aMaj; + if (bMin !== aMin) return bMin - aMin; + return bPatch - aPatch; + } + + // Flatten and transform doc items (filter index, shorten labels) + function flattenDocs(list) { + const result = []; + for (const item of list) { + if (item.type === 'doc' && item.id === 'index') continue; + if (item.type === 'doc') { + const label = item.id.replace(/\/index$/, ''); + result.push({...item, label}); + } else if (item.type === 'category') { + if (item.link && item.link.type === 'doc' && item.link.id !== 'index') { + const id = item.link.id; + const label = id.replace(/\/index$/, ''); + result.push({type: 'doc', id, label}); + } else { + result.push(...flattenDocs(item.items)); + } + } + } + return result; + } + + const docItems = flattenDocs(items); + + // Group by year + const byYear = {}; + for (const item of docItems) { + const year = docYearMap[item.id] || 'Other'; + if (!byYear[year]) byYear[year] = []; + byYear[year].push(item); + } + + // Sort each year's items by version descending + for (const year of Object.keys(byYear)) { + byYear[year].sort(compareVersionsDesc); + } + + // Build categories sorted by year descending + const years = Object.keys(byYear).sort((a, b) => { + // Object.keys() returns strings; avoid numeric subtraction type errors. + const na = Number.parseInt(a, 10); + const nb = Number.parseInt(b, 10); + return nb - na; + }); + return years.map(year => ({ + type: 'category', + label: String(year), + collapsed: year !== String(years[0]), + items: byYear[year], + })); + }, }, ], [ @@ -130,6 +200,20 @@ const config = { }; }, }), + // Ensure gtag exists before the GA script loads. + () => ({ + name: 'gtag-shim', + injectHtmlTags() { + return { + headTags: [ + { + tagName: 'script', + innerHTML: `window.dataLayer=window.dataLayer||[];function gtag(){dataLayer.push(arguments);}if(!window.gtag){window.gtag=gtag;}`, + }, + ], + }; + }, + }), ], presets: [ @@ -137,10 +221,13 @@ const config = { 'classic', /** @type {import('@docusaurus/preset-classic').Options} */ ({ - gtag: { - trackingID: 'G-K7K215ZVNC', - anonymizeIP: true, - }, + gtag: + process.env.NODE_ENV === 'production' + ? { + trackingID: 'G-K7K215ZVNC', + anonymizeIP: true, + } + : undefined, docs: { sidebarPath: require.resolve('./sidebars.js'), }, @@ -181,34 +268,39 @@ const config = { label: 'Docs', }, { + type: 'docSidebar', + sidebarId: 'learnSidebar', + position: 'left', + label: 'Learn', + }, + { + type: 'docSidebar', sidebarId: 'integrationsSidebar', position: 'left', label: 'Integrations', - to: "docs/integrations" }, { - sidebarId: 'tutorialSidebar', position: 'left', label: 'Enterprise', to: "docs/enterprise" }, - { to: '/release_notes', label: 'Release Notes', position: 'left' }, { to: '/blog', label: 'Blog', position: 'left' }, - { - href: 'https://models.litellm.ai/', - label: '💸 LLM Model Cost Map', - position: 'right', - }, { href: 'https://github.com/BerriAI/litellm', - label: 'GitHub', position: 'right', + className: 'header-github-link', + 'aria-label': 'GitHub repository', }, { href: 'https://www.litellm.ai/support', - label: 'Slack/Discord', position: 'right', - } + className: 'header-discord-link', + 'aria-label': 'Discord / Slack community', + }, + { + type: 'search', + position: 'right', + }, ], }, footer: { diff --git a/docs/my-website/img/ephemeral_token.png b/docs/my-website/img/ephemeral_token.png new file mode 100644 index 00000000000..28a05f9eb1b Binary files /dev/null and b/docs/my-website/img/ephemeral_token.png differ diff --git a/docs/my-website/img/hero.png b/docs/my-website/img/hero.png new file mode 100644 index 00000000000..9f77a28d718 Binary files /dev/null and b/docs/my-website/img/hero.png differ diff --git a/docs/my-website/img/mcp_zero_trust_gateway.png b/docs/my-website/img/mcp_zero_trust_gateway.png new file mode 100644 index 00000000000..3955cef0553 Binary files /dev/null and b/docs/my-website/img/mcp_zero_trust_gateway.png differ diff --git a/docs/my-website/img/webrtc_flow.png b/docs/my-website/img/webrtc_flow.png new file mode 100644 index 00000000000..a53ec10a7b7 Binary files /dev/null and b/docs/my-website/img/webrtc_flow.png differ diff --git a/docs/my-website/release_notes/index.md b/docs/my-website/release_notes/index.md new file mode 100644 index 00000000000..e2b7edf3222 --- /dev/null +++ b/docs/my-website/release_notes/index.md @@ -0,0 +1,52 @@ +--- +title: Release Notes +sidebar_label: Overview +slug: / +--- + +# Release Notes + +LiteLLM ships new releases regularly with new provider support, performance improvements, and enterprise features. Use the sidebar to browse all releases. + +## Latest Release + +### [v1.82.3 — Nebius AI, gpt-5.4, Gemini 3.x, FLUX Kontext, and 116 New Models](/release_notes/v1.82.3/v1-82-3) + +_March 16, 2026_ + +116 new models including Nebius AI, gpt-5.4, Gemini 3.x, and FLUX Kontext. + +--- + +## Recent Releases + +| Version | Date | Highlights | +| ----------------------------------- | ------------ | ---------------------------------------------------------- | +| [v1.82.0](/release_notes/v1.82.0/v1-82-0) | Feb 28, 2026 | Realtime Guardrails, Projects Management, and 10+ Performance Optimizations | +| [v1.81.14](/release_notes/v1.81.14/v1-81-14) | Feb 21, 2026 | New Gateway Level Guardrails & Compliance Playground | +| [v1.81.12](/release_notes/v1.81.12/v1-81-12) | Feb 14, 2026 | Guardrail Policy Templates & Action Builder | +| [v1.81.9](/release_notes/v1.81.9/v1-81-9) | Feb 7, 2026 | Control which MCP Servers are exposed on the Internet | +| [v1.81.6](/release_notes/v1.81.6/v1-81-6) | Jan 31, 2026 | Logs v2 with Tool Call Tracing | +| [v1.81.3](/release_notes/v1.81.3-stable/v1-81-3) | Jan 26, 2026 | Performance — 25% CPU Usage Reduction | +| [v1.81.0](/release_notes/v1.81.0/v1-81-0) | Jan 18, 2026 | Claude Code — Web Search Across All Providers | +| [v1.80.15](/release_notes/v1.80.15/v1-80-15) | Jan 10, 2026 | Manus API Support | +| [v1.80.8](/release_notes/v1.80.8-stable/v1-80-8) | Dec 6, 2025 | Introducing A2A Agent Gateway | +| [v1.80.5](/release_notes/v1.80.5-stable/v1-80-5) | Nov 22, 2025 | Gemini 3.0 Support | +| [v1.80.0](/release_notes/v1.80.0-stable/v1-80-0) | Nov 15, 2025 | Introducing Agent Hub: Register, Publish, and Share Agents | +| [v1.79.3](/release_notes/v1.79.3-stable/v1-79-3) | Nov 8, 2025 | Built-in Guardrails on AI Gateway | +| [v1.79.0](/release_notes/v1.79.0-stable/v1-79-0) | Oct 26, 2025 | Search APIs | +| [v1.78.5](/release_notes/v1.78.5-stable/v1-78-5) | Oct 18, 2025 | Native OCR Support | +| [v1.78.0](/release_notes/v1.78.0-stable/v1-78-0) | Oct 11, 2025 | MCP Gateway: Control Tool Access by Team, Key | +| [v1.77.7](/release_notes/v1.77.7-stable/v1-77-7) | Oct 4, 2025 | 2.9x Lower Median Latency | +| [v1.77.5](/release_notes/v1.77.5-stable/v1-77-5) | Sep 29, 2025 | MCP OAuth 2.0 Support | +| [v1.77.3](/release_notes/v1.77.3-stable/v1-77-3) | Sep 21, 2025 | Priority Based Rate Limiting | + +--- + +## Stay Updated + +- **GitHub**: Watch the [BerriAI/litellm](https://github.com/BerriAI/litellm) repository for release notifications +- **Discord**: Join our [community](https://discord.com/invite/wuPM9dRgDw) for announcements +- **Twitter**: Follow [@LiteLLM](https://twitter.com/LiteLLM) + +Use the sidebar to browse the full release history. diff --git a/docs/my-website/release_notes/v1.63.14/index.md b/docs/my-website/release_notes/v1.63.14/index.md index 1ac713fc2d5..d34b2c7b335 100644 --- a/docs/my-website/release_notes/v1.63.14/index.md +++ b/docs/my-website/release_notes/v1.63.14/index.md @@ -62,10 +62,10 @@ Here's a Demo Instance to test changes: - Infer aws region from bedrock application profile id - (`arn:aws:bedrock:us-east-1:...`) - Ollama - support calling via `/v1/completions` [Get Started](../../docs/providers/ollama#using-ollama-fim-on-v1completions) - Bedrock - support `us.deepseek.r1-v1:0` model name [Docs](../../docs/providers/bedrock#supported-aws-bedrock-models) -- OpenRouter - `OPENROUTER_API_BASE` env var support [Docs](../../docs/providers/openrouter.md) +- OpenRouter - `OPENROUTER_API_BASE` env var support [Docs](../../docs/providers/openrouter) - Azure - add audio model parameter support - [Docs](../../docs/providers/azure#azure-audio-model) - OpenAI - PDF File support [Docs](../../docs/completion/document_understanding#openai-file-message-type) -- OpenAI - o1-pro Responses API streaming support [Docs](../../docs/response_api.md#streaming) +- OpenAI - o1-pro Responses API streaming support [Docs](../../docs/response_api#streaming) - [BETA] MCP - Use MCP Tools with LiteLLM SDK [Docs](../../docs/mcp) 2. **Bug Fixes** @@ -102,7 +102,7 @@ Here's a Demo Instance to test changes: - fix logging to just log the LLM I/O [PR](https://github.com/BerriAI/litellm/pull/9353) - Dynamic API Key/Space param support [Get Started](../../docs/observability/arize_integration#pass-arize-spacekey-per-request) - StandardLoggingPayload - Log litellm_model_name in payload. Allows knowing what the model sent to API provider was [Get Started](../../docs/proxy/logging_spec#standardlogginghiddenparams) -- Prompt Management - Allow building custom prompt management integration [Get Started](../../docs/proxy/custom_prompt_management.md) +- Prompt Management - Allow building custom prompt management integration [Get Started](../../docs/proxy/custom_prompt_management) ## Performance / Reliability improvements @@ -128,4 +128,4 @@ Here's a Demo Instance to test changes: ## Complete Git Diff -[Here's the complete git diff](https://github.com/BerriAI/litellm/compare/v1.63.11-stable...v1.63.14.rc) \ No newline at end of file +[Here's the complete git diff](https://github.com/BerriAI/litellm/compare/v1.63.11-stable...v1.63.14.rc) diff --git a/docs/my-website/release_notes/v1.63.2-stable/index.md b/docs/my-website/release_notes/v1.63.2-stable/index.md index a248aa94342..18233f25c21 100644 --- a/docs/my-website/release_notes/v1.63.2-stable/index.md +++ b/docs/my-website/release_notes/v1.63.2-stable/index.md @@ -64,15 +64,15 @@ Here's a Demo Instance to test changes: 9. Bedrock - handle thinking blocks in assistant message. [Get Started](https://docs.litellm.ai/docs/providers/bedrock#usage---thinking--reasoning-content) 10. Anthropic - Return `signature` on streaming. [Get Started](https://docs.litellm.ai/docs/providers/bedrock#usage---thinking--reasoning-content) - Note: We've also migrated from `signature_delta` to `signature`. [Read more](https://docs.litellm.ai/release_notes/v1.63.0) -11. Support format param for specifying image type. [Get Started](../../docs/completion/vision.md#explicitly-specify-image-type) -12. Anthropic - `/v1/messages` endpoint - `thinking` param support. [Get Started](../../docs/anthropic_unified.md) +11. Support format param for specifying image type. [Get Started](../../docs/completion/vision#explicitly-specify-image-type) +12. Anthropic - `/v1/messages` endpoint - `thinking` param support. [Get Started](../../docs/anthropic_unified) - Note: this refactors the [BETA] unified `/v1/messages` endpoint, to just work for the Anthropic API. 13. Vertex AI - handle $id in response schema when calling vertex ai. [Get Started](https://docs.litellm.ai/docs/providers/vertex#json-schema) ## Spend Tracking Improvements 1. Batches API - Fix cost calculation to run on retrieve_batch. [Get Started](https://docs.litellm.ai/docs/batches) -2. Batches API - Log batch models in spend logs / standard logging payload. [Get Started](../../docs/proxy/logging_spec.md#standardlogginghiddenparams) +2. Batches API - Log batch models in spend logs / standard logging payload. [Get Started](../../docs/proxy/logging_spec#standardlogginghiddenparams) ## Management Endpoints / UI @@ -109,4 +109,4 @@ Here's a Demo Instance to test changes: ## Complete Git Diff -[Here's the complete git diff](https://github.com/BerriAI/litellm/compare/v1.61.20-stable...v1.63.2-stable) \ No newline at end of file +[Here's the complete git diff](https://github.com/BerriAI/litellm/compare/v1.61.20-stable...v1.63.2-stable) diff --git a/docs/my-website/release_notes/v1.80.15/index.md b/docs/my-website/release_notes/v1.80.15/index.md index 4037a0d9b5d..15b49822965 100644 --- a/docs/my-website/release_notes/v1.80.15/index.md +++ b/docs/my-website/release_notes/v1.80.15/index.md @@ -53,7 +53,7 @@ pip install litellm==1.80.15 - **MCP Global Mode** - [Configure MCP servers globally with visibility controls](../../docs/mcp) - **Interactions API Bridge** - [Use all LiteLLM providers with the Interactions API](../../docs/interactions) - **RAG Query Endpoint** - [New RAG Search/Query endpoint for retrieval-augmented generation](../../docs/search/index) -- **UI Usage - Endpoint Activity** - [Users can now see Endpoint Activity Metrics in the UI](../../docs/proxy/endpoint_activity.md) +- **UI Usage - Endpoint Activity** - [Users can now see Endpoint Activity Metrics in the UI](../../docs/proxy/endpoint_activity) - **50% Overhead Reduction** - LiteLLM now sends 2.5× more requests to LLM providers @@ -640,4 +640,3 @@ Users can now see Endpoint Activity Metrics in the UI. **[View complete changelog on GitHub](https://github.com/BerriAI/litellm/compare/v1.80.11.rc.1...v1.80.15-stable.1)** - diff --git a/docs/my-website/release_notes/v1.81.0/index.md b/docs/my-website/release_notes/v1.81.0/index.md index e61d7d2d593..5953c572a7a 100644 --- a/docs/my-website/release_notes/v1.81.0/index.md +++ b/docs/my-website/release_notes/v1.81.0/index.md @@ -48,7 +48,7 @@ pip install litellm==1.81.0 - **Claude Code** - Support for using web search across Bedrock, Vertex AI, and all LiteLLM providers - **Major Change** - [50MB limit on image URL downloads](#major-change---chatcompletions-image-url-download-size-limit) to improve reliability - **Performance** - [25% CPU Usage Reduction](#performance---25-cpu-usage-reduction) by removing premature model.dump() calls from the hot path -- **Deleted Keys Audit Table on UI** - [View deleted keys and teams for audit purposes](../../docs/proxy/deleted_keys_teams.md) with spend and budget information at the time of deletion +- **Deleted Keys Audit Table on UI** - [View deleted keys and teams for audit purposes](../../docs/proxy/deleted_keys_teams) with spend and budget information at the time of deletion --- @@ -166,7 +166,7 @@ LiteLLM now reduces CPU usage by removing premature `model.dump()` calls from th -LiteLLM now provides a comprehensive audit table for deleted API keys and teams directly in the UI. This feature allows you to easily track the spend of deleted keys, view their associated team information, and maintain accurate financial records for auditing and compliance purposes. The table displays key details including key aliases, team associations, and spend information captured at the time of deletion. For more information on how to use this feature, see the [Deleted Keys & Teams documentation](../../docs/proxy/deleted_keys_teams.md). +LiteLLM now provides a comprehensive audit table for deleted API keys and teams directly in the UI. This feature allows you to easily track the spend of deleted keys, view their associated team information, and maintain accurate financial records for auditing and compliance purposes. The table displays key details including key aliases, team associations, and spend information captured at the time of deletion. For more information on how to use this feature, see the [Deleted Keys & Teams documentation](../../docs/proxy/deleted_keys_teams). --- diff --git a/docs/my-website/release_notes/v1.81.12.md b/docs/my-website/release_notes/v1.81.12/index.md similarity index 99% rename from docs/my-website/release_notes/v1.81.12.md rename to docs/my-website/release_notes/v1.81.12/index.md index 0b7c1e146ab..1bbea5b82bc 100644 --- a/docs/my-website/release_notes/v1.81.12.md +++ b/docs/my-website/release_notes/v1.81.12/index.md @@ -62,13 +62,13 @@ This release fixes out-of-memory (OOM) risks from unbounded `asyncio.Queue()` us This release adds a visual action builder for guardrail policies with conditional execution support. You can now chain guardrails into multi-step pipelines — if a simple guardrail fails, route to an advanced one instead of immediately blocking. Each step has configurable ON PASS and ON FAIL actions (Next Step, Block, or Allow), and you can test the full pipeline with a sample message before saving. -![Guardrail Action Builder](../img/release_notes/guard_actions.png) +![Guardrail Action Builder](../../img/release_notes/guard_actions.png) ### Access Groups Access Groups simplify defining resource access across your organization. One group can grant access to models, MCP servers, and agents—simply attach it to a key or team. Create groups in the Admin UI, define which resources each group includes, then assign the group when creating keys or teams. Updates to a group apply automatically to all attached keys and teams. - + ## New Providers and Endpoints diff --git a/docs/my-website/release_notes/v1.81.14.md b/docs/my-website/release_notes/v1.81.14/index.md similarity index 99% rename from docs/my-website/release_notes/v1.81.14.md rename to docs/my-website/release_notes/v1.81.14/index.md index c342bc47ee9..92c22bc0ea3 100644 --- a/docs/my-website/release_notes/v1.81.14.md +++ b/docs/my-website/release_notes/v1.81.14/index.md @@ -56,7 +56,7 @@ pip install litellm==1.81.14 AI Platform Admins can now browse built-in and partner guardrails from the Guardrail Garden. Guardrails are organized by use case — blocking financial advice, filtering insults, detecting competitor mentions, and more — so you can find the right one and deploy it in a few clicks. -![Guardrail Garden](../img/release_notes/guardrail_garden.png) +![Guardrail Garden](../../img/release_notes/guardrail_garden.png) ### 3 New Built-in Guardrails @@ -72,7 +72,7 @@ These guardrails are built for production and on our benchmarks had a 100% Recal Previously, the `store_model_in_db` setting could only be configured in `proxy_config.yaml` under `general_settings`, requiring a proxy restart to take effect. Now you can enable or disable this setting directly from the Admin UI without any restarts. This is especially useful for cloud deployments where you don't have direct access to config files or want to avoid downtime. Enable `store_model_in_db` to move model definitions from your YAML into the database—reducing config complexity, improving scalability, and enabling dynamic model management across multiple proxy instances. -![Store model in DB Setting](../img/ui_store_model_in_db.png) +![Store model in DB Setting](../../img/ui_store_model_in_db.png) #### Eval results @@ -91,14 +91,14 @@ We benchmarked our new built-in guardrails against labeled datasets before shipp The Compliance Playground lets you test any guardrail against our pre-built eval datasets or your own custom datasets, so you can see precision, recall, and false positive rate before rolling it out to production. -![Compliance Playground](../img/release_notes/compliance_playground.png) +![Compliance Playground](../../img/release_notes/compliance_playground.png) --- ## Performance & Reliability — Up to 13% Lower Latency - + This release cuts latency across all percentiles through 20+ micro-optimizations across logging, cost calculation, routing, and connection management. See [benchmarking](../../docs/benchmarks) for more info about how to benchmark yourself. diff --git a/docs/my-website/release_notes/v1.81.6.md b/docs/my-website/release_notes/v1.81.6/index.md similarity index 100% rename from docs/my-website/release_notes/v1.81.6.md rename to docs/my-website/release_notes/v1.81.6/index.md diff --git a/docs/my-website/release_notes/v1.81.9.md b/docs/my-website/release_notes/v1.81.9/index.md similarity index 99% rename from docs/my-website/release_notes/v1.81.9.md rename to docs/my-website/release_notes/v1.81.9/index.md index 80be4179b46..d11b52e892c 100644 --- a/docs/my-website/release_notes/v1.81.9.md +++ b/docs/my-website/release_notes/v1.81.9/index.md @@ -81,7 +81,7 @@ This release makes it safe to expose MCP servers on the public internet by addin [Get started](../../docs/mcp_public_internet) @@ -92,7 +92,7 @@ Set a soft budget on any team to receive email alerts when spending crosses the [Get started](../../docs/proxy/ui_team_soft_budget_alerts) diff --git a/docs/my-website/release_notes/v1.82.0.md b/docs/my-website/release_notes/v1.82.0/index.md similarity index 100% rename from docs/my-website/release_notes/v1.82.0.md rename to docs/my-website/release_notes/v1.82.0/index.md diff --git a/docs/my-website/release_notes/v1.82.3/index.md b/docs/my-website/release_notes/v1.82.3/index.md new file mode 100644 index 00000000000..20be8826718 --- /dev/null +++ b/docs/my-website/release_notes/v1.82.3/index.md @@ -0,0 +1,530 @@ +--- +title: "v1.82.3 - Nebius AI, gpt-5.4, Gemini 3.x, FLUX Kontext, and 116 New Models" +slug: "v1-82-3" +date: 2026-03-16T00:00:00 +authors: + - name: Krrish Dholakia + title: CEO, LiteLLM + url: https://www.linkedin.com/in/krish-d/ + image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg + - name: Ishaan Jaff + title: CTO, LiteLLM + url: https://www.linkedin.com/in/reffajnaahsi/ + image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg +hide_table_of_contents: false +--- + +## Deploy this version + +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + + + + +``` showLineNumbers title="docker run litellm" +docker run \ +-e STORE_MODEL_IN_DB=True \ +-p 4000:4000 \ +ghcr.io/berriai/litellm:main-1.82.3-stable +``` + + + + +``` showLineNumbers title="pip install litellm" +pip install litellm==1.82.3 +``` + + + + +## Key Highlights + +- **Nebius AI — new provider** — [30 models across DeepSeek, Qwen, Llama, Mistral, NVIDIA, and BAAI available via Nebius AI cloud](../../docs/providers/nebius) - [PR #22614](https://github.com/BerriAI/litellm/pull/22614) +- **OpenAI gpt-5.4 / gpt-5.4-pro — day 0** — Full pricing and routing support for `gpt-5.4` (1M context, $2.50/$15.00) and `gpt-5.4-pro` ($30.00/$180.00) on OpenAI and Azure +- **Gemini 3.x models** — `gemini-3-flash-preview`, `gemini-3.1-pro-preview`, `gemini-3.1-flash-image-preview`, and `gemini-embedding-2-preview` added to cost map for Google AI and Vertex AI +- **FLUX Kontext image editing** — `flux-kontext-pro` and `flux-kontext-max` added to Black Forest Labs, alongside `flux-pro-1.0-fill` and `flux-pro-1.0-expand` for inpainting and outpainting +- **116 new models, 132 deprecated models cleaned up** — Major model map refresh including Mistral Magistral, Dashscope Qwen3 VL, xAI Grok via Azure AI, ZAI GLM-5, Serper Search; removal of OpenAI GPT-3.5/GPT-4 legacy variants, Gemini 1.5, and Vertex AI PaLM2 +- **SageMaker Nova provider** — [New `sagemaker_nova` provider for Amazon Nova models on SageMaker](../../docs/providers/aws_sagemaker) - [PR #21542](https://github.com/BerriAI/litellm/pull/21542) +- **Hashicorp Vault secret manager** — Config override backend powered by Hashicorp Vault, with full UI for managing vault-sourced credentials - [PR #22939](https://github.com/BerriAI/litellm/pull/22939), [PR #23036](https://github.com/BerriAI/litellm/pull/23036) +- **Responses API WebSocket streaming** — Real-time WebSocket streaming for the Responses API, including support across all providers - [PR #22559](https://github.com/BerriAI/litellm/pull/22559), [PR #22771](https://github.com/BerriAI/litellm/pull/22771) +- **Org Admin RBAC expansion** — Org Admins can now access team management endpoints, view and invite internal users, and manage team membership without requiring a global admin role - [PR #23085](https://github.com/BerriAI/litellm/pull/23085), [PR #23080](https://github.com/BerriAI/litellm/pull/23080) +- **Guardrail mode defaults and tag-based modes** — Set a default guardrail mode list globally, and specify a list of modes in tag-based guardrail configs - [PR #22676](https://github.com/BerriAI/litellm/pull/22676), [PR #23020](https://github.com/BerriAI/litellm/pull/23020) +- **Secret redaction in logs** — API keys, tokens, and credentials automatically scrubbed from all proxy log output. Enabled by default; opt out with `LITELLM_DISABLE_REDACT_SECRETS=true` - [PR #23668](https://github.com/BerriAI/litellm/pull/23668) +- **Streaming stability fix** — Critical fix for `RuntimeError: Cannot send a request, as the client has been closed.` crashes after ~1 hour in production - [PR #22926](https://github.com/BerriAI/litellm/pull/22926) + +--- + +## New Providers and Endpoints + +### New Providers (7 new providers) + +| Provider | Supported LiteLLM Endpoints | Description | +| -------- | --------------------------- | ----------- | +| [Nebius AI](../../docs/providers/nebius) (`nebius/`) | `/chat/completions`, `/embeddings` | EU-based AI cloud with 30+ open models — DeepSeek, Qwen3, Llama 3.1/3.3, NVIDIA Nemotron, BAAI embeddings | +| [ZAI](../../docs/providers/zai) (`zai/`) | `/chat/completions` | ZhipuAI GLM-5 models via ZAI cloud | +| [Black Forest Labs](../../docs/providers/black_forest_labs) (`black_forest_labs/`) | `/images/generations`, `/images/edits` | FLUX image generation and editing — Kontext Pro/Max, Pro 1.0 Fill/Expand | +| [Serper](../../docs/providers/serper) (`serper/`) | `/search` | Web search via Serper API | +| [SageMaker Nova](../../docs/providers/aws_sagemaker) (`sagemaker_nova/`) | `/chat/completions` | Amazon Nova models via SageMaker endpoint | +| [Google Search API](../../docs/providers/google_search) (`google_search/`) | `/search` | Google Search API integration - [PR #22752](https://github.com/BerriAI/litellm/pull/22752) | +| [Bedrock Mantle](../../docs/providers/bedrock) (`bedrock_mantle/`) | `/chat/completions` | Amazon Bedrock via Mantle — alternative auth and routing path for Bedrock models - [PR #22866](https://github.com/BerriAI/litellm/pull/22866) | + +--- + +## New Models / Updated Models + +#### New Model Support (116 new models) + +| Provider | Model | Context Window | Input ($/1M tokens) | Output ($/1M tokens) | Features | +| -------- | ----- | -------------- | ------------------- | -------------------- | -------- | +| OpenAI | `gpt-5.4` | 1.05M | $2.50 | $15.00 | chat, vision, tools, reasoning | +| OpenAI | `gpt-5.4-pro` | 1.05M | $30.00 | $180.00 | responses, vision, tools, reasoning | +| OpenAI | `gpt-5.3-chat-latest` | 128K | $1.75 | $14.00 | chat, vision, tools, reasoning | +| Azure OpenAI | `azure/gpt-5.4` | 1.05M | $2.50 | $15.00 | chat, vision, tools, reasoning | +| Azure OpenAI | `azure/gpt-5.4-pro` | 1.05M | $30.00 | $180.00 | responses, vision, tools, reasoning | +| Azure OpenAI | `azure/gpt-5.3-chat` | 128K | $1.75 | $14.00 | chat, vision, tools, reasoning | +| Google Gemini | `gemini/gemini-3-flash-preview` | 1M | $0.50 | $3.00 | chat, vision, tools, reasoning | +| Google Gemini | `gemini/gemini-3.1-pro-preview` | 1M | $2.00 | $12.00 | chat, vision, tools, reasoning | +| Google Gemini | `gemini/gemini-3.1-flash-image-preview` | 65K | $0.25 | $1.50 | image generation, vision | +| Google Gemini | `gemini/gemini-3.1-flash-lite-preview` | - | - | - | chat | +| Google Gemini | `gemini/gemini-3-pro-image-preview` | - | - | - | image generation | +| Google Gemini | `gemini/gemini-embedding-2-preview` | 8K | $0.20 | - | embeddings | +| Google Vertex AI | `vertex_ai/gemini-3-flash-preview` | - | - | - | chat | +| Google Vertex AI | `vertex_ai/gemini-3.1-pro-preview` | - | - | - | chat | +| Google Vertex AI | `vertex_ai/gemini-3.1-flash-lite-preview` | - | - | - | chat | +| Google Vertex AI | `vertex_ai/gemini-embedding-2-preview` | - | $0.20 | - | embeddings | +| Mistral | `mistral/magistral-medium-1-2-2509` | 40K | $2.00 | $5.00 | chat, tools, reasoning | +| Mistral | `mistral/magistral-small-1-2-2509` | 40K | $0.50 | $1.50 | chat, tools, reasoning | +| Mistral | `mistral/mistral-large-2512` | 262K | $0.50 | $1.50 | chat, vision, tools | +| Mistral | `mistral/mistral-medium-3-1-2508` | - | - | - | chat | +| Mistral | `mistral/mistral-small-3-2-2506` | - | - | - | chat | +| Mistral | `mistral/ministral-3-3b-2512` | - | - | - | chat | +| Mistral | `mistral/ministral-3-8b-2512` | - | - | - | chat | +| Mistral | `mistral/ministral-3-14b-2512` | - | - | - | chat | +| Black Forest Labs | `black_forest_labs/flux-kontext-pro` | - | - | - | image editing | +| Black Forest Labs | `black_forest_labs/flux-kontext-max` | - | - | - | image editing | +| Black Forest Labs | `black_forest_labs/flux-pro-1.0-fill` | - | - | - | image editing (inpaint) | +| Black Forest Labs | `black_forest_labs/flux-pro-1.0-expand` | - | - | - | image editing (outpaint) | +| Black Forest Labs | `black_forest_labs/flux-pro-1.1` | - | - | - | image generation | +| Black Forest Labs | `black_forest_labs/flux-pro-1.1-ultra` | - | - | - | image generation | +| Black Forest Labs | `black_forest_labs/flux-dev` | - | - | - | image generation | +| Black Forest Labs | `black_forest_labs/flux-pro` | - | - | - | image generation | +| Azure AI | `azure_ai/grok-4-1-fast-non-reasoning` | 131K | $0.20 | $0.50 | chat, tools | +| Azure AI | `azure_ai/grok-4-1-fast-reasoning` | 131K | $0.20 | $0.50 | chat, tools, reasoning | +| Azure AI | `azure_ai/mistral-document-ai-2512` | - | - | - | OCR | +| Dashscope | `dashscope/qwen3-next-80b-a3b-instruct` | 262K | $0.15 | $1.20 | chat | +| Dashscope | `dashscope/qwen3-next-80b-a3b-thinking` | 262K | $0.15 | $1.20 | chat, reasoning | +| Dashscope | `dashscope/qwen3-vl-235b-a22b-instruct` | 131K | $0.40 | $1.60 | chat, vision | +| Dashscope | `dashscope/qwen3-vl-235b-a22b-thinking` | 131K | $0.40 | $4.00 | chat, vision, reasoning | +| Dashscope | `dashscope/qwen3-vl-32b-instruct` | 131K | $0.16 | $0.64 | chat, vision | +| Dashscope | `dashscope/qwen3-vl-32b-thinking` | 131K | $0.16 | $2.87 | chat, vision, reasoning | +| Dashscope | `dashscope/qwen3-vl-plus` | 260K | - | - | chat, vision | +| Dashscope | `dashscope/qwen3.5-plus` | 992K | - | - | chat | +| Dashscope | `dashscope/qwen3-max-2026-01-23` | 258K | - | - | chat | +| Nebius AI | `nebius/deepseek-ai/DeepSeek-R1` | 128K | $0.80 | $2.40 | chat, reasoning | +| Nebius AI | `nebius/deepseek-ai/DeepSeek-R1-0528` | 164K | $0.80 | $2.40 | chat, reasoning | +| Nebius AI | `nebius/deepseek-ai/DeepSeek-V3` | 128K | $0.50 | $1.50 | chat | +| Nebius AI | `nebius/deepseek-ai/DeepSeek-V3-0324` | 128K | $0.50 | $1.50 | chat | +| Nebius AI | `nebius/deepseek-ai/DeepSeek-R1-Distill-Llama-70B` | 128K | $0.25 | $0.75 | chat | +| Nebius AI | `nebius/Qwen/Qwen3-235B-A22B` | 262K | $0.20 | $0.60 | chat | +| Nebius AI | `nebius/Qwen/Qwen3-32B` | 32K | $0.10 | $0.30 | chat | +| Nebius AI | `nebius/Qwen/Qwen3-30B-A3B` | 32K | $0.10 | $0.30 | chat | +| Nebius AI | `nebius/Qwen/Qwen3-14B` | 32K | $0.08 | $0.24 | chat | +| Nebius AI | `nebius/Qwen/Qwen3-4B` | 32K | $0.08 | $0.24 | chat | +| Nebius AI | `nebius/Qwen/QwQ-32B` | 32K | $0.15 | $0.45 | chat | +| Nebius AI | `nebius/Qwen/Qwen2.5-72B-Instruct` | 128K | $0.13 | $0.40 | chat | +| Nebius AI | `nebius/Qwen/Qwen2.5-32B-Instruct` | 128K | $0.06 | $0.20 | chat | +| Nebius AI | `nebius/Qwen/Qwen2.5-VL-72B-Instruct` | 131K | $0.13 | $0.40 | chat, vision | +| Nebius AI | `nebius/Qwen/Qwen2-VL-72B-Instruct` | 131K | $0.13 | $0.40 | chat, vision | +| Nebius AI | `nebius/Qwen/Qwen2-VL-7B-Instruct` | 131K | $0.02 | $0.06 | chat, vision | +| Nebius AI | `nebius/meta-llama/Meta-Llama-3.1-405B-Instruct` | 128K | $1.00 | $3.00 | chat | +| Nebius AI | `nebius/meta-llama/Meta-Llama-3.1-70B-Instruct` | 128K | $0.13 | $0.40 | chat | +| Nebius AI | `nebius/meta-llama/Meta-Llama-3.1-8B-Instruct` | 128K | $0.02 | $0.06 | chat | +| Nebius AI | `nebius/meta-llama/Llama-3.3-70B-Instruct` | 128K | $0.13 | $0.40 | chat | +| Nebius AI | `nebius/meta-llama/Llama-Guard-3-8B` | 128K | $0.02 | $0.06 | chat | +| Nebius AI | `nebius/nvidia/Llama-3.1-Nemotron-Ultra-253B-v1` | 128K | $0.60 | $1.80 | chat | +| Nebius AI | `nebius/nvidia/Llama-3.3-Nemotron-Super-49B-v1` | 131K | $0.10 | $0.40 | chat | +| Nebius AI | `nebius/NousResearch/Hermes-3-Llama-3.1-405B` | 128K | $1.00 | $3.00 | chat | +| Nebius AI | `nebius/google/gemma-3-27b-it` | 128K | $0.06 | $0.20 | chat | +| Nebius AI | `nebius/mistralai/Mistral-Nemo-Instruct-2407` | 128K | $0.04 | $0.12 | chat | +| Nebius AI | `nebius/Qwen/Qwen2.5-Coder-7B` | 32K | $0.01 | $0.03 | chat | +| Nebius AI | `nebius/BAAI/bge-en-icl` | 32K | $0.01 | - | embeddings | +| Nebius AI | `nebius/BAAI/bge-multilingual-gemma2` | 8K | $0.01 | - | embeddings | +| Nebius AI | `nebius/intfloat/e5-mistral-7b-instruct` | 32K | $0.01 | - | embeddings | +| AWS Bedrock | `mistral.devstral-2-123b` | 256K | $0.40 | $2.00 | chat, tools | +| AWS Bedrock | `zai.glm-4.7-flash` | 200K | $0.07 | $0.40 | chat, tools, reasoning | +| ZAI | `zai/glm-5` | 200K | $1.00 | $3.20 | chat, tools, reasoning | +| ZAI | `zai/glm-5-code` | 200K | $1.20 | $5.00 | chat, tools, reasoning | +| OpenRouter | `openrouter/anthropic/claude-sonnet-4.6` | - | - | - | chat | +| OpenRouter | `openrouter/google/gemini-3.1-pro-preview` | - | - | - | chat | +| OpenRouter | `openrouter/openai/gpt-5.1-codex-max` | - | - | - | chat | +| OpenRouter | `openrouter/qwen/qwen3-coder-plus` | - | - | - | chat | +| OpenRouter | `openrouter/qwen/qwen3.5-*` (5 models) | - | - | - | chat | +| OpenRouter | `openrouter/z-ai/glm-5` | - | - | - | chat | +| Together AI | `together_ai/Qwen/Qwen3.5-397B-A17B` | - | - | - | chat | +| Perplexity | `perplexity/pplx-embed-v1-0.6b` | 32K | $0.00 | - | embeddings | +| Perplexity | `perplexity/pplx-embed-v1-4b` | 32K | $0.03 | - | embeddings | +| Serper | `serper/search` | - | - | - | search | + +#### Updated Models + +- **[AWS Bedrock](../../docs/providers/bedrock)** + - Add `cache_read_input_token_cost` and `cache_creation_input_token_cost` to Bedrock-hosted Anthropic models (`claude-3-opus`, `claude-3-sonnet`, `claude-3-haiku`, and APAC/EU variants) — prompt caching is now tracked for cost estimation + - Rename `apac.anthropic.claude-sonnet-4-6` → `au.anthropic.claude-sonnet-4-6` to reflect correct regional identifier + +- **[Azure OpenAI](../../docs/providers/azure)** + - Add `supports_none_reasoning_effort` to all `gpt-5.1-chat`, `gpt-5.1-codex`, and `gpt-5.4` variants (global, EU, standard deployments) — allows passing `reasoning_effort: null` to disable reasoning + +- **[Azure OpenAI](../../docs/providers/azure)** — Removed deprecated models + - Remove `azure/gpt-35-turbo-0301` (deprecated 2025-02-13) + - Remove `azure/gpt-35-turbo-0613` (deprecated 2025-02-13) + +#### Features + +- **[OpenAI](../../docs/providers/openai)** + - Day 0 support for `gpt-5.4` and `gpt-5.4-pro` on OpenAI and Azure + +- **[Google Gemini](../../docs/providers/gemini)** + - Add Gemini 3.x model cost map entries — `gemini-3-flash-preview`, `gemini-3.1-pro-preview`, `gemini-3.1-flash-lite-preview`, `gemini-3-pro-image-preview`, `gemini-embedding-2-preview` + - Add Gemini 2.0 Flash and Flash Lite to cost map (re-added with updated pricing) + +- **[Google Vertex AI](../../docs/providers/vertex)** + - Add `gemini-3-flash-preview`, `gemini-3.1-flash-lite-preview`, `gemini-flash-experimental`, and `gemini-embedding-2-preview` to Vertex AI model cost map + +- **[Mistral](../../docs/providers/mistral)** + - Add Magistral reasoning models (`magistral-medium-1-2-2509`, `magistral-small-1-2-2509`) + - Add `mistral-large-2512`, `mistral-medium-3-1-2508`, `mistral-small-3-2-2506`, `ministral-3-*` variants + +- **[Dashscope / Qwen](../../docs/providers/dashscope)** + - Add Qwen3 VL multimodal models (`qwen3-vl-235b`, `qwen3-vl-32b` — instruct and thinking variants) + - Add `qwen3-next-80b-a3b` (instruct + thinking), `qwen3.5-plus`, `qwen3-max-2026-01-23` + +- **[Black Forest Labs](../../docs/providers/black_forest_labs)** + - Add FLUX Kontext image editing models (`flux-kontext-pro`, `flux-kontext-max`) + - Add FLUX Pro 1.0 Fill (inpainting) and Expand (outpainting) + - Add `flux-pro-1.1`, `flux-pro-1.1-ultra`, `flux-dev`, `flux-pro` + +- **[Azure AI](../../docs/providers/azure_ai)** + - Add xAI Grok models via Azure AI Foundry (`grok-4-1-fast-non-reasoning`, `grok-4-1-fast-reasoning`) + - Add Mistral Document AI (`mistral-document-ai-2512`) — OCR mode + +- **[AWS Bedrock](../../docs/providers/bedrock)** + - Add `mistral.devstral-2-123b` (256K context, tools) + - Add `zai.glm-4.7-flash` via Bedrock Converse (200K context, tools, reasoning) + +- **[SageMaker](../../docs/providers/aws_sagemaker)** + - Add `sagemaker_nova` provider for Amazon Nova models on SageMaker - [PR #21542](https://github.com/BerriAI/litellm/pull/21542) + +#### Deprecated / Removed Models + +**OpenAI** — Legacy models removed from cost map: +- `gpt-3.5-turbo-0301`, `gpt-3.5-turbo-0613`, `gpt-3.5-turbo-16k-0613` +- `gpt-4-0314`, `gpt-4-32k`, `gpt-4-32k-0314`, `gpt-4-32k-0613`, `gpt-4-1106-vision-preview`, `gpt-4-vision-preview` +- `gpt-4.5-preview`, `gpt-4.5-preview-2025-02-27` +- `gpt-4o-audio-preview-2024-10-01`, `gpt-4o-realtime-preview-2024-10-01` +- `o1-mini`, `o1-mini-2024-09-12`, `o1-preview`, `o1-preview-2024-09-12` + +**Google Gemini** — Gemini 1.5 and legacy 2.0 variants removed: +- All `gemini-1.5-*` variants (flash, flash-8b, pro, and dated versions) +- `gemini-2.0-flash-exp`, `gemini-2.0-pro-exp-02-05`, `gemini-2.5-flash-preview-04-17`, `gemini-2.5-flash-preview-05-20` + +**Google Vertex AI** — PaLM 2 / legacy models removed: +- All `chat-bison`, `text-bison`, `codechat-bison`, `code-bison`, `code-gecko` variants +- Gemini 1.0 Pro, 1.5 Flash/Pro, 2.0 Flash experimental, and preview variants + +**Perplexity** — Legacy Llama-sonar models removed: +- `llama-3.1-sonar-huge-128k-online`, `llama-3.1-sonar-large/small-128k-chat/online` + +--- + +## LLM API Endpoints + +#### Features + +- **[Responses API](../../docs/response_api)** + - Handle `response.failed`, `response.incomplete`, and `response.cancelled` terminal event types in background streaming — previously only `response.completed` was handled - [PR #23492](https://github.com/BerriAI/litellm/pull/23492) + - WebSocket streaming support for Responses API — real-time streaming via WebSocket for all providers - [PR #22559](https://github.com/BerriAI/litellm/pull/22559), [PR #22771](https://github.com/BerriAI/litellm/pull/22771) + - WebRTC support for real-time audio/video communication - [PR #23446](https://github.com/BerriAI/litellm/pull/23446) + - Responses API support for OpenAI-compatible JSON providers (`openai_like`) - [PR #21398](https://github.com/BerriAI/litellm/pull/21398) + - Route `gpt-5.4+` calls using both tools and reasoning to the Responses API automatically - [PR #23577](https://github.com/BerriAI/litellm/pull/23577) + +- **[Anthropic Files API](../../docs/providers/anthropic)** + - Full Anthropic Files API support — upload, retrieve, list, and delete files; use file references in messages - [PR #16594](https://github.com/BerriAI/litellm/pull/16594) + +- **[Mistral](../../docs/providers/mistral)** + - Voxtral audio transcription support — `mistral/voxtral-mini-*` and `mistral/voxtral-*` for audio transcription via Mistral - [PR #22801](https://github.com/BerriAI/litellm/pull/22801) + +- **[OpenAI](../../docs/providers/openai)** + - `litellm.acount_tokens()` public API — async token counting with full OpenAI provider support - [PR #22809](https://github.com/BerriAI/litellm/pull/22809) + - Normalize `reasoning_effort` dict to string for chat completion API - [PR #22981](https://github.com/BerriAI/litellm/pull/22981) + +- **[OpenRouter](../../docs/providers/openrouter)** + - Image edit support for OpenRouter models - [PR #22403](https://github.com/BerriAI/litellm/pull/22403) + +- **[Google Vertex AI](../../docs/providers/vertex)** + - VIDEO modality token usage tracking in `completion_tokens_details` - [PR #22550](https://github.com/BerriAI/litellm/pull/22550) + +- **Images API** + - `input_fidelity` parameter for image edit API - [PR #23201](https://github.com/BerriAI/litellm/pull/23201) + +- **General** + - Per-request `enable_json_schema_validation` flag for thread-safe JSON schema validation - [PR #21233](https://github.com/BerriAI/litellm/pull/21233) + - Model cost aliases expansion — define aliases in the cost map that inherit pricing from a parent model - [PR #23314](https://github.com/BerriAI/litellm/pull/23314), [PR #23457](https://github.com/BerriAI/litellm/pull/23457) + - Wildcards model support for the Files API - [PR #22740](https://github.com/BerriAI/litellm/pull/22740) + +#### Bugs + +- **[Anthropic](../../docs/providers/anthropic)** + - Preserve native tool format (web_search, bash, tool_search, etc.) when guardrails convert tools for the Anthropic Messages API - [PR #23526](https://github.com/BerriAI/litellm/pull/23526) + - Enforce `type: "object"` on tool input schemas in `_map_tool_helper` — fixes tool call failures for strict-schema providers - [PR #23103](https://github.com/BerriAI/litellm/pull/23103) + - Deduplicate `tool_result` messages by `tool_call_id` — prevents duplicate tool result errors in multi-turn conversations - [PR #23104](https://github.com/BerriAI/litellm/pull/23104) + - Map `reasoning_effort` to `output_config` for Claude 4.6 models - [PR #22220](https://github.com/BerriAI/litellm/pull/22220) + +- **[Google Gemini](../../docs/providers/gemini)** + - Correct streaming `finish_reason` for tool calls — was incorrectly returning `null` instead of `tool_calls` - [PR #21577](https://github.com/BerriAI/litellm/pull/21577) + - Preserve `$ref` in JSON Schema for Gemini 2.0+ — schema references were being stripped, breaking structured output - [PR #21597](https://github.com/BerriAI/litellm/pull/21597) + - Handle `minimal` `reasoning_effort` param for Gemini 3.1 models - [PR #22920](https://github.com/BerriAI/litellm/pull/22920) + +- **[Google Vertex AI](../../docs/providers/vertex)** + - Pass through native Gemini `imageConfig` params for image generation - [PR #21585](https://github.com/BerriAI/litellm/pull/21585) + - Prevent content truncation when `finish_reason` races ahead of content chunks in streaming - [PR #22692](https://github.com/BerriAI/litellm/pull/22692) + - Strip LiteLLM-internal keys from `extra_body` before merging to Gemini request body - [PR #23131](https://github.com/BerriAI/litellm/pull/23131) + - Drop unsupported `output_config` parameter from all Vertex AI requests - [PR #22884](https://github.com/BerriAI/litellm/pull/22884) + - Skip schema transforms for Gemini 2.0+ tool parameters — avoids breaking native Gemini schema handling - [PR #23265](https://github.com/BerriAI/litellm/pull/23265) + +- **[OpenRouter](../../docs/providers/openrouter)** + - Pattern-based fix for native model double-stripping when provider prefix matches model name - [PR #22320](https://github.com/BerriAI/litellm/pull/22320) + - Use provider-reported usage in streaming responses when `stream_options` is not set - [PR #21592](https://github.com/BerriAI/litellm/pull/21592) + +- **[AWS Bedrock](../../docs/providers/bedrock)** + - Extract region and model ID from `bedrock/{region}/{model}` path format - [PR #22546](https://github.com/BerriAI/litellm/pull/22546) + - Strip `scope` from `cache_control` for Anthropic messages on Bedrock and Azure AI - [PR #22867](https://github.com/BerriAI/litellm/pull/22867) + - Populate `completion_tokens_details` in Responses API responses - [PR #23243](https://github.com/BerriAI/litellm/pull/23243) + +- **[Azure AI](../../docs/providers/azure_ai)** + - Resolve `api_base` from environment variable in Document Intelligence OCR - [PR #21581](https://github.com/BerriAI/litellm/pull/21581) + +- **[Moonshot / Kimi](../../docs/providers/openai_compatible)** + - Auto-fill `reasoning_content` for Moonshot Kimi reasoning models - [PR #23580](https://github.com/BerriAI/litellm/pull/23580) + - Preserve `image_url` blocks in multimodal messages for Moonshot - [PR #21595](https://github.com/BerriAI/litellm/pull/21595) + +- **[HuggingFace](../../docs/providers/huggingface)** + - Forward `extra_headers` to HuggingFace embedding API - [PR #23525](https://github.com/BerriAI/litellm/pull/23525) + +- **Token Counting / Cost** + - Fix `count_tokens` to include system prompts and tools in token counting API requests - [PR #22301](https://github.com/BerriAI/litellm/pull/22301) + - Pass all custom pricing fields to `register_model` in `completion()` and `embedding()` - [PR #22552](https://github.com/BerriAI/litellm/pull/22552) + +- **Tools / Function Calling** + - Gracefully repair truncated JSON in tool call arguments — prevents crashes on malformed tool responses - [PR #22503](https://github.com/BerriAI/litellm/pull/22503) + - Fix `output_item.done` for function calls not emitting `finish_reason` in streaming - [PR #22553](https://github.com/BerriAI/litellm/pull/22553) + - Preserve thinking block order with multiple web searches - [PR #23093](https://github.com/BerriAI/litellm/pull/23093) + +- **General** + - Normalize `content_filtered` finish reason across providers - [PR #23564](https://github.com/BerriAI/litellm/pull/23564) + - Unify `finish_reason` mapping to OpenAI-compatible values across all providers - [PR #22138](https://github.com/BerriAI/litellm/pull/22138) + - Fix custom cost tracking on deployments for `/v1/messages` and `/v1/responses` - [PR #23647](https://github.com/BerriAI/litellm/pull/23647) + - Fix per-request custom pricing when `router_model_id` has no pricing data — now falls back to model name + - Fix batch list showing stale `validating` status after completion - [PR #22982](https://github.com/BerriAI/litellm/pull/22982) + - Fix batch retrieve returning raw `output_file_id` when `model_id` is missing - [PR #23194](https://github.com/BerriAI/litellm/pull/23194) + - Encode batch IDs when `x-litellm-model` header is used - [PR #22653](https://github.com/BerriAI/litellm/pull/22653) + - Map `reasoning` to `reasoning_content` in streaming Delta for gpt-oss providers - [PR #22803](https://github.com/BerriAI/litellm/pull/22803) + +--- + +## Management Endpoints / UI + +#### Features + +- **Virtual Keys** + - Add Organization dropdown to Create/Edit Key form — `organization_id` is now a first-class field in Key Ownership - [PR #23595](https://github.com/BerriAI/litellm/pull/23595) + - Allow setting `organization_id` on `/key/update` — keys can be assigned or moved to a different organization after creation - [PR #23557](https://github.com/BerriAI/litellm/pull/23557) + - Manual Spend Reset for virtual keys from the UI — admins can reset key spend to zero on demand - [PR #22715](https://github.com/BerriAI/litellm/pull/22715) + - BYOK (Bring Your Own Key) — client-side provider API key takes precedence over proxy key for Anthropic `/v1/messages` - [PR #22964](https://github.com/BerriAI/litellm/pull/22964) + - UI login session duration configurable via `LITELLM_UI_SESSION_DURATION` environment variable - [PR #22182](https://github.com/BerriAI/litellm/pull/22182) + - Auto-redirect UI login to SSO via `auto_redirect_ui_login_to_sso: true` in config.yaml - [PR #23367](https://github.com/BerriAI/litellm/pull/23367) + +- **Access Control (RBAC)** + - Org Admins can now access team management endpoints — `/team/new`, `/team/update`, `/team/delete`, `/team/member_add`, `/team/member_delete` - [PR #23085](https://github.com/BerriAI/litellm/pull/23085), [PR #23095](https://github.com/BerriAI/litellm/pull/23095) + - Org Admins can view and invite internal users — full user management without requiring global admin role - [PR #23080](https://github.com/BerriAI/litellm/pull/23080) + - Allow Admin Viewers to access Audit Logs — view-only admin role now includes audit log access - [PR #23419](https://github.com/BerriAI/litellm/pull/23419) + - RBAC for Vector Stores and Agents — key/team-level access control for vector store and agent resources - [PR #22858](https://github.com/BerriAI/litellm/pull/22858) + - User filter scope (`scope_user_search_to_org`) is now opt-in — previously default-on, causing unintended restriction - [PR #23057](https://github.com/BerriAI/litellm/pull/23057) + +- **Vector Stores** + - Vector Store management endpoints — retrieve, list, update, and delete vector stores via `/v1/vector_stores/*` - [PR #23435](https://github.com/BerriAI/litellm/pull/23435) + +- **Teams** + - Batch expiry setting for teams — configure a default expiry duration for all team keys - [PR #22705](https://github.com/BerriAI/litellm/pull/22705) + - Team Admin can reset key spend - [PR #22725](https://github.com/BerriAI/litellm/pull/22725) + +- **Internal Users** + - Add/Remove Team Membership directly from the Internal Users info page — includes searchable dropdown and role selector; no longer requires navigating to each team - [PR #23638](https://github.com/BerriAI/litellm/pull/23638) + +- **Models** + - Attach knowledge base to model via UI - [PR #22656](https://github.com/BerriAI/litellm/pull/22656) + +- **Default Team Settings** + - Modernize page to antd (consistent with rest of app) - [PR #23614](https://github.com/BerriAI/litellm/pull/23614) + - Fix: default team params (budget, duration, tpm, rpm, permissions) now correctly applied on `/team/new` - [PR #23614](https://github.com/BerriAI/litellm/pull/23614) + - Fix: settings persist across proxy restarts (`default_team_params` added to `LITELLM_SETTINGS_SAFE_DB_OVERRIDES`) - [PR #23614](https://github.com/BerriAI/litellm/pull/23614) + - Fix: resolved race condition in `_update_litellm_setting` where `get_config()` could overwrite freshly saved values - [PR #23614](https://github.com/BerriAI/litellm/pull/23614) + +- **Usage** + - Auto-paginate daily spend data — all entity views (teams, orgs, customers, tags, agents, users) fetch pages progressively with charts updating after each page - [PR #23622](https://github.com/BerriAI/litellm/pull/23622) + +- **Models / Cost** + - Azure Model Router cost breakdown in UI — show per-sub-model `additional_costs` from `hidden_params` in `CostBreakdownViewer` - [PR #23550](https://github.com/BerriAI/litellm/pull/23550) + +- **User Management** + - New `/user/info/v2` endpoint — scoped, paginated replacement for the existing god endpoint that caused memory and stability issues on large installs - [PR #23437](https://github.com/BerriAI/litellm/pull/23437) + +#### Bugs + +- Fix Tag list endpoint returning 500 due to invalid Prisma `group_by` kwargs - [PR #23606](https://github.com/BerriAI/litellm/pull/23606) +- Fix Team Admin getting 403 on `/user/filter/ui` when `scope_user_search_to_org` is enabled - [PR #23671](https://github.com/BerriAI/litellm/pull/23671) +- Fix Public Model Hub not showing config-defined models after save - [PR #23501](https://github.com/BerriAI/litellm/pull/23501) +- Fix fallback popup model dropdown z-index issue - [PR #23516](https://github.com/BerriAI/litellm/pull/23516) +- Fix double-counting bug in org/team key limit checks on `/key/update` +- Fix invite link allowing multiple password resets for the same link - [PR #22462](https://github.com/BerriAI/litellm/pull/22462) +- Fix key expiry default duration not being applied when `duration` is not set - [PR #22956](https://github.com/BerriAI/litellm/pull/22956) +- Fix all proxy models not including model access groups in key creation - [PR #23236](https://github.com/BerriAI/litellm/pull/23236) +- Fix admin viewers unable to see all organizations - [PR #22940](https://github.com/BerriAI/litellm/pull/22940) +- Fix Audit Logs UI: added server-side pagination, filtering, and drawer view - [PR #22476](https://github.com/BerriAI/litellm/pull/22476) +- Fix virtual keys in teams view not applying the team filter correctly - [PR #23065](https://github.com/BerriAI/litellm/pull/23065) +- Fix team expiry enforcement validation - [PR #22728](https://github.com/BerriAI/litellm/pull/22728) + +--- + +## AI Integrations + +### Logging + +- **[Helicone](../../docs/observability/helicone_integration)** + - Add Gemini and Vertex AI support to HeliconeLogger — routes Gemini and Vertex AI requests through the correct Helicone provider URL - [PR #19288](https://github.com/BerriAI/litellm/pull/19288) + - Fix correct provider URL for Vertex AI Gemini models - [PR #22603](https://github.com/BerriAI/litellm/pull/22603) + +- **[Langfuse](../../docs/proxy/logging#langfuse)** + - Fix failure path kwargs inconsistency causing dropped traces on failed requests - [PR #22390](https://github.com/BerriAI/litellm/pull/22390) + +- **[Vantage](https://vantage.sh)** + - Add Vantage integration for FOCUS 1.2 CSV export — export LiteLLM proxy spend data as FinOps Open Cost & Usage Specification reports, with time-windowed filenames to prevent overwrites - [PR #23333](https://github.com/BerriAI/litellm/pull/23333) + +- **General** + - Fix silent metrics race condition causing metric collision across experiments - [PR #23542](https://github.com/BerriAI/litellm/pull/23542) + +### Guardrails + +- **Guardrail mode default list** — Configure a default list of guardrail modes applied globally when no per-request mode is specified - [PR #22676](https://github.com/BerriAI/litellm/pull/22676) +- **Tag-based guardrail mode lists** — Specify a list of modes in tag-based guardrail configs instead of a single mode - [PR #23020](https://github.com/BerriAI/litellm/pull/23020) +- **Fix presidio PII token leak** — Edge case where Anthropic handle in Presidio caused PII data exposure in token response - [PR #22627](https://github.com/BerriAI/litellm/pull/22627) +- **Fix OTEL orphaned guardrail traces** — Span redundancy and missing response IDs in OpenTelemetry guardrail traces - [PR #23001](https://github.com/BerriAI/litellm/pull/23001) + +### Prompt Management + +No major prompt management changes in this release. + +### Secret Managers + +- **[Hashicorp Vault](../../docs/secret_managers)** — Full Hashicorp Vault integration as a config override backend — secrets defined in Vault are fetched at startup and override `config.yaml` values. UI support for managing vault-sourced credentials included - [PR #22939](https://github.com/BerriAI/litellm/pull/22939), [PR #23036](https://github.com/BerriAI/litellm/pull/23036) + +--- + +## MCP Gateway + +#### Features + +- **Token authentication for MCP servers** — configure `auth_type: "bearer"` per MCP server to require token-based auth on tool calls - [PR #23260](https://github.com/BerriAI/litellm/pull/23260) +- **Team-scoped MCP server filtering** — keys created under a team only see MCP servers available to that team - [PR #23323](https://github.com/BerriAI/litellm/pull/23323) +- **Per-server health recheck in UI** — trigger a health check for individual MCP servers without reloading all servers - [PR #23328](https://github.com/BerriAI/litellm/pull/23328) + +#### Bugs + +- Fix MCP server URL and tools management issues causing tool discovery to fail - [PR #22751](https://github.com/BerriAI/litellm/pull/22751) +- Fix MCP server health checks triggering on server deletion - [PR #23063](https://github.com/BerriAI/litellm/pull/23063) + +--- + +## Spend Tracking, Budgets and Rate Limiting + +- **Fix budget-linked keys never having spend reset** — Keys linked to budget objects were not having their spend reset on the configured reset interval - [PR #20688](https://github.com/BerriAI/litellm/pull/20688) +- **Flex pricing support** — Add `flex_pricing` field to cost map for providers that offer dynamic pricing tiers - [PR #22992](https://github.com/BerriAI/litellm/pull/22992) +- **Fix spend log cleanup** — Resolved lock tracking, integer retention, and skip-log-level issues in spend log cleanup job - [PR #22687](https://github.com/BerriAI/litellm/pull/22687) +- **Fix WebSearch spend log deduplication** — WebSearch interception was failing with thinking enabled; fixed along with spend log dedup - [PR #22679](https://github.com/BerriAI/litellm/pull/22679) +- **Fix TypeError when request has no API key** — Spend tracking was throwing unhandled exception when API key was absent from request - [PR #23363](https://github.com/BerriAI/litellm/pull/23363) + +--- + +## Performance / Loadbalancing / Reliability improvements + +- **Fix streaming crashes after ~1 hour** — `LLMClientCache._remove_key()` no longer calls `close()`/`aclose()` on evicted HTTP/SDK clients. In-flight requests were crashing with `RuntimeError: Cannot send a request, as the client has been closed.` after the 1-hour TTL expired. Cleanup now happens only at shutdown via `close_litellm_async_clients()` - [PR #22926](https://github.com/BerriAI/litellm/pull/22926) +- **Fix OOM / Prisma connection loss** on large installs — unbounded managed-object poll was exhausting Prisma connections after ~60–70 minutes on instances with 336K+ queued response rows - [PR #23472](https://github.com/BerriAI/litellm/pull/23472) +- **Centralize logging kwarg updates** — root cause fix migrating all logging updates to a single function, eliminating kwarg inconsistencies across logging paths - [PR #23659](https://github.com/BerriAI/litellm/pull/23659) +- **Fix tiktoken cache for non-root offline containers** — tiktoken cache now works correctly in offline environments running as non-root users - [PR #23498](https://github.com/BerriAI/litellm/pull/23498) +- **Block proxy startup when Redis transaction buffer has no Redis** — prevents silent data loss when `use_redis_transaction_buffer: true` is set without a Redis connection - [PR #23019](https://github.com/BerriAI/litellm/pull/23019) +- **Fix `InFlightRequestsMiddleware` crash** — undefined kwargs in middleware were causing request failures - [PR #22523](https://github.com/BerriAI/litellm/pull/22523) +- **Fix `BaseModelResponseIterator` crash on non-string stream chunks** — streaming was crashing when providers returned non-string chunk data - [PR #23497](https://github.com/BerriAI/litellm/pull/23497) +- **Fix `SERVER_ROOT_PATH` prefix handling** — strip prefix before checking mapped pass-through routes to prevent double-prefix issues - [PR #23414](https://github.com/BerriAI/litellm/pull/23414) +- **Add CodSpeed continuous performance benchmarks** — automated performance regression tracking on CI - [PR #23676](https://github.com/BerriAI/litellm/pull/23676) + +--- + +## Security + +- **Secret redaction in proxy logs** — Adds a `SecretRedactionFilter` to all LiteLLM loggers that scrubs API keys, tokens, and credentials from log messages, format args, exception tracebacks, and extra fields. Enabled by default; opt out with `LITELLM_DISABLE_REDACT_SECRETS=true` - [PR #23668](https://github.com/BerriAI/litellm/pull/23668), [PR #23667](https://github.com/BerriAI/litellm/pull/23667) +- **Bump PyJWT to `^2.12.0`** — addresses security vulnerability in `^2.10.1` - [PR #23678](https://github.com/BerriAI/litellm/pull/23678) +- **Bump `tar` to 7.5.11 and `tornado` to 6.5.5** — addresses CVEs in transitive dependencies - [PR #23602](https://github.com/BerriAI/litellm/pull/23602) + +--- + +## Database / Proxy Operations + +- **Fix Prisma migrate deploy on pre-existing instances** — resolved multiple bugs in migration recovery logic: missing return in the P3018 idempotent error handler and unhandled exceptions in `_roll_back_migration` that caused silent failures even after successful recovery - [PR #23655](https://github.com/BerriAI/litellm/pull/23655) +- **Make DB migration failure exit opt-in** — proxy no longer exits on `prisma migrate deploy` failure by default; enable with `--enforce_prisma_migration_check` - [PR #23675](https://github.com/BerriAI/litellm/pull/23675) + +--- + +## Documentation Updates + +- Add Anthropic `/v1/messages` → `/responses` parameter mapping reference - [PR #22893](https://github.com/BerriAI/litellm/pull/22893) +- Update Okta SSO docs and custom SSO handler example - [PR #22786](https://github.com/BerriAI/litellm/pull/22786) +- Add `LITELLM_MAX_BUDGET_PER_SESSION_TTL` to environment variables reference - [PR #23186](https://github.com/BerriAI/litellm/pull/23186) +- Add DB query performance guidelines to `CLAUDE.md` - [PR #23196](https://github.com/BerriAI/litellm/pull/23196) +- Add Gemini Vertex AI PayGo/priority cost tracking docs - [PR #22948](https://github.com/BerriAI/litellm/pull/22948) + +--- + +## New Contributors + +* @ryanh-ai made their first contribution in [PR #21542](https://github.com/BerriAI/litellm/pull/21542) +* @ryan-crabbe made their first contribution in [PR #23668](https://github.com/BerriAI/litellm/pull/23668) +* @Jah-yee made their first contribution in [PR #23525](https://github.com/BerriAI/litellm/pull/23525) +* @gambletan made their first contribution in [PR #23516](https://github.com/BerriAI/litellm/pull/23516) +* @awais786 made their first contribution in [PR #23183](https://github.com/BerriAI/litellm/pull/23183) +* @pradyyadav made their first contribution in [PR #23580](https://github.com/BerriAI/litellm/pull/23580) +* @xianzongxie-stripe made their first contribution in [PR #23492](https://github.com/BerriAI/litellm/pull/23492) +* @Harshit28j made their first contribution in [PR #23333](https://github.com/BerriAI/litellm/pull/23333) +* @codspeed-hq[bot] made their first contribution in [PR #23676](https://github.com/BerriAI/litellm/pull/23676) + +--- + +## Diff Summary + +## 03/16/2026 +* New Providers: 7 +* New Models / Updated Models: 116 new, 132 removed +* LLM API Endpoints: 37 +* Management Endpoints / UI: 31 +* AI Integrations: 8 +* MCP Gateway: 5 +* Spend Tracking, Budgets and Rate Limiting: 5 +* Performance / Loadbalancing / Reliability improvements: 9 +* Security: 3 +* Database / Proxy Operations: 2 +* Documentation Updates: 5 + +--- + +## Full Changelog +[v1.82.0-stable...v1.82.3-stable](https://github.com/BerriAI/litellm/compare/v1.82.0-stable...v1.82.3-stable) diff --git a/docs/my-website/sidebars-release-notes.js b/docs/my-website/sidebars-release-notes.js new file mode 100644 index 00000000000..6ed29003ce1 --- /dev/null +++ b/docs/my-website/sidebars-release-notes.js @@ -0,0 +1,14 @@ +// @ts-check + +/** @type {import('@docusaurus/plugin-content-docs').SidebarsConfig} */ +const sidebars = { + releaseNotesSidebar: [ + { type: 'doc', id: 'index', label: 'Release Notes' }, + { + type: 'autogenerated', + dirName: '.', + }, + ], +}; + +module.exports = sidebars; diff --git a/docs/my-website/sidebars.js b/docs/my-website/sidebars.js index 8383b826f0d..4c0471fb8f4 100644 --- a/docs/my-website/sidebars.js +++ b/docs/my-website/sidebars.js @@ -20,6 +20,7 @@ const sidebars = { { type: "category", label: "Observability", + link: { type: "doc", id: "integrations/observability_index" }, items: [ { type: "category", @@ -29,61 +30,24 @@ const sidebars = { type: "autogenerated", dirName: "contribute_integration" } - ] + ], }, { type: "autogenerated", dirName: "observability" - } + }, ], }, { type: "category", - label: "Guardrails", + label: "Guardrail Providers", + link: { + type: "generated-index", + title: "Guardrail Providers", + description: "Add safety and content filtering to LLM calls", + slug: "/guardrail_providers" + }, items: [ - "proxy/guardrails/quick_start", - "proxy/guardrails/team_based_guardrails", - "proxy/guardrails/guardrail_load_balancing", - "proxy/guardrails/test_playground", - "proxy/guardrails/litellm_content_filter", - "proxy/guardrails/realtime_guardrails", - { - type: "category", - label: "Providers", - items: [ - ...[ - "proxy/guardrails/qualifire", - "proxy/guardrails/aim_security", - "proxy/guardrails/onyx_security", - "proxy/guardrails/aporia_api", - "proxy/guardrails/azure_content_guardrail", - "proxy/guardrails/bedrock", - "proxy/guardrails/crowdstrike_aidr", - "proxy/guardrails/enkryptai", - "proxy/guardrails/ibm_guardrails", - "proxy/guardrails/grayswan", - "proxy/guardrails/hiddenlayer", - "proxy/guardrails/lasso_security", - "proxy/guardrails/guardrails_ai", - "proxy/guardrails/lakera_ai", - "proxy/guardrails/model_armor", - "proxy/guardrails/noma_security", - "proxy/guardrails/dynamoai", - "proxy/guardrails/openai_moderation", - "proxy/guardrails/pangea", - "proxy/guardrails/pillar_security", - "proxy/guardrails/pii_masking_v2", - "proxy/guardrails/panw_prisma_airs", - "proxy/guardrails/secret_detection", - "proxy/guardrails/custom_guardrail", - "proxy/guardrails/custom_code_guardrail", - "proxy/guardrails/prompt_injection", - "proxy/guardrails/tool_permission", - "proxy/guardrails/zscaler_ai_guard", - "proxy/guardrails/javelin" - ].sort(), - ], - }, { type: "category", label: "Contributing to Guardrails", @@ -93,16 +57,42 @@ const sidebars = { "adding_provider/adding_guardrail_support", ] }, - ], - }, - { - type: "category", - label: "Policies", - items: [ - "proxy/guardrails/guardrail_policies", - "proxy/guardrails/policy_flow_builder", - "proxy/guardrails/policy_templates", - "proxy/guardrails/policy_tags", + { + type: "doc", + id: "proxy/guardrails/team_based_guardrails", + label: "Team Bring-Your-Own Guardrails", + }, + ...[ + "proxy/guardrails/qualifire", + "proxy/guardrails/aim_security", + "proxy/guardrails/onyx_security", + "proxy/guardrails/aporia_api", + "proxy/guardrails/azure_content_guardrail", + "proxy/guardrails/bedrock", + "proxy/guardrails/crowdstrike_aidr", + "proxy/guardrails/enkryptai", + "proxy/guardrails/ibm_guardrails", + "proxy/guardrails/grayswan", + "proxy/guardrails/hiddenlayer", + "proxy/guardrails/lasso_security", + "proxy/guardrails/guardrails_ai", + "proxy/guardrails/lakera_ai", + "proxy/guardrails/model_armor", + "proxy/guardrails/noma_security", + "proxy/guardrails/dynamoai", + "proxy/guardrails/openai_moderation", + "proxy/guardrails/pangea", + "proxy/guardrails/pillar_security", + "proxy/guardrails/pii_masking_v2", + "proxy/guardrails/panw_prisma_airs", + "proxy/guardrails/secret_detection", + "proxy/guardrails/custom_guardrail", + "proxy/guardrails/custom_code_guardrail", + "proxy/guardrails/prompt_injection", + "proxy/guardrails/tool_permission", + "proxy/guardrails/zscaler_ai_guard", + "proxy/guardrails/javelin" + ].sort(), ], }, { @@ -115,11 +105,6 @@ const sidebars = { "proxy/pyroscope_profiling" ] }, - { - type: "doc", - id: "integrations/websearch_interception", - label: "Web Search Integration" - }, { type: "category", label: "[Beta] Prompt Management", @@ -140,7 +125,7 @@ const sidebars = { }, { type: "category", - label: "AI Tools (OpenWebUI, Claude Code, etc.)", + label: "AI Tools", link: { type: "generated-index", title: "AI Tools", @@ -167,14 +152,13 @@ const sidebars = { }, "tutorials/opencode_integration", "tutorials/openclaw_integration", - "tutorials/cost_tracking_coding", "tutorials/cursor_integration", "tutorials/github_copilot_integration", "tutorials/litellm_gemini_cli", - "tutorials/google_genai_sdk", "tutorials/litellm_qwen_code_cli", "tutorials/openai_codex", - "tutorials/retool_assist" + "tutorials/retool_assist", + "tutorials/cost_tracking_coding" ] }, { @@ -191,15 +175,45 @@ const sidebars = { "tutorials/claude_agent_sdk", "tutorials/copilotkit_sdk", "tutorials/google_adk", + "tutorials/google_genai_sdk", "tutorials/livekit_xai_realtime", + "integrations/letta", + { type: "doc", id: "tutorials/instructor", label: "Instructor with LiteLLM" }, + { type: "doc", id: "langchain/langchain", label: "LangChain with LiteLLM" }, "projects/openai-agents" ] }, + { + type: "category", + label: "Manage with AI Agents", + link: { + type: "generated-index", + title: "Manage with AI Agents", + description: "Use AI agents to manage your LiteLLM deployment — create users, teams, keys, models, and more via natural language.", + slug: "/manage_with_ai_agents" + }, + items: [ + "tutorials/claude_code_skills", + ] + }, ], // But you can create a sidebar manually tutorialSidebar: [ - { type: "doc", id: "index", label: "Getting Started" }, + // ════════════════════════════════════════════════════════════ + // GET STARTED + // ════════════════════════════════════════════════════════════ + { + type: "category", + label: "Get Started", + collapsible: false, + collapsed: false, + items: [ + { type: "doc", id: "index", label: "Quickstart" }, + { type: "link", label: "Models & Pricing", href: "https://models.litellm.ai" }, + { type: "link", label: "Changelog", href: "/release_notes" }, + ], + }, { type: "category", @@ -267,11 +281,6 @@ const sidebars = { }, "completion/token_usage", "exception_mapping", - { - type: "category", - label: "LangChain, LlamaIndex, Instructor", - items: ["langchain/langchain", "tutorials/instructor"], - } ], }, { @@ -284,16 +293,46 @@ const sidebars = { slug: "/simple_proxy", }, items: [ - "proxy/docker_quick_start", + { type: "doc", id: "proxy/docker_quick_start", label: "Getting Started Tutorial" }, { - type: "link", - label: "A2A Agent Gateway", - href: "https://docs.litellm.ai/docs/a2a", - }, - { - type: "link", - label: "MCP Gateway", - href: "https://docs.litellm.ai/docs/mcp", + type: "category", + label: "Agent & MCP Gateway", + items: [ + { + type: "category", + label: "A2A Agent Gateway", + items: [ + "a2a", + "a2a_invoking_agents", + "a2a_agent_headers", + "a2a_cost_tracking", + "a2a_agent_permissions", + "a2a_iteration_budgets", + ], + }, + { + type: "category", + label: "MCP Gateway", + items: [ + "mcp", + "mcp_usage", + "mcp_openapi", + "mcp_oauth", + "mcp_aws_sigv4", + "mcp_zero_trust", + "mcp_public_internet", + "mcp_semantic_filter", + "mcp_control", + "mcp_cost", + "mcp_guardrail", + { + type: "link", + label: "MCP Troubleshooting Guide", + href: "/docs/mcp_troubleshoot" + }, + ], + }, + ], }, { "type": "category", @@ -332,6 +371,7 @@ const sidebars = { label: "Setup & SSO", items: [ "proxy/admin_ui_sso", + "proxy/ui/ui_edit_logo", "proxy/custom_sso", "proxy/custom_root_ui", "tutorials/scim_litellm", @@ -351,7 +391,11 @@ const sidebars = { type: "category", label: "Teams & Organizations", items: [ - "proxy/access_control", + { + type: "link", + label: "Role-based Access Controls (RBAC) →", + href: "/docs/proxy/access_control" + }, "proxy/self_serve", "proxy/public_teams", "proxy/ui_project_management", @@ -434,14 +478,40 @@ const sidebars = { }, "proxy/caching", { - type: "link", + type: "category", label: "Guardrails", - href: "https://docs.litellm.ai/docs/proxy/guardrails/quick_start", + items: [ + "proxy/guardrails/quick_start", + "proxy/guardrails/team_based_guardrails", + "proxy/guardrails/guardrail_load_balancing", + "proxy/guardrails/test_playground", + "proxy/guardrails/litellm_content_filter", + "proxy/guardrails/realtime_guardrails", + { + type: "link", + label: "Providers →", + href: "/docs/guardrail_providers", + }, + { + type: "category", + label: "Contributing to Guardrails", + items: [ + "adding_provider/generic_guardrail_api", + "adding_provider/simple_guardrail_tutorial", + "adding_provider/adding_guardrail_support", + ] + }, + ], }, { - type: "link", + type: "category", label: "Policies", - href: "https://docs.litellm.ai/docs/proxy/guardrails/guardrail_policies", + items: [ + "proxy/guardrails/guardrail_policies", + "proxy/guardrails/policy_flow_builder", + "proxy/guardrails/policy_templates", + "proxy/guardrails/policy_tags", + ], }, { type: "category", @@ -537,16 +607,9 @@ const sidebars = { }, items: [ { - type: "category", + type: "link", label: "/a2a - A2A Agent Gateway", - items: [ - "a2a", - "a2a_invoking_agents", - "a2a_agent_headers", - "a2a_cost_tracking", - "a2a_agent_permissions", - "a2a_iteration_budgets" - ], + href: "/docs/a2a", }, "assistants", "audio_transcription", @@ -617,11 +680,13 @@ const sidebars = { "mcp_openapi", "mcp_oauth", "mcp_aws_sigv4", + "mcp_zero_trust", "mcp_public_internet", "mcp_semantic_filter", "mcp_control", "mcp_cost", "mcp_guardrail", + "mcp_zero_trust", "mcp_troubleshoot", ] }, @@ -669,6 +734,7 @@ const sidebars = { "rag_ingest", "rag_query", "realtime", + "proxy/realtime_webrtc", "rerank", "response_api", "response_api_compact", @@ -958,51 +1024,14 @@ const sidebars = { "providers/zai", ], }, - { - type: "category", - label: "Guides", - items: [ - "budget_manager", - "completion/computer_use", - "completion/web_search", - "completion/web_fetch", - "completion/function_call", - "completion/audio", - "completion/document_understanding", - "completion/drop_params", - "completion/image_generation_chat", - "completion/json_mode", - "completion/knowledgebase", - "providers/anthropic_tool_search", - "guides/code_interpreter", - "completion/message_trimming", - "completion/message_sanitization", - "completion/model_alias", - "completion/mock_requests", - "completion/predict_outputs", - "completion/prefix", - "completion/prompt_caching", - "completion/prompt_formatting", - "completion/reliable_completions", - "completion/stream", - "completion/provider_specific_params", - "completion/vision", - "exception_mapping", - "completion/batching", - "guides/finetuned_models", - "guides/security_settings", - "proxy/veo_video_generation", - "reasoning_content", - "extras/creating_adapters", - ] - }, + { type: "category", - label: "Routing, Loadbalancing & Fallbacks", + label: "Routing & Load Balancing", link: { type: "generated-index", - title: "Routing, Loadbalancing & Fallbacks", + title: "Routing & Load Balancing", description: "Learn how to load balance, route, and set fallbacks for your LLM requests", slug: "/routing-load-balancing", }, @@ -1030,43 +1059,6 @@ const sidebars = { "load_test_rpm", ] }, - { - type: "category", - label: "Tutorials", - items: [ - { - type: "link", - label: "AI Coding Tools (OpenWebUI, Claude Code, Gemini CLI, OpenAI Codex, etc.)", - href: "/docs/ai_tools", - }, - "tutorials/anthropic_file_usage", - "tutorials/default_team_self_serve", - "tutorials/msft_sso", - "tutorials/prompt_caching", - "tutorials/tag_management", - 'tutorials/litellm_proxy_aporia', - "tutorials/presidio_pii_masking", - "tutorials/elasticsearch_logging", - "tutorials/gemini_realtime_with_audio", - "tutorials/claude_code_beta_headers", - { - type: "category", - label: "LiteLLM Python SDK Tutorials", - items: [ - 'tutorials/azure_openai', - 'tutorials/instructor', - "tutorials/gradio_integration", - "tutorials/huggingface_codellama", - "tutorials/huggingface_tutorial", - "tutorials/TogetherAI_liteLLM", - "tutorials/finetuned_chat_gpt", - "tutorials/text_completion", - "tutorials/first_playground", - "tutorials/model_fallbacks", - ], - }, - ] - }, { type: "category", label: "Contributing", @@ -1123,12 +1115,11 @@ const sidebars = { "projects/Codium PR Agent", "projects/Prompt2Model", "projects/SalesGPT", + "projects/Softgen", "projects/Quivr", "projects/Langstream", "projects/Otter", - "projects/GPT Migrate", "projects/YiVal", - "projects/LiteLLM Proxy", "projects/llm_cord", "projects/pgai", "projects/GPTLocalhost", @@ -1166,23 +1157,309 @@ const sidebars = { "troubleshoot", ], }, + ], +}; + +const learnSidebar = { + learnSidebar: [ + // ── Landing page ────────────────────────────────────────────────── + { type: "doc", id: "learn/index", label: "Learn" }, { type: "category", - label: "Blog", + label: "Start Here", + collapsible: true, + collapsed: false, + items: [ + "learn/sdk_quickstart", + "learn/gateway_quickstart", + ], + }, + + // ── Guides ──────────────────────────────────────────────────────── + { + type: "category", + label: "Guides", + collapsible: true, + collapsed: false, + link: { type: "doc", id: "guides/index" }, items: [ { - type: "link", - label: "Day 0 Support: Claude Sonnet 4.6", - href: "/blog/claude_sonnet_4_6", + type: "category", + label: "Core Requests", + collapsible: true, + collapsed: false, + link: { + type: "generated-index", + title: "Core Requests", + description: "Streaming, batching, structured outputs, and reasoning behavior", + slug: "/guides/core_request_response_patterns" + }, + items: [ + "completion/stream", + "completion/batching", + "completion/json_mode", + "reasoning_content", + ], + }, + { + type: "category", + label: "Tool Calling", + collapsible: true, + collapsed: true, + link: { + type: "generated-index", + title: "Tool Calling", + description: "Function calling, web tools, interception patterns, computer use, code interpreter, and tool-call hygiene", + slug: "/guides/tools_integrations" + }, + items: [ + "completion/function_call", + "completion/web_search", + { + type: "doc", + id: "integrations/websearch_interception", + label: "Web Search Interception", + }, + "completion/web_fetch", + "completion/computer_use", + "guides/code_interpreter", + "completion/message_sanitization", + ], + }, + { + type: "category", + label: "Multimodal I/O", + collapsible: true, + collapsed: true, + link: { + type: "generated-index", + title: "Multimodal I/O", + description: "Vision, audio, PDFs, image generation, and video generation", + slug: "/guides/multimodal_io" + }, + items: [ + "completion/vision", + "completion/audio", + "completion/document_understanding", + "completion/image_generation_chat", + "proxy/veo_video_generation", + ], + }, + { + type: "category", + label: "Retrieval & Knowledge", + collapsible: true, + collapsed: true, + link: { + type: "generated-index", + title: "Retrieval & Knowledge", + description: "Vector stores, file search, citations, and knowledge-base routing", + slug: "/guides/retrieval_knowledge" + }, + items: [ + "completion/knowledgebase", + ], + }, + { + type: "category", + label: "Prompts & Context", + collapsible: true, + collapsed: true, + link: { + type: "generated-index", + title: "Prompts & Context", + description: "Prompt caching, trimming, formatting, assistant prefill, and predicted outputs", + slug: "/guides/prompts_context" + }, + items: [ + "completion/prefix", + "completion/predict_outputs", + "completion/message_trimming", + "completion/prompt_caching", + "completion/prompt_formatting", + ], + }, + { + type: "category", + label: "Compatibility & Extensibility", + collapsible: true, + collapsed: true, + link: { + type: "generated-index", + title: "Compatibility & Extensibility", + description: "Provider-specific params, model aliases, fine-tuned models, and adapters", + slug: "/guides/compatibility_extensibility" + }, + items: [ + "completion/provider_specific_params", + "completion/drop_params", + "completion/model_alias", + "guides/finetuned_models", + "extras/creating_adapters", + ], + }, + { + type: "category", + label: "Reliability, Testing & Spend", + collapsible: true, + collapsed: true, + link: { + type: "generated-index", + title: "Reliability, Testing & Spend", + description: "Retries, fallbacks, mock responses, and budget controls", + slug: "/guides/reliability_testing_spend" + }, + items: [ + "completion/mock_requests", + "completion/reliable_completions", + "budget_manager", + ], + }, + { + type: "category", + label: "Security & Network", + collapsible: true, + collapsed: true, + link: { + type: "generated-index", + title: "Security & Network", + description: "SSL, custom CA bundles, HTTP proxy settings, and per-service verification", + slug: "/guides/security_network" + }, + items: [ + "guides/security_settings", + ], + }, + ], + }, + + // ── Tutorials ───────────────────────────────────────────────────── + { + type: "category", + label: "Tutorials", + collapsible: true, + collapsed: false, + link: { type: "doc", id: "tutorials/index" }, + items: [ + { + type: "category", + label: "Getting Started", + collapsed: false, + link: { + type: "generated-index", + title: "Getting Started", + description: "Installation, playground, text completion, and mock completions", + slug: "/tutorials/getting_started" + }, + items: [ + "tutorials/installation", + "tutorials/first_playground", + "tutorials/text_completion", + "tutorials/mock_completion", + ], }, { type: "link", - label: "Incident: Broken Model Cost Map", - href: "/blog/model-cost-map-incident", + label: "Agent SDKs & Frameworks", + href: "/docs/agent_sdks", + }, + { + type: "link", + label: "AI Coding Tools", + href: "/docs/ai_tools", + }, + { + type: "category", + label: "Python SDK", + collapsed: true, + link: { + type: "generated-index", + title: "Python SDK", + description: "Tutorials using only the Python SDK — no proxy server required", + slug: "/tutorials/python_sdk" + }, + items: [ + "tutorials/gradio_integration", + "tutorials/provider_specific_params", + "tutorials/model_fallbacks", + "tutorials/fallbacks", + ], + }, + { + type: "category", + label: "Provider Setup", + collapsed: true, + link: { + type: "generated-index", + title: "Provider Setup", + description: "Connect LiteLLM to Azure OpenAI, HuggingFace, TogetherAI, local models, and more", + slug: "/tutorials/provider_tutorials" + }, + items: [ + "tutorials/azure_openai", + "tutorials/TogetherAI_liteLLM", + "tutorials/huggingface_tutorial", + "tutorials/huggingface_codellama", + "tutorials/finetuned_chat_gpt", + "tutorials/oobabooga", + ], + }, + { + type: "category", + label: "Proxy: Admin & Access", + collapsed: true, + link: { + type: "generated-index", + title: "Proxy: Admin & Access", + description: "User and team management, SSO, SCIM, and routing rules", + slug: "/tutorials/proxy_admin_access" + }, + items: [ + "tutorials/default_team_self_serve", + "tutorials/msft_sso", + "tutorials/scim_litellm", + "tutorials/tag_management", + ], + }, + { + type: "category", + label: "Proxy: Features & Safety", + collapsed: true, + link: { + type: "generated-index", + title: "Proxy: Features & Safety", + description: "Prompt caching, passthrough APIs, realtime, guardrails, and PII masking", + slug: "/tutorials/proxy_features_safety" + }, + items: [ + "tutorials/prompt_caching", + "tutorials/anthropic_file_usage", + "tutorials/gemini_realtime_with_audio", + "tutorials/litellm_proxy_aporia", + "tutorials/presidio_pii_masking", + ], + }, + { + type: "category", + label: "Observability & Evaluation", + collapsed: true, + link: { + type: "generated-index", + title: "Observability & Evaluation", + description: "Logging, monitoring, benchmarking, and evaluation suites", + slug: "/tutorials/observability_evaluation" + }, + items: [ + "tutorials/elasticsearch_logging", + "tutorials/compare_llms", + "tutorials/litellm_Test_Multiple_Providers", + "tutorials/eval_suites", + "tutorials/lm_evaluation_harness", + ], }, ], }, ], }; -module.exports = sidebars; +module.exports = { ...sidebars, ...learnSidebar }; diff --git a/docs/my-website/src/components/NavigationCards/index.js b/docs/my-website/src/components/NavigationCards/index.js new file mode 100644 index 00000000000..5efd89ee140 --- /dev/null +++ b/docs/my-website/src/components/NavigationCards/index.js @@ -0,0 +1,44 @@ +import React from 'react'; +import Link from '@docusaurus/Link'; +import styles from './styles.module.css'; + +export default function NavigationCards({ items, columns = 2 }) { + return ( +
+ {items.map((item, i) => { + const isExternal = + item.to && (item.to.startsWith('http://') || item.to.startsWith('https://')); + return ( + + {item.icon && ( +
{item.icon}
+ )} +
{item.title}
+ {item.description && ( +
{item.description}
+ )} + {item.listDescription && ( +
    + {item.listDescription.map((line, j) => ( +
  • {line}
    • + ))} +
+ )} + {isExternal && ( + ↗ + )} + + ); + })} +
+ ); +} diff --git a/docs/my-website/src/components/NavigationCards/styles.module.css b/docs/my-website/src/components/NavigationCards/styles.module.css new file mode 100644 index 00000000000..64f5a42374b --- /dev/null +++ b/docs/my-website/src/components/NavigationCards/styles.module.css @@ -0,0 +1,82 @@ +.grid { + display: grid; + grid-template-columns: repeat(var(--nav-columns, 2), 1fr); + gap: 0.75rem; + margin: 1.25rem 0; +} + +@media (max-width: 768px) { + .grid { + grid-template-columns: 1fr; + } +} + +.card { + position: relative; + display: flex; + flex-direction: column; + padding: 1rem 1.1rem; + border: 1px solid var(--ifm-color-emphasis-200); + border-radius: 6px; + text-decoration: none !important; + color: inherit !important; + background: var(--ifm-background-surface-color); + transition: border-color 0.15s ease, box-shadow 0.15s ease; +} + +.card:hover { + border-color: var(--ifm-color-primary); + box-shadow: 0 0 0 1px var(--ifm-color-primary); + text-decoration: none !important; +} + +[data-theme='dark'] .card { + background: var(--ifm-background-surface-color); + border-color: #2d3748; +} + +[data-theme='dark'] .card:hover { + border-color: var(--ifm-color-primary); + box-shadow: 0 0 0 1px var(--ifm-color-primary); +} + +.icon { + font-size: 1.4rem; + margin-bottom: 0.5rem; + line-height: 1; +} + +.title { + font-size: 14px; + font-weight: 600; + margin-bottom: 0.35rem; + color: var(--ifm-heading-color); +} + +.description { + font-size: 13px; + line-height: 1.5; + color: var(--ifm-color-emphasis-700); + margin-bottom: 0.5rem; +} + +.list { + margin: 0.35rem 0 0 0; + padding-left: 1.1rem; + list-style: disc; +} + +.list li { + font-size: 12.5px; + color: var(--ifm-color-emphasis-700); + line-height: 1.6; + margin-bottom: 0; +} + +.externalIcon { + position: absolute; + top: 0.75rem; + right: 0.75rem; + font-size: 12px; + color: var(--ifm-color-emphasis-500); +} diff --git a/docs/my-website/src/components/WebRTCTester.jsx b/docs/my-website/src/components/WebRTCTester.jsx new file mode 100644 index 00000000000..3ade6dd7689 --- /dev/null +++ b/docs/my-website/src/components/WebRTCTester.jsx @@ -0,0 +1,83 @@ +import DashboardWebRTCTester from "../../../../ui/litellm-dashboard/src/components/WebRTCTester.jsx"; + +const LIGHT_MODE_OVERRIDES = ` +.wrt-wrap { + background: #1f2937; + border: 1px solid #334155; +} +.wrt-toggle, +.wrt-toggle:hover { + background: #111827; +} +.wrt-toggle-title, +.we-msg { + color: #e2e8f0; +} +.wrt-toggle-sub, +.wrt-label, +.wrt-field label, +.wrt-flow-box, +.wrt-flow-arrow, +.wrt-meta-row span:first-child, +.wrt-header-title, +.wrt-tab, +.we-time { + color: #94a3b8; +} +.wrt-body, +.wrt-sidebar, +.wrt-main, +.wrt-header, +.wrt-tabs, +.wrt-sdp-box, +.wrt-sdp-hdr, +.wrt-divider { + border-color: #334155; +} +.wrt-header { + background: #111827; +} +.wrt-field input, +.wrt-mic-btn, +.wrt-status-pill { + background: #0b1220; + border-color: #334155; + color: #e2e8f0; +} +.wrt-field input:focus, +.wrt-btn-ghost:hover { + border-color: #60a5fa; +} +.wrt-btn-ghost { + background: #0b1220; + border-color: #334155; + color: #e2e8f0; +} +.wrt-log::-webkit-scrollbar-thumb { + background: #475569; +} +.wrt-tab.active { + color: #93c5fd; + border-bottom-color: #93c5fd; +} +.wrt-empty, +.wrt-audio-status, +.wrt-meta-row span:last-child { + color: #cbd5e1; +} +.wrt-sdp-dot { + background: #475569; +} +.wrt-sdp-pane textarea { + color: #e2e8f0; +} +`; + +export default function WebRTCTester() { + return ( + <> + + + + ); +} diff --git a/docs/my-website/src/css/custom.css b/docs/my-website/src/css/custom.css index 9fa4443afc9..d0702fcc57c 100644 --- a/docs/my-website/src/css/custom.css +++ b/docs/my-website/src/css/custom.css @@ -1,10 +1,16 @@ /** - * Any CSS included here will be global. The classic template - * bundles Infima by default. Infima is a CSS framework designed to - * work well for content-centric websites. + * Global CSS overrides for LiteLLM docs. + * Infima (Docusaurus CSS framework) variables + custom styling. */ -/* You can override the default Infima variables here. */ +/* ========================================= + FONTS + ========================================= */ +@import url('https://fonts.googleapis.com/css2?family=Inter:wght@300;400;500;600;700&display=swap'); + +/* ========================================= + ROOT — Light Mode Variables + ========================================= */ :root { --ifm-color-primary: #2e8555; --ifm-color-primary-dark: #29784c; @@ -13,11 +19,22 @@ --ifm-color-primary-light: #33925d; --ifm-color-primary-lighter: #359962; --ifm-color-primary-lightest: #3cad6e; - --ifm-code-font-size: 95%; - --docusaurus-highlighted-code-line-bg: rgba(0, 0, 0, 0.1); + --ifm-code-font-size: 85%; + --ifm-menu-color: #6b7280; + --docusaurus-highlighted-code-line-bg: rgba(0, 0, 0, 0.08); + --ifm-font-family-base: 'Inter', -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif; + --ifm-heading-font-weight: 600; + --ifm-font-size-base: 15px; + --ifm-line-height-base: 1.65; + --ifm-border-radius: 6px; + /* Wider reading column — reduces excessive whitespace on large monitors */ + --ifm-container-width: 1380px; + --ifm-container-width-xl: 1560px; } -/* For readability concerns, you should choose a lighter palette in dark mode. */ +/* ========================================= + DARK MODE Variables + ========================================= */ [data-theme='dark'] { --ifm-color-primary: #25c2a0; --ifm-color-primary-dark: #21af90; @@ -26,10 +43,710 @@ --ifm-color-primary-light: #29d5b0; --ifm-color-primary-lighter: #32d8b4; --ifm-color-primary-lightest: #4fddbf; - --docusaurus-highlighted-code-line-bg: rgba(0, 0, 0, 0.3); + --ifm-background-color: #0d1117; + --ifm-background-surface-color: #161b22; + --docusaurus-highlighted-code-line-bg: rgba(255, 255, 255, 0.07); } -/* Levo logo sizing and theme switching */ +/* ========================================= + TYPOGRAPHY + ========================================= */ +.theme-doc-markdown h1 { + font-size: 2.2rem; + letter-spacing: -0.02em; + line-height: 1.2; +} + +.theme-doc-markdown h2 { + font-size: 1.6rem; + letter-spacing: -0.01em; + line-height: 1.3; +} + +.theme-doc-markdown h3 { + font-size: 1.25rem; + line-height: 1.4; +} + +.theme-doc-markdown p, +.theme-doc-markdown ul, +.theme-doc-markdown ol { + font-size: 0.9rem; +} + +.theme-doc-markdown table { + font-size: 0.875rem; +} + +.theme-doc-markdown td { + font-size: 0.85rem; +} + +/* ========================================= + NAVBAR + ========================================= */ +[data-theme='light'] .navbar { + background-color: #ffffff; + box-shadow: 0 1px 0 0 #e5e7eb; +} + +[data-theme='dark'] .navbar { + background-color: var(--ifm-background-color); + border-bottom: 1px solid #21262d; + box-shadow: none; +} + +.navbar__link { + font-weight: 400 !important; + font-size: 14px !important; + border-bottom: 2px solid transparent !important; + padding-bottom: 2px; +} + +.navbar__link--active { + font-weight: 500 !important; + border-bottom: 2px solid var(--ifm-color-primary) !important; +} + +@media (max-width: 1330px) { + .navbar__link { + font-size: 13px !important; + } +} + +/* Three-column navbar: logo | center nav | right icons */ +@media (min-width: 997px) { + .navbar__inner { + display: flex !important; + align-items: center; + justify-content: space-between; + } + + .navbar__brand-col { + display: flex; + align-items: center; + flex: 0 0 auto; + margin-left: 1rem; + } + + .navbar__brand-col .navbar__brand { + font-size: 1.25rem; + } + + .navbar__brand-col .navbar__logo { + height: 2rem; + width: auto; + } + + .navbar__center-col { + display: flex; + align-items: center; + justify-content: center; + flex: 1; + } + + .navbar__right-col { + display: flex; + align-items: center; + flex: 0 0 auto; + gap: 0.25rem; + margin-right: 2rem; + } +} + +/* ========================================= + ALERTS / ADMONITIONS + ========================================= */ +.alert { + padding: 0.75rem 1rem; + font-size: 14px; + border-radius: var(--ifm-border-radius); + border-left-width: 3px; +} + +/* Light mode */ +.alert--info { + --ifm-alert-background-color: #f0f7ff !important; + --ifm-alert-border-color: #2264ab !important; +} + +.alert--success { + --ifm-alert-background-color: #f0fff8 !important; + --ifm-alert-border-color: #09bda8 !important; +} + +.alert--secondary { + --ifm-alert-background-color: #f8fafc !important; + --ifm-alert-border-color: #64748b !important; +} + +.alert--danger { + --ifm-alert-background-color: #fff0f5 !important; + --ifm-alert-border-color: #e11d48 !important; +} + +.alert--warning { + --ifm-alert-background-color: #fffbeb !important; + --ifm-alert-border-color: #d97706 !important; +} + +/* Dark mode */ +[data-theme='dark'] .alert--info { + --ifm-alert-background-color: #0c1e30 !important; + --ifm-alert-border-color: #3b82f6 !important; + color: #bfdbfe !important; +} + +[data-theme='dark'] .alert--success { + --ifm-alert-background-color: #022c22 !important; + --ifm-alert-border-color: #10b981 !important; +} + +[data-theme='dark'] .alert--secondary { + --ifm-alert-background-color: #0f172a !important; + --ifm-alert-border-color: #475569 !important; +} + +[data-theme='dark'] .alert--danger { + --ifm-alert-background-color: #2d0a14 !important; + --ifm-alert-border-color: #f43f5e !important; +} + +[data-theme='dark'] .alert--warning { + --ifm-alert-background-color: #1c1200 !important; + --ifm-alert-border-color: #f59e0b !important; +} + +/* ========================================= + COLLAPSIBLE / DETAILS + ========================================= */ +details { + color: #1d232e; + background-color: #ffffff; + border: 1px solid #e9eef2 !important; + border-radius: var(--ifm-border-radius) !important; + padding: 0.75rem !important; + margin-bottom: 1rem !important; + margin-top: 1.5rem !important; + box-shadow: 0 1px 4px 0 rgba(0, 0, 0, 0.05) !important; +} + +details summary { + font-weight: 600 !important; +} + +details [class*='collapsibleContent'] { + border-top: 1px solid #e2e8f0 !important; +} + +details [class*='collapsibleContent'] p, +details [class*='collapsibleContent'] ul { + font-size: 13px !important; + line-height: 1.75; +} + +[data-theme='dark'] details { + background-color: #1c2130 !important; + color: #e5e7eb !important; + border: 1px solid #2d3748 !important; + box-shadow: 0 1px 6px 0 rgba(0, 0, 0, 0.3) !important; +} + +[data-theme='dark'] details summary { + color: #f3f4f6 !important; +} + +[data-theme='dark'] details [class*='collapsibleContent'] { + border-top: 1px solid #2d3748 !important; +} + +/* ========================================= + TABS + ========================================= */ +.tabs-container > div { + padding: 1rem; + border: 1px solid #e2e8f0; + border-radius: var(--ifm-border-radius); +} + +/* Remove styling from nested tabs containers */ +.tabs-container .tabs-container > div { + padding: 0; + border: none; + border-radius: 0; +} + +[data-theme='dark'] .tabs-container > div { + border-color: #2d3748; +} + +ul.tabs { + border-bottom: 1px solid #e2e8f0 !important; + column-gap: 0.5rem !important; +} + +[data-theme='dark'] ul.tabs { + border-bottom-color: #2d3748 !important; +} + +li.tabs__item { + padding: 0.5rem !important; + font-weight: 500 !important; + font-size: 14px !important; + border-bottom: 2px solid transparent !important; +} + +li.tabs__item--active { + border-bottom: 2px solid var(--ifm-color-primary) !important; +} + +/* ========================================= + CODE BLOCKS + ========================================= */ +.prism-code { + border-radius: var(--ifm-border-radius); + font-size: 12.5px !important; + line-height: 1.6; +} + +[data-theme='dark'] .prism-code { + border: 1px solid #21262d; +} + +[class*='codeLineNumber']::before { + font-size: 12px !important; + display: flex !important; + align-items: center !important; + justify-content: center !important; +} + +[class*='codeBlockTitle'] { + font-size: 13px !important; + font-weight: 500 !important; + padding: 0.5rem 1rem !important; + border-bottom: 1px solid #334155 !important; +} + +.theme-code-block-highlighted-line { + background-color: rgba(0, 0, 0, 0.1) !important; +} + +[data-theme='dark'] .theme-code-block-highlighted-line { + background-color: rgba(255, 255, 255, 0.06) !important; +} + +.theme-code-block-highlighted-line > span { + background-color: transparent !important; +} + +/* ========================================= + SIDEBAR / MENU + ========================================= */ +.menu { + font-weight: 400; + padding: 0.5rem 0.25rem !important; + background-image: radial-gradient(rgba(0, 0, 0, 0.07) 1px, transparent 1px); + background-size: 24px 24px; +} + +[data-theme='dark'] .menu { + background-image: radial-gradient(rgba(255, 255, 255, 0.04) 1px, transparent 1px); + background-size: 24px 24px; +} + +.menu__link { + font-size: 14px; + padding: 0.22rem 0.75rem !important; + border-radius: 4px; +} + +.menu__link--active { + font-weight: 600 !important; + background-color: rgba(46, 133, 85, 0.08) !important; +} + +[data-theme='dark'] .menu__link--active { + background-color: rgba(37, 194, 160, 0.1) !important; +} + +/* ─── Sidebar collapse arrows — uniform size & alignment ────── */ + +/* 1. Categories WITHOUT a link prop: + The button itself holds the text + ::after arrow. + Make it flex so the arrow never wraps to a new line. */ +.menu__link--sublist-caret { + display: flex !important; + align-items: center !important; + justify-content: space-between !important; + gap: 0.5rem; + padding-right: 0.625rem !important; +} + +.menu__link--sublist-caret::after { + content: '' !important; + display: block !important; + flex-shrink: 0 !important; + width: 1.25rem !important; + height: 1.25rem !important; + min-width: 1.25rem !important; + background: var(--ifm-menu-link-sublist-icon) center / 1.25rem 1.25rem no-repeat !important; + margin: 0 !important; +} + +/* 2. Categories WITH a link prop: + A separate + - - )} Chat diff --git a/ui/litellm-dashboard/src/app/page.tsx b/ui/litellm-dashboard/src/app/page.tsx index bddcb0ab591..5f2921203ff 100644 --- a/ui/litellm-dashboard/src/app/page.tsx +++ b/ui/litellm-dashboard/src/app/page.tsx @@ -547,7 +547,7 @@ function CreateKeyPageContent() { ) : page == "policies" ? ( ) : page == "agents" ? ( - + ) : page == "prompts" ? ( ) : page == "transform-request" ? ( diff --git a/ui/litellm-dashboard/src/components/AIHub/AgentHubTableColumns.test.tsx b/ui/litellm-dashboard/src/components/AIHub/AgentHubTableColumns.test.tsx new file mode 100644 index 00000000000..083e67c297a --- /dev/null +++ b/ui/litellm-dashboard/src/components/AIHub/AgentHubTableColumns.test.tsx @@ -0,0 +1,146 @@ +import { render, screen } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import { vi } from "vitest"; +import { flexRender, getCoreRowModel, useReactTable } from "@tanstack/react-table"; +import { getAgentHubTableColumns, AgentHubData } from "./AgentHubTableColumns"; + +const mockAgent: AgentHubData = { + agent_id: "agent-1", + protocolVersion: "1.0", + name: "Test Agent", + description: "A test agent for unit testing", + url: "https://agent.example.com", + version: "2.0", + capabilities: { streaming: true, caching: false }, + defaultInputModes: ["text"], + defaultOutputModes: ["text", "image"], + skills: [ + { id: "s1", name: "Skill One", description: "First skill" }, + { id: "s2", name: "Skill Two", description: "Second skill" }, + { id: "s3", name: "Skill Three", description: "Third skill" }, + ], + is_public: true, +}; + +function TestTable({ + data, + publicPage = false, + showModal = vi.fn(), + copyToClipboard = vi.fn(), +}: { + data: AgentHubData[]; + publicPage?: boolean; + showModal?: ReturnType; + copyToClipboard?: ReturnType; +}) { + const columns = getAgentHubTableColumns(showModal, copyToClipboard, publicPage); + const table = useReactTable({ data, columns, getCoreRowModel: getCoreRowModel() }); + + return ( + + + {table.getHeaderGroups().map((hg) => ( + + {hg.headers.map((h) => ( + + ))} + + ))} + + + {table.getRowModel().rows.map((row) => ( + + {row.getVisibleCells().map((cell) => ( + + ))} + + ))} + +
{flexRender(h.column.columnDef.header, h.getContext())}
{flexRender(cell.column.columnDef.cell, cell.getContext())}
+ ); +} + +describe("AgentHubTableColumns", () => { + it("should render", () => { + render(); + expect(screen.getByText("Test Agent")).toBeInTheDocument(); + }); + + it("should display the agent description", () => { + render(); + // Description appears in both the description column and the mobile view within agent name column + expect(screen.getAllByText("A test agent for unit testing").length).toBeGreaterThanOrEqual(1); + }); + + it("should display the version with a 'v' prefix", () => { + render(); + expect(screen.getByText("v2.0")).toBeInTheDocument(); + }); + + it("should display the protocol version", () => { + render(); + expect(screen.getByText("1.0")).toBeInTheDocument(); + }); + + it("should show skill count with correct pluralization", () => { + render(); + expect(screen.getByText("3 skills")).toBeInTheDocument(); + }); + + it("should show first two skills and '+1' for overflow", () => { + render(); + expect(screen.getByText("Skill One")).toBeInTheDocument(); + expect(screen.getByText("Skill Two")).toBeInTheDocument(); + expect(screen.getByText("+1")).toBeInTheDocument(); + }); + + it("should show only true capabilities as badges", () => { + render(); + expect(screen.getByText("streaming")).toBeInTheDocument(); + expect(screen.queryByText("caching")).not.toBeInTheDocument(); + }); + + it("should display I/O modes", () => { + render(); + // "In:" and "Out:" are in children; getByText with exact:false + // matches against the element's full textContent across child nodes + expect(screen.getByText((_, el) => + el?.tagName === "P" && el.textContent === "In: text" + )).toBeInTheDocument(); + expect(screen.getByText((_, el) => + el?.tagName === "P" && el.textContent === "Out: text, image" + )).toBeInTheDocument(); + }); + + it("should display 'Yes' badge for public agents", () => { + render(); + expect(screen.getByText("Yes")).toBeInTheDocument(); + }); + + it("should display 'No' badge for non-public agents", () => { + const privateAgent = { ...mockAgent, is_public: false }; + render(); + expect(screen.getByText("No")).toBeInTheDocument(); + }); + + it("should display a Details button", () => { + render(); + expect(screen.getByRole("button", { name: /details|info/i })).toBeInTheDocument(); + }); + + it("should show '-' when agent has no capabilities", () => { + const noCapAgent = { ...mockAgent, capabilities: {} }; + render(); + // The dash is rendered in the capabilities column + expect(screen.getByText("-")).toBeInTheDocument(); + }); + + it("should show singular 'skill' for one skill", () => { + const oneSkillAgent = { + ...mockAgent, + skills: [{ id: "s1", name: "Only Skill", description: "One" }], + }; + render(); + expect(screen.getByText("1 skill")).toBeInTheDocument(); + }); +}); diff --git a/ui/litellm-dashboard/src/components/AIHub/AgentHubTableColumns.tsx b/ui/litellm-dashboard/src/components/AIHub/AgentHubTableColumns.tsx index c6a8c0b9daa..09b1c147615 100644 --- a/ui/litellm-dashboard/src/components/AIHub/AgentHubTableColumns.tsx +++ b/ui/litellm-dashboard/src/components/AIHub/AgentHubTableColumns.tsx @@ -194,7 +194,6 @@ export const getAgentHubTableColumns = ( return publicA - publicB; }, cell: ({ row }) => { - console.log(`CHECKPOINT 1: ${JSON.stringify(row.original)}`); const agent = row.original; return agent.is_public === true ? ( diff --git a/ui/litellm-dashboard/src/components/DebugWarningBanner.test.tsx b/ui/litellm-dashboard/src/components/DebugWarningBanner.test.tsx new file mode 100644 index 00000000000..2412a84da3e --- /dev/null +++ b/ui/litellm-dashboard/src/components/DebugWarningBanner.test.tsx @@ -0,0 +1,41 @@ +import { renderWithProviders, screen } from "../../tests/test-utils"; +import { vi } from "vitest"; +import { DebugWarningBanner } from "./DebugWarningBanner"; + +vi.mock("@/app/(dashboard)/hooks/healthReadiness/useHealthReadiness", () => ({ + useHealthReadiness: vi.fn(), +})); + +import { useHealthReadiness } from "@/app/(dashboard)/hooks/healthReadiness/useHealthReadiness"; + +describe("DebugWarningBanner", () => { + it("should render", () => { + vi.mocked(useHealthReadiness).mockReturnValue({ data: { is_detailed_debug: true } } as any); + renderWithProviders(); + expect(screen.getByRole("alert")).toBeInTheDocument(); + }); + + it("should show warning when detailed debug mode is active", () => { + vi.mocked(useHealthReadiness).mockReturnValue({ data: { is_detailed_debug: true } } as any); + renderWithProviders(); + expect(screen.getByText(/Performance Warning: Detailed Debug Mode Active/i)).toBeInTheDocument(); + }); + + it("should mention LITELLM_LOG=DEBUG in the description", () => { + vi.mocked(useHealthReadiness).mockReturnValue({ data: { is_detailed_debug: true } } as any); + renderWithProviders(); + expect(screen.getByText("LITELLM_LOG=DEBUG")).toBeInTheDocument(); + }); + + it("should render nothing when is_detailed_debug is false", () => { + vi.mocked(useHealthReadiness).mockReturnValue({ data: { is_detailed_debug: false } } as any); + const { container } = renderWithProviders(); + expect(container).toBeEmptyDOMElement(); + }); + + it("should render nothing when health data is undefined", () => { + vi.mocked(useHealthReadiness).mockReturnValue({ data: undefined } as any); + const { container } = renderWithProviders(); + expect(container).toBeEmptyDOMElement(); + }); +}); diff --git a/ui/litellm-dashboard/src/components/DefaultUserSettings.tsx b/ui/litellm-dashboard/src/components/DefaultUserSettings.tsx index 988a3bcec92..314946c520b 100644 --- a/ui/litellm-dashboard/src/components/DefaultUserSettings.tsx +++ b/ui/litellm-dashboard/src/components/DefaultUserSettings.tsx @@ -1,6 +1,6 @@ import React, { useState, useEffect } from "react"; -import { Card, Title, Text, Divider, Button, TextInput } from "@tremor/react"; -import { Typography, Spin, Switch, Select, InputNumber } from "antd"; +import { Card, Title, Text, Divider, TextInput } from "@tremor/react"; +import { Button, Typography, Spin, Switch, Select, InputNumber } from "antd"; import { PlusOutlined, DeleteOutlined } from "@ant-design/icons"; import { getInternalUserSettings, updateInternalUserSettings, modelAvailableCall } from "./networking"; import BudgetDurationDropdown, { getBudgetDurationLabel } from "./common_components/budget_duration_dropdown"; @@ -160,11 +160,10 @@ const DefaultUserSettings: React.FC = ({
Team {index + 1} @@ -208,7 +207,7 @@ const DefaultUserSettings: React.FC = ({
))} - @@ -462,7 +461,6 @@ const DefaultUserSettings: React.FC = ({ (isEditing ? (
-
) : ( - + ))} diff --git a/ui/litellm-dashboard/src/components/EntityUsageExport/ExportFormatSelector.test.tsx b/ui/litellm-dashboard/src/components/EntityUsageExport/ExportFormatSelector.test.tsx new file mode 100644 index 00000000000..8447df4a6b9 --- /dev/null +++ b/ui/litellm-dashboard/src/components/EntityUsageExport/ExportFormatSelector.test.tsx @@ -0,0 +1,20 @@ +import { renderWithProviders, screen } from "../../../tests/test-utils"; +import { vi } from "vitest"; +import ExportFormatSelector from "./ExportFormatSelector"; + +describe("ExportFormatSelector", () => { + it("should render", () => { + renderWithProviders(); + expect(screen.getByText("Format")).toBeInTheDocument(); + }); + + it("should display the current value", () => { + renderWithProviders(); + expect(screen.getByText("CSV (Excel, Google Sheets)")).toBeInTheDocument(); + }); + + it("should display JSON label when json is selected", () => { + renderWithProviders(); + expect(screen.getByText("JSON (includes metadata)")).toBeInTheDocument(); + }); +}); diff --git a/ui/litellm-dashboard/src/components/EntityUsageExport/ExportSummary.test.tsx b/ui/litellm-dashboard/src/components/EntityUsageExport/ExportSummary.test.tsx new file mode 100644 index 00000000000..8ebeb33b1eb --- /dev/null +++ b/ui/litellm-dashboard/src/components/EntityUsageExport/ExportSummary.test.tsx @@ -0,0 +1,59 @@ +import { renderWithProviders, screen } from "../../../tests/test-utils"; +import ExportSummary from "./ExportSummary"; + +describe("ExportSummary", () => { + it("should render", () => { + const dateRange = { + from: new Date("2024-01-01"), + to: new Date("2024-01-31"), + }; + const { container } = renderWithProviders( + + ); + expect(container).not.toBeEmptyDOMElement(); + }); + + it("should display the date range", () => { + const from = new Date(2024, 0, 1); + const to = new Date(2024, 0, 31); + const dateRange = { from, to }; + renderWithProviders( + + ); + expect(screen.getByText(new RegExp(from.toLocaleDateString()))).toBeInTheDocument(); + expect(screen.getByText(new RegExp(to.toLocaleDateString()))).toBeInTheDocument(); + }); + + it("should show filter count when filters are selected", () => { + const dateRange = { + from: new Date("2024-01-01"), + to: new Date("2024-01-31"), + }; + renderWithProviders( + + ); + expect(screen.getByText(/3 filters/)).toBeInTheDocument(); + }); + + it("should show singular 'filter' for one filter", () => { + const dateRange = { + from: new Date("2024-01-01"), + to: new Date("2024-01-31"), + }; + renderWithProviders( + + ); + expect(screen.getByText(/1 filter$/)).toBeInTheDocument(); + }); + + it("should not show filter count when no filters selected", () => { + const dateRange = { + from: new Date("2024-01-01"), + to: new Date("2024-01-31"), + }; + renderWithProviders( + + ); + expect(screen.queryByText(/filter/)).not.toBeInTheDocument(); + }); +}); diff --git a/ui/litellm-dashboard/src/components/EntityUsageExport/ExportTypeSelector.test.tsx b/ui/litellm-dashboard/src/components/EntityUsageExport/ExportTypeSelector.test.tsx new file mode 100644 index 00000000000..36d2e12b717 --- /dev/null +++ b/ui/litellm-dashboard/src/components/EntityUsageExport/ExportTypeSelector.test.tsx @@ -0,0 +1,46 @@ +import { renderWithProviders, screen } from "../../../tests/test-utils"; +import userEvent from "@testing-library/user-event"; +import { vi } from "vitest"; +import ExportTypeSelector from "./ExportTypeSelector"; + +describe("ExportTypeSelector", () => { + it("should render", () => { + renderWithProviders( + + ); + expect(screen.getByText("Export type")).toBeInTheDocument(); + }); + + it("should display entity type in radio labels", () => { + renderWithProviders( + + ); + expect(screen.getByText(/Day-by-day breakdown by team$/)).toBeInTheDocument(); + expect(screen.getByText(/Day-by-day breakdown by team and key/)).toBeInTheDocument(); + expect(screen.getByText(/Day-by-day by team and model/)).toBeInTheDocument(); + }); + + it("should display the correct entity type for different entities", () => { + renderWithProviders( + + ); + expect(screen.getByText(/Day-by-day breakdown by organization$/)).toBeInTheDocument(); + }); + + it("should call onChange when a radio option is selected", async () => { + const user = userEvent.setup(); + const onChange = vi.fn(); + renderWithProviders( + + ); + await user.click(screen.getByRole("radio", { name: /Day-by-day breakdown by team and key/i })); + expect(onChange).toHaveBeenCalledWith("daily_with_keys"); + }); + + it("should have the correct radio checked", () => { + renderWithProviders( + + ); + expect(screen.getByRole("radio", { name: /Day-by-day by team and model/i })).toBeChecked(); + }); +}); diff --git a/ui/litellm-dashboard/src/components/EntityUsageExport/UsageExportHeader.test.tsx b/ui/litellm-dashboard/src/components/EntityUsageExport/UsageExportHeader.test.tsx new file mode 100644 index 00000000000..729d6fd3406 --- /dev/null +++ b/ui/litellm-dashboard/src/components/EntityUsageExport/UsageExportHeader.test.tsx @@ -0,0 +1,73 @@ +import { renderWithProviders, screen } from "../../../tests/test-utils"; +import userEvent from "@testing-library/user-event"; +import { vi } from "vitest"; +import UsageExportHeader from "./UsageExportHeader"; +import type { EntitySpendData } from "./types"; + +vi.mock("./EntityUsageExportModal", () => ({ + default: ({ isOpen, onClose }: { isOpen: boolean; onClose: () => void }) => + isOpen ? ( +
+ +
+ ) : null, +})); + +const defaultProps = { + dateValue: { from: new Date("2025-01-01"), to: new Date("2025-01-31") }, + entityType: "team" as const, + spendData: { + results: [], + metadata: { + total_spend: 0, + total_api_requests: 0, + total_successful_requests: 0, + total_failed_requests: 0, + total_tokens: 0, + }, + } satisfies EntitySpendData, +}; + +describe("UsageExportHeader", () => { + it("should render", () => { + renderWithProviders(); + expect(screen.getByRole("button", { name: /export data/i })).toBeInTheDocument(); + }); + + it("should open the export modal when the export button is clicked", async () => { + const user = userEvent.setup(); + renderWithProviders(); + await user.click(screen.getByRole("button", { name: /export data/i })); + expect(screen.getByTestId("export-modal")).toBeInTheDocument(); + }); + + it("should close the export modal when onClose is called", async () => { + const user = userEvent.setup(); + renderWithProviders(); + await user.click(screen.getByRole("button", { name: /export data/i })); + await user.click(screen.getByRole("button", { name: /close/i })); + expect(screen.queryByTestId("export-modal")).not.toBeInTheDocument(); + }); + + it("should not show filter dropdown when showFilters is false", () => { + renderWithProviders(); + expect(screen.queryByText(/filter/i)).not.toBeInTheDocument(); + }); + + it("should show filter dropdown when showFilters is true and options provided", () => { + renderWithProviders( + , + ); + expect(screen.getByText("Team")).toBeInTheDocument(); + }); +}); diff --git a/ui/litellm-dashboard/src/components/EntityUsageExport/utils.test.ts b/ui/litellm-dashboard/src/components/EntityUsageExport/utils.test.ts index 67de4b69174..2ca7ad7ef31 100644 --- a/ui/litellm-dashboard/src/components/EntityUsageExport/utils.test.ts +++ b/ui/litellm-dashboard/src/components/EntityUsageExport/utils.test.ts @@ -11,6 +11,7 @@ import { getEntityBreakdown, handleExportCSV, handleExportJSON, + resolveEntities, } from "./utils"; vi.mock("@/utils/dataUtils", () => ({ @@ -1561,4 +1562,137 @@ describe("EntityUsageExport utils", () => { window.Blob = originalBlob; }); }); + + describe("resolveEntities and aggregated endpoint fallback", () => { + // Simulates the response from /user/daily/activity/aggregated which has + // empty entities but populated api_keys at the breakdown level. + // Derived from mockSpendData: flatten all entities' api_key_breakdowns + // into top-level api_keys, clear entities, and add a second key for team-1 + // to test multi-key grouping. + const aggregatedSpendData: EntitySpendData = { + ...mockSpendData, + results: mockSpendData.results.slice(0, 1).map((day) => ({ + ...day, + breakdown: { + entities: {}, + api_keys: { + ...Object.fromEntries( + Object.values(day.breakdown.entities as Record).flatMap((e: any) => + Object.entries(e.api_key_breakdown || {}), + ), + ), + // Extra key on team-1 to test multi-key-per-team aggregation + key1b: { + metrics: { spend: 5, api_requests: 50, successful_requests: 48, failed_requests: 2, total_tokens: 500 }, + metadata: { team_id: "team-1", key_alias: "staging-key" }, + }, + }, + models: { "gpt-4": { metrics: { spend: 35, api_requests: 350, total_tokens: 3500 } } }, + }, + })), + }; + + describe("resolveEntities", () => { + it("should return entities when populated", () => { + const breakdown = { + entities: { e1: { metrics: { spend: 1 } } }, + api_keys: { k1: { metrics: { spend: 2 }, metadata: { team_id: "t1" } } }, + }; + const result = resolveEntities(breakdown); + expect(result).toBe(breakdown.entities); + }); + + it("should aggregate api_keys into entities when entities is empty", () => { + const breakdown = aggregatedSpendData.results[0].breakdown; + const result = resolveEntities(breakdown); + + // Two teams: team-1 (key1+key2) and team-2 (key3) + expect(Object.keys(result)).toHaveLength(2); + expect(result["team-1"]).toBeDefined(); + expect(result["team-2"]).toBeDefined(); + + // team-1 spend = 10.5 (key1) + 5 (key1b) + expect(result["team-1"].metrics.spend).toBe(15.5); + expect(result["team-1"].metrics.api_requests).toBe(150); + expect(result["team-1"].metrics.total_tokens).toBe(1500); + + // team-2 spend = 20.3 (key2) + expect(result["team-2"].metrics.spend).toBe(20.3); + expect(result["team-2"].metrics.api_requests).toBe(200); + }); + + it("should use 'Unassigned' for keys without team_id", () => { + const breakdown = { + entities: {}, + api_keys: { + k1: { + metrics: { spend: 7, api_requests: 10, successful_requests: 10, failed_requests: 0, total_tokens: 100 }, + metadata: {}, + }, + }, + }; + const result = resolveEntities(breakdown); + expect(result["Unassigned"]).toBeDefined(); + expect(result["Unassigned"].metrics.spend).toBe(7); + }); + + it("should handle missing or empty api_keys gracefully", () => { + expect(Object.keys(resolveEntities({ entities: {}, api_keys: {} }))).toHaveLength(0); + expect(Object.keys(resolveEntities({ entities: {} }))).toHaveLength(0); + }); + + it("should preserve api_key_breakdown on aggregated entities", () => { + const breakdown = aggregatedSpendData.results[0].breakdown; + const result = resolveEntities(breakdown); + + // team-1 should have key1 and key1b in api_key_breakdown + expect(Object.keys(result["team-1"].api_key_breakdown)).toEqual(["key1", "key1b"]); + // team-2 should have key2 + expect(Object.keys(result["team-2"].api_key_breakdown)).toEqual(["key2"]); + }); + }); + + describe("getEntityBreakdown with aggregated data", () => { + it("should produce breakdown from api_keys when entities is empty", () => { + const result = getEntityBreakdown(aggregatedSpendData); + expect(result.length).toBeGreaterThan(0); + + // Sorted by spend desc: team-2 (20.3) then team-1 (15.5) + expect(result[0].metrics.spend).toBe(20.3); + expect(result[1].metrics.spend).toBe(15.5); + }); + + }); + + describe("generateDailyData with aggregated data", () => { + it("should produce rows from api_keys when entities is empty", () => { + const result = generateDailyData(aggregatedSpendData, "Team"); + expect(result.length).toBeGreaterThan(0); + expect(result[0]).toHaveProperty("Date"); + expect(result[0]).toHaveProperty("Team"); + }); + }); + + describe("generateDailyWithKeysData with aggregated data", () => { + it("should produce rows from api_keys when entities is empty", () => { + const result = generateDailyWithKeysData(aggregatedSpendData, "Team"); + expect(result.length).toBeGreaterThan(0); + + // Should have 3 key rows (key1, key1b, key2) + expect(result).toHaveLength(3); + const keyIds = result.map((r) => r["Key ID"]); + expect(keyIds).toContain("key1"); + expect(keyIds).toContain("key1b"); + expect(keyIds).toContain("key2"); + }); + }); + + describe("generateDailyWithModelsData with aggregated data", () => { + it("should produce rows from api_keys when entities is empty", () => { + const result = generateDailyWithModelsData(aggregatedSpendData, "Team"); + expect(result.length).toBeGreaterThan(0); + expect(result[0]).toHaveProperty("Model"); + }); + }); + }); }); diff --git a/ui/litellm-dashboard/src/components/EntityUsageExport/utils.ts b/ui/litellm-dashboard/src/components/EntityUsageExport/utils.ts index ebef3da8a77..45bf21a6e7d 100644 --- a/ui/litellm-dashboard/src/components/EntityUsageExport/utils.ts +++ b/ui/litellm-dashboard/src/components/EntityUsageExport/utils.ts @@ -17,6 +17,49 @@ const extractTeamIdFromApiKeyBreakdown = (apiKeyBreakdown: Record | return null; }; +// Mirrors backend SpendMetrics fields (litellm/types/activity_tracking.py). +// If the backend adds a field, add it here too. +const METRIC_KEYS = [ + "spend", "api_requests", "successful_requests", "failed_requests", + "total_tokens", "prompt_tokens", "completion_tokens", + "cache_read_input_tokens", "cache_creation_input_tokens", +] as const; + +// When breakdown.entities is empty (aggregated endpoint), reconstruct entities +// from breakdown.api_keys by grouping on metadata.team_id. +const aggregateApiKeysIntoEntities = (breakdown: Record): Record => { + const apiKeys = breakdown.api_keys; + if (!apiKeys || Object.keys(apiKeys).length === 0) return {}; + + const grouped: Record = {}; + + for (const [keyId, keyData] of Object.entries(apiKeys)) { + const teamId = keyData?.metadata?.team_id || "Unassigned"; + if (!grouped[teamId]) { + grouped[teamId] = { + metrics: Object.fromEntries(METRIC_KEYS.map((k) => [k, 0])), + api_key_breakdown: {}, + }; + } + const m = grouped[teamId].metrics; + const km = keyData?.metrics || {}; + for (const k of METRIC_KEYS) { + m[k] += km[k] || 0; + } + grouped[teamId].api_key_breakdown[keyId] = keyData; + } + + return grouped; +}; + +// Returns breakdown.entities if populated, otherwise falls back to +// reconstructing entities from breakdown.api_keys. +export const resolveEntities = (breakdown: Record): Record => { + const entities = breakdown.entities; + if (entities && Object.keys(entities).length > 0) return entities; + return aggregateApiKeysIntoEntities(breakdown); +}; + export const getEntityBreakdown = ( spendData: EntitySpendData, teamAliasMap: Record = {}, @@ -24,7 +67,7 @@ export const getEntityBreakdown = ( const entitySpend: { [key: string]: EntityBreakdown } = {}; spendData.results.forEach((day) => { - Object.entries(day.breakdown.entities || {}).forEach(([entity, data]: [string, any]) => { + Object.entries(resolveEntities(day.breakdown)).forEach(([entity, data]: [string, any]) => { // Extract team_id from api_key_breakdown metadata (not data.metadata which is empty) const teamId = extractTeamIdFromApiKeyBreakdown(data.api_key_breakdown) || entity; // Extract key_alias from the first API key that has one @@ -80,7 +123,7 @@ export const generateDailyData = ( const dailyBreakdown: any[] = []; spendData.results.forEach((day) => { - Object.entries(day.breakdown.entities || {}).forEach(([entity, data]: [string, any]) => { + Object.entries(resolveEntities(day.breakdown)).forEach(([entity, data]: [string, any]) => { // Extract team_id from api_key_breakdown metadata (not data.metadata which is empty) const teamId = extractTeamIdFromApiKeyBreakdown(data.api_key_breakdown); const teamAlias = teamId ? teamAliasMap[teamId] || null : null; @@ -129,7 +172,7 @@ export const generateDailyWithKeysData = ( } = {}; spendData.results.forEach((day) => { - Object.entries(day.breakdown.entities || {}).forEach(([entity, data]: [string, any]) => { + Object.entries(resolveEntities(day.breakdown)).forEach(([entity, data]: [string, any]) => { const apiKeyBreakdown = data.api_key_breakdown || {}; // Iterate through each API key in the breakdown @@ -202,7 +245,7 @@ export const generateDailyWithModelsData = ( spendData.results.forEach((day) => { const dailyEntityModels: { [key: string]: { [key: string]: any } } = {}; - Object.entries(day.breakdown.entities || {}).forEach(([entity, entityData]: [string, any]) => { + Object.entries(resolveEntities(day.breakdown)).forEach(([entity, entityData]: [string, any]) => { if (!dailyEntityModels[entity]) { dailyEntityModels[entity] = {}; } @@ -230,7 +273,7 @@ export const generateDailyWithModelsData = ( }); Object.entries(dailyEntityModels).forEach(([entity, models]) => { - const entityData = day.breakdown.entities?.[entity]; + const entityData = resolveEntities(day.breakdown)[entity]; // Extract team_id from api_key_breakdown metadata (not entityData.metadata which is empty) const teamId = extractTeamIdFromApiKeyBreakdown(entityData?.api_key_breakdown); const teamAlias = teamId ? teamAliasMap[teamId] || null : null; diff --git a/ui/litellm-dashboard/src/components/GuardrailsMonitor/GuardrailConfig.test.tsx b/ui/litellm-dashboard/src/components/GuardrailsMonitor/GuardrailConfig.test.tsx new file mode 100644 index 00000000000..54c7ebabe77 --- /dev/null +++ b/ui/litellm-dashboard/src/components/GuardrailsMonitor/GuardrailConfig.test.tsx @@ -0,0 +1,98 @@ +import { render, screen, act } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import { vi } from "vitest"; +import { GuardrailConfig } from "./GuardrailConfig"; + +describe("GuardrailConfig", () => { + const defaultProps = { + guardrailName: "Content Safety", + guardrailType: "Content Safety", + provider: "bedrock", + }; + + afterEach(() => { + vi.useRealTimers(); + }); + + it("should render", () => { + render(); + expect(screen.getByText("Parameters")).toBeInTheDocument(); + }); + + it("should display the guardrail name in the parameters description", () => { + render(); + expect(screen.getByText(/Configure Content Safety behavior/)).toBeInTheDocument(); + }); + + // Note: Version history entries are hardcoded placeholders in the component. + // These assertions will need updating when wired to real API data. + it("should show version history when 'View history' is clicked", async () => { + const user = userEvent.setup(); + render(); + await user.click(screen.getByRole("button", { name: /view history/i })); + expect(screen.getByText("Initial configuration")).toBeInTheDocument(); + expect(screen.getByText("Added custom categories list")).toBeInTheDocument(); + }); + + it("should toggle version history text between View/Hide", async () => { + const user = userEvent.setup(); + render(); + const button = screen.getByRole("button", { name: /view history/i }); + await user.click(button); + expect(screen.getByRole("button", { name: /hide history/i })).toBeInTheDocument(); + }); + + it("should show custom code textarea when custom code override is toggled on", async () => { + const user = userEvent.setup(); + render(); + // Walk up from "Custom Code Override" heading to find the enclosing section, + // then locate the switch within it + const heading = screen.getByText("Custom Code Override"); + let container = heading.parentElement; + let customCodeSwitch: Element | null = null; + while (container && !customCodeSwitch) { + customCodeSwitch = container.querySelector('[role="switch"]'); + container = container.parentElement; + } + if (!customCodeSwitch) { + throw new Error("Could not find the Custom Code Override switch via DOM traversal"); + } + await user.click(customCodeSwitch); + expect(screen.getByPlaceholderText(/async def evaluate/)).toBeInTheDocument(); + }); + + it("should hide custom code textarea when custom code override is off", () => { + render(); + // There's an input for categories, but no textarea + expect(screen.queryByPlaceholderText(/async def evaluate/)).not.toBeInTheDocument(); + }); + + it("should show the re-run button in idle state", () => { + render(); + expect(screen.getByRole("button", { name: /re-run on failing logs/i })).toBeInTheDocument(); + }); + + it("should show loading state when re-run is clicked", async () => { + vi.useFakeTimers({ shouldAdvanceTime: true }); + const user = userEvent.setup({ advanceTimers: vi.advanceTimersByTime }); + render(); + await user.click(screen.getByRole("button", { name: /re-run on failing logs/i })); + expect(screen.getByText(/Running on 10 samples/)).toBeInTheDocument(); + }); + + it("should show success message after re-run completes", async () => { + vi.useFakeTimers({ shouldAdvanceTime: true }); + const user = userEvent.setup({ advanceTimers: vi.advanceTimersByTime }); + render(); + await user.click(screen.getByRole("button", { name: /re-run on failing logs/i })); + await act(async () => { vi.advanceTimersByTime(2500); }); + expect(screen.getByText(/7\/10 would now pass/)).toBeInTheDocument(); + }); + + it("should display the Revert and Save buttons", () => { + render(); + expect(screen.getByRole("button", { name: /revert/i })).toBeInTheDocument(); + // The component's hardcoded default version is "v3", so Save shows "v4" + expect(screen.getByRole("button", { name: /save as v\d+/i })).toBeInTheDocument(); + }); +}); diff --git a/ui/litellm-dashboard/src/components/GuardrailsMonitor/MetricCard.test.tsx b/ui/litellm-dashboard/src/components/GuardrailsMonitor/MetricCard.test.tsx new file mode 100644 index 00000000000..d30df1a0062 --- /dev/null +++ b/ui/litellm-dashboard/src/components/GuardrailsMonitor/MetricCard.test.tsx @@ -0,0 +1,49 @@ +import { renderWithProviders, screen } from "../../../tests/test-utils"; +import React from "react"; +import { MetricCard } from "./MetricCard"; + +describe("MetricCard", () => { + it("should render", () => { + renderWithProviders(); + expect(screen.getByText("Total Requests")).toBeInTheDocument(); + }); + + it("should display the label and value", () => { + renderWithProviders(); + expect(screen.getByText("Success Rate")).toBeInTheDocument(); + expect(screen.getByText("98.5%")).toBeInTheDocument(); + }); + + it("should display numeric values", () => { + renderWithProviders(); + expect(screen.getByText("42")).toBeInTheDocument(); + }); + + it("should render icon when provided", () => { + renderWithProviders( + icon} + /> + ); + expect(screen.getByTestId("test-icon")).toBeInTheDocument(); + }); + + it("should not render icon container when no icon provided", () => { + renderWithProviders(); + expect(screen.queryByTestId("test-icon")).not.toBeInTheDocument(); + }); + + it("should render subtitle when provided", () => { + renderWithProviders( + + ); + expect(screen.getByText("Last 24 hours")).toBeInTheDocument(); + }); + + it("should not render subtitle when not provided", () => { + renderWithProviders(); + expect(screen.queryByText("Last 24 hours")).not.toBeInTheDocument(); + }); +}); diff --git a/ui/litellm-dashboard/src/components/GuardrailsMonitor/ScoreChart.test.tsx b/ui/litellm-dashboard/src/components/GuardrailsMonitor/ScoreChart.test.tsx new file mode 100644 index 00000000000..848807ce5a0 --- /dev/null +++ b/ui/litellm-dashboard/src/components/GuardrailsMonitor/ScoreChart.test.tsx @@ -0,0 +1,54 @@ +import React from "react"; +import { describe, it, expect, vi } from "vitest"; +import { screen } from "@testing-library/react"; +import { renderWithProviders } from "../../../tests/test-utils"; +import { ScoreChart } from "./ScoreChart"; + +vi.mock("@tremor/react", async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + BarChart: ({ data, categories }: { data: any[]; categories: string[] }) => ( +
+ {data.map((d, i) => ( + + {d.date}: {categories.map((c) => `${c}=${d[c]}`).join(", ")} + + ))} +
+ ), + }; +}); + +describe("ScoreChart", () => { + it("should render the title", () => { + renderWithProviders(); + + expect(screen.getByText("Request Outcomes Over Time")).toBeInTheDocument(); + }); + + it("should show empty state when no data is provided", () => { + renderWithProviders(); + + expect(screen.getByText("No chart data for this period")).toBeInTheDocument(); + }); + + it("should show empty state when data is an empty array", () => { + renderWithProviders(); + + expect(screen.getByText("No chart data for this period")).toBeInTheDocument(); + }); + + it("should render the chart when data is provided", () => { + const data = [ + { date: "2026-03-01", passed: 10, blocked: 2 }, + { date: "2026-03-02", passed: 15, blocked: 1 }, + ]; + + renderWithProviders(); + + expect(screen.queryByText("No chart data for this period")).not.toBeInTheDocument(); + expect(screen.getByText(/2026-03-01/)).toBeInTheDocument(); + expect(screen.getByText(/2026-03-02/)).toBeInTheDocument(); + }); +}); diff --git a/ui/litellm-dashboard/src/components/HelpLink.test.tsx b/ui/litellm-dashboard/src/components/HelpLink.test.tsx new file mode 100644 index 00000000000..a35a09b9c3f --- /dev/null +++ b/ui/litellm-dashboard/src/components/HelpLink.test.tsx @@ -0,0 +1,155 @@ +import React from "react"; +import { describe, it, expect, vi } from "vitest"; +import { screen } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import { renderWithProviders } from "../../tests/test-utils"; +import { HelpLink, HelpIcon, DocsMenu } from "./HelpLink"; + +describe("HelpLink", () => { + it("should render with default children and open in new tab", () => { + renderWithProviders(); + + const link = screen.getByRole("link", { name: /learn more/i }); + expect(link).toHaveAttribute("href", "https://docs.example.com"); + expect(link).toHaveAttribute("target", "_blank"); + expect(link).toHaveAttribute("rel", "noopener noreferrer"); + }); + + it("should render custom children text", () => { + renderWithProviders( + Custom docs link + ); + + expect(screen.getByText("Custom docs link")).toBeInTheDocument(); + }); + + it("should have the correct href", () => { + renderWithProviders(); + expect(screen.getByRole("link")).toHaveAttribute("href", "https://docs.example.com/test"); + }); + + it("should include a screen-reader-only label for accessibility", () => { + renderWithProviders(); + + expect(screen.getByText("(opens in a new tab)")).toBeInTheDocument(); + }); +}); + +describe("HelpIcon", () => { + it("should render a help button with accessible label", () => { + renderWithProviders(); + + expect(screen.getByRole("button", { name: /help information/i })).toBeInTheDocument(); + }); + + it("should show tooltip content on hover", async () => { + const user = userEvent.setup(); + renderWithProviders(); + + await user.hover(screen.getByRole("button", { name: /help information/i })); + + expect(screen.getByText("Tooltip help text")).toBeInTheDocument(); + }); + + it("should hide tooltip content when not hovered", () => { + renderWithProviders(); + expect(screen.queryByText("Hidden tooltip")).not.toBeInTheDocument(); + }); + + it("should show learn more link when learnMoreHref is provided", async () => { + const user = userEvent.setup(); + renderWithProviders( + + ); + await user.hover(screen.getByRole("button", { name: /help information/i })); + expect(screen.getByText("Learn more")).toBeInTheDocument(); + }); + + it("should use custom learn more text when provided", async () => { + const user = userEvent.setup(); + renderWithProviders( + + ); + + await user.hover(screen.getByRole("button", { name: /help information/i })); + + const link = screen.getByRole("link", { name: /read docs/i }); + expect(link).toHaveAttribute("href", "https://docs.example.com"); + }); + + it("should not show learn more link when learnMoreHref is not provided", async () => { + const user = userEvent.setup(); + renderWithProviders(); + + await user.hover(screen.getByRole("button", { name: /help information/i })); + + expect(screen.queryByRole("link")).not.toBeInTheDocument(); + }); +}); + +describe("DocsMenu", () => { + const items = [ + { label: "Custom pricing", href: "https://docs.example.com/pricing" }, + { label: "Cost tracking", href: "https://docs.example.com/cost" }, + ]; + + it("should render the menu button with default text", () => { + renderWithProviders(); + + expect(screen.getByRole("button", { name: /docs/i })).toBeInTheDocument(); + }); + + it("should hide menu items initially", () => { + renderWithProviders(); + expect(screen.queryByText("Custom pricing")).not.toBeInTheDocument(); + }); + + it("should show menu items when button is clicked", async () => { + const user = userEvent.setup(); + renderWithProviders(); + + await user.click(screen.getByRole("button", { name: /docs/i })); + + expect(screen.getByText("Custom pricing")).toBeInTheDocument(); + expect(screen.getByText("Cost tracking")).toBeInTheDocument(); + }); + + it("should close the menu when an item is clicked", async () => { + const user = userEvent.setup(); + renderWithProviders(); + + await user.click(screen.getByRole("button", { name: /docs/i })); + await user.click(screen.getByText("Custom pricing")); + + expect(screen.queryByText("Cost tracking")).not.toBeInTheDocument(); + }); + + it("should set aria-expanded correctly based on menu state", async () => { + const user = userEvent.setup(); + renderWithProviders(); + + const button = screen.getByRole("button", { name: /docs/i }); + expect(button).toHaveAttribute("aria-expanded", "false"); + + await user.click(button); + expect(button).toHaveAttribute("aria-expanded", "true"); + }); + + it("should close menu when clicking outside", async () => { + const user = userEvent.setup(); + renderWithProviders( +
+ + +
, + ); + await user.click(screen.getByRole("button", { name: /docs/i })); + expect(screen.getByText("Custom pricing")).toBeInTheDocument(); + await user.click(screen.getByRole("button", { name: /outside/i })); + expect(screen.queryByText("Custom pricing")).not.toBeInTheDocument(); + }); +}); diff --git a/ui/litellm-dashboard/src/components/Settings/AdminSettings/HashicorpVault/HashicorpVaultEmptyPlaceholder.test.tsx b/ui/litellm-dashboard/src/components/Settings/AdminSettings/HashicorpVault/HashicorpVaultEmptyPlaceholder.test.tsx new file mode 100644 index 00000000000..4214d5fda7c --- /dev/null +++ b/ui/litellm-dashboard/src/components/Settings/AdminSettings/HashicorpVault/HashicorpVaultEmptyPlaceholder.test.tsx @@ -0,0 +1,29 @@ +import { describe, it, expect, vi } from "vitest"; +import { render, screen } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import HashicorpVaultEmptyPlaceholder from "./HashicorpVaultEmptyPlaceholder"; + +describe("HashicorpVaultEmptyPlaceholder", () => { + it("should render the empty state message and configure button", () => { + render(); + expect(screen.getByText("No Vault Configuration Found")).toBeInTheDocument(); + expect(screen.getByRole("button", { name: /configure vault/i })).toBeInTheDocument(); + }); + + it("should call onAdd when the configure button is clicked", async () => { + const onAdd = vi.fn(); + const user = userEvent.setup(); + render(); + + await user.click(screen.getByRole("button", { name: /configure vault/i })); + + expect(onAdd).toHaveBeenCalledOnce(); + }); + + it("should display the description text about Vault purpose", () => { + render(); + expect( + screen.getByText(/Configure Hashicorp Vault to securely manage provider API keys/), + ).toBeInTheDocument(); + }); +}); diff --git a/ui/litellm-dashboard/src/components/Settings/AdminSettings/UISettings/PageVisibilitySettings.test.tsx b/ui/litellm-dashboard/src/components/Settings/AdminSettings/UISettings/PageVisibilitySettings.test.tsx new file mode 100644 index 00000000000..798572b2037 --- /dev/null +++ b/ui/litellm-dashboard/src/components/Settings/AdminSettings/UISettings/PageVisibilitySettings.test.tsx @@ -0,0 +1,77 @@ +import { describe, it, expect, vi } from "vitest"; +import { render, screen } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import PageVisibilitySettings from "./PageVisibilitySettings"; + +vi.mock("@/components/page_utils", () => ({ + getAvailablePages: () => [ + { page: "usage", label: "Usage", description: "View usage stats", group: "Analytics" }, + { page: "models", label: "Models", description: "Manage models", group: "Analytics" }, + { page: "keys", label: "API Keys", description: "Manage API keys", group: "Access" }, + ], +})); + +describe("PageVisibilitySettings", () => { + it("should render the not-set tag when enabledPagesInternalUsers is null", () => { + render( + , + ); + expect(screen.getByText("Not set (all pages visible)")).toBeInTheDocument(); + }); + + it("should show the selected page count tag when pages are configured", () => { + render( + , + ); + expect(screen.getByText("2 pages selected")).toBeInTheDocument(); + }); + + it("should show singular 'page' when exactly one page is selected", () => { + render( + , + ); + expect(screen.getByText("1 page selected")).toBeInTheDocument(); + }); + + it("should call onUpdate with null when reset button is clicked", async () => { + const onUpdate = vi.fn(); + const user = userEvent.setup(); + render( + , + ); + + // Expand the collapse panel first to reveal the reset button + await user.click(screen.getByRole("button", { name: /configure page visibility/i })); + await user.click(await screen.findByRole("button", { name: /reset to default/i })); + + expect(onUpdate).toHaveBeenCalledWith({ enabled_ui_pages_internal_users: null }); + }); + + it("should display the property description when provided", () => { + render( + , + ); + expect(screen.getByText("Controls which pages are visible")).toBeInTheDocument(); + }); +}); diff --git a/ui/litellm-dashboard/src/components/Settings/AdminSettings/UISettings/UISettings.tsx b/ui/litellm-dashboard/src/components/Settings/AdminSettings/UISettings/UISettings.tsx index a22c78c9430..fb7c38449b0 100644 --- a/ui/litellm-dashboard/src/components/Settings/AdminSettings/UISettings/UISettings.tsx +++ b/ui/litellm-dashboard/src/components/Settings/AdminSettings/UISettings/UISettings.tsx @@ -24,6 +24,7 @@ export default function UISettings() { const disableVectorStoresProperty = schema?.properties?.disable_vector_stores_for_internal_users; const allowVectorStoresTeamAdminsProperty = schema?.properties?.allow_vector_stores_for_team_admins; const scopeUserSearchProperty = schema?.properties?.scope_user_search_to_org; + const disableCustomApiKeysProperty = schema?.properties?.disable_custom_api_keys; const values = data?.values ?? {}; const isDisabledForInternalUsers = Boolean(values.disable_model_add_for_internal_users); const isDisabledTeamAdminDeleteTeamUser = Boolean(values.disable_team_admin_delete_team_user); @@ -182,6 +183,20 @@ export default function UISettings() { ); }; + const handleToggleDisableCustomApiKeys = (checked: boolean) => { + updateSettings( + { disable_custom_api_keys: checked }, + { + onSuccess: () => { + NotificationManager.success("UI settings updated successfully"); + }, + onError: (error) => { + NotificationManager.fromBackend(error); + }, + }, + ); + }; + return ( {isLoading ? ( @@ -382,6 +397,26 @@ export default function UISettings() { + {/* Disable custom Virtual key values */} + + + + Disable custom Virtual key values + + {disableCustomApiKeysProperty?.description ?? + "If true, users cannot specify custom key values. All keys must be auto-generated."} + + + + + + {/* Page Visibility for Internal Users */} trigger.parentElement || document.body} filterOption={(input, option) => (option?.label ?? "").toLowerCase().includes(input.toLowerCase()) } @@ -125,6 +126,7 @@ export function FallbackGroupConfig({ value={group.fallbackModels} onChange={handleFallbackSelect} disabled={!group.primaryModel} + getPopupContainer={(trigger) => trigger.parentElement || document.body} options={availableFallbackOptions.map((m) => ({ label: m, value: m, diff --git a/ui/litellm-dashboard/src/components/TeamSSOSettings.test.tsx b/ui/litellm-dashboard/src/components/TeamSSOSettings.test.tsx index ae93b118799..5006afb61e2 100644 --- a/ui/litellm-dashboard/src/components/TeamSSOSettings.test.tsx +++ b/ui/litellm-dashboard/src/components/TeamSSOSettings.test.tsx @@ -18,14 +18,18 @@ vi.mock("./common_components/budget_duration_dropdown", () => { aria-label="Budget duration" > - - + + + ); BudgetDurationDropdown.displayName = "BudgetDurationDropdown"; return { default: BudgetDurationDropdown, - getBudgetDurationLabel: vi.fn((value: string) => `Budget: ${value}`), + getBudgetDurationLabel: vi.fn((value: string) => { + const map: Record = { "24h": "daily", "7d": "weekly", "30d": "monthly" }; + return map[value] || value; + }), }; }); @@ -56,6 +60,7 @@ vi.mock("./ModelSelect/ModelSelect", () => { vi.mock("antd", async (importOriginal) => { const actual = await importOriginal(); const React = await import("react"); + const SelectComponent = ({ value, onChange, @@ -88,37 +93,53 @@ vi.mock("antd", async (importOriginal) => { ); }; SelectComponent.displayName = "Select"; + const SelectOption = ({ value: optionValue, children: optionChildren }: { value: string; children: React.ReactNode }) => React.createElement("option", { value: optionValue }, optionChildren); SelectOption.displayName = "SelectOption"; SelectComponent.Option = SelectOption; - const Spin = ({ size }: { size?: string }) => React.createElement("div", { "data-testid": "spinner", "data-size": size }); + + const Spin = ({ size }: { size?: string }) => + React.createElement("div", { "data-testid": "spinner", "data-size": size }); Spin.displayName = "Spin"; - const Switch = ({ checked, onChange }: { checked: boolean; onChange: (checked: boolean) => void }) => + + const InputNumber = ({ + value, + onChange, + placeholder, + prefix, + }: { + value: number | null; + onChange: (value: number | null) => void; + placeholder?: string; + prefix?: string; + min?: number; + className?: string; + style?: React.CSSProperties; + }) => React.createElement("input", { - type: "checkbox", - role: "switch", - checked: checked, - onChange: (e) => onChange(e.target.checked), - "aria-label": "Toggle switch", + type: "number", + value: value ?? "", + onChange: (e: React.ChangeEvent) => { + const v = e.target.value === "" ? null : Number(e.target.value); + onChange(v); + }, + placeholder, + "data-prefix": prefix, + "aria-label": "number input", }); - Switch.displayName = "Switch"; - const Paragraph = ({ children }: { children: React.ReactNode }) => React.createElement("p", {}, children); - Paragraph.displayName = "Paragraph"; + InputNumber.displayName = "InputNumber"; + return { ...actual, Spin, - Switch, Select: SelectComponent, - Typography: { - Paragraph, - }, + InputNumber, }; }); const mockGetDefaultTeamSettings = vi.mocked(networking.getDefaultTeamSettings); const mockUpdateDefaultTeamSettings = vi.mocked(networking.updateDefaultTeamSettings); -const mockModelAvailableCall = vi.mocked(networking.modelAvailableCall); const mockNotificationsManager = vi.mocked(NotificationsManager); describe("TeamSSOSettings", () => { @@ -128,77 +149,33 @@ describe("TeamSSOSettings", () => { userRole: "admin", }; - const mockSettings = { + const mockSettingsResponse = { values: { - budget_duration: "monthly", max_budget: 1000, - enabled: true, - allowed_models: ["gpt-4", "claude-3"], + budget_duration: "30d", + tpm_limit: 500, + rpm_limit: 100, models: ["gpt-4"], - status: "active", - }, - field_schema: { - description: "Default team settings schema", - properties: { - budget_duration: { - type: "string", - description: "Budget duration setting", - }, - max_budget: { - type: "number", - description: "Maximum budget amount", - }, - enabled: { - type: "boolean", - description: "Enable feature", - }, - allowed_models: { - type: "array", - items: { - enum: ["gpt-4", "claude-3", "gpt-3.5-turbo"], - }, - description: "Allowed models", - }, - models: { - type: "array", - description: "Selected models", - }, - status: { - type: "string", - enum: ["active", "inactive", "pending"], - description: "Status", - }, - }, + team_member_permissions: ["/key/generate", "/key/update"], }, }; beforeEach(() => { vi.clearAllMocks(); - mockModelAvailableCall.mockResolvedValue({ - data: [{ id: "gpt-4" }, { id: "claude-3" }], - }); }); - it("should render", async () => { - mockGetDefaultTeamSettings.mockResolvedValue(mockSettings); - - renderWithProviders(); - - await waitFor(() => { - expect(screen.getByText("Default Team Settings")).toBeInTheDocument(); - }); - }); + // --- Loading & Error States --- it("should show loading spinner while fetching settings", () => { - mockGetDefaultTeamSettings.mockImplementation(() => new Promise(() => { })); + mockGetDefaultTeamSettings.mockImplementation(() => new Promise(() => {})); renderWithProviders(); expect(screen.getByTestId("spinner")).toBeInTheDocument(); }); - it("should display message when no settings are available", async () => { - mockGetDefaultTeamSettings.mockResolvedValue(null as any); + it("should display error message when fetch fails", async () => { + mockGetDefaultTeamSettings.mockRejectedValue(new Error("Fetch failed")); renderWithProviders(); @@ -207,6 +184,7 @@ describe("TeamSSOSettings", () => { screen.getByText("No team settings available or you do not have permission to view them."), ).toBeInTheDocument(); }); + expect(mockNotificationsManager.fromBackend).toHaveBeenCalledWith("Failed to fetch team settings"); }); it("should not fetch settings when access token is null", async () => { @@ -217,432 +195,273 @@ describe("TeamSSOSettings", () => { }); }); - it("should display settings fields with correct values", async () => { - mockGetDefaultTeamSettings.mockResolvedValue(mockSettings); + // --- View Mode --- + + it("should render title and subtitle", async () => { + mockGetDefaultTeamSettings.mockResolvedValue(mockSettingsResponse); + + renderWithProviders(); + + await waitFor(() => { + expect(screen.getByText("Default Team Settings")).toBeInTheDocument(); + expect(screen.getByText("These settings will be applied by default when creating new teams.")).toBeInTheDocument(); + }); + }); + + it("should render section headers", async () => { + mockGetDefaultTeamSettings.mockResolvedValue(mockSettingsResponse); + + renderWithProviders(); + + await waitFor(() => { + expect(screen.getByText("Budget & Rate Limits")).toBeInTheDocument(); + expect(screen.getByText("Access & Permissions")).toBeInTheDocument(); + }); + }); + + it("should display all field labels and descriptions", async () => { + mockGetDefaultTeamSettings.mockResolvedValue(mockSettingsResponse); renderWithProviders(); await waitFor(() => { - expect(screen.getByText("Budget Duration")).toBeInTheDocument(); expect(screen.getByText("Max Budget")).toBeInTheDocument(); + expect(screen.getByText("Budget Duration")).toBeInTheDocument(); + expect(screen.getByText("TPM Limit")).toBeInTheDocument(); + expect(screen.getByText("RPM Limit")).toBeInTheDocument(); + expect(screen.getByText("Models")).toBeInTheDocument(); + expect(screen.getByText("Team Member Permissions")).toBeInTheDocument(); }); - expect(screen.getByText("Budget: monthly")).toBeInTheDocument(); - expect(screen.getByText("1000")).toBeInTheDocument(); - const enabledTexts = screen.getAllByText("Enabled"); - expect(enabledTexts.length).toBeGreaterThan(0); + // Descriptions + expect(screen.getByText("Maximum budget (in USD) for new automatically created teams.")).toBeInTheDocument(); + expect(screen.getByText("How frequently the team's budget resets.")).toBeInTheDocument(); + }); + + it("should display formatted values in view mode", async () => { + mockGetDefaultTeamSettings.mockResolvedValue(mockSettingsResponse); + + renderWithProviders(); + + await waitFor(() => { + // max_budget displayed with $ + expect(screen.getByText("$1,000")).toBeInTheDocument(); + // budget_duration through getBudgetDurationLabel + expect(screen.getByText("monthly")).toBeInTheDocument(); + // tpm_limit formatted + expect(screen.getByText("500")).toBeInTheDocument(); + // rpm_limit formatted + expect(screen.getByText("100")).toBeInTheDocument(); + }); + }); + + it("should display models as tags in view mode", async () => { + mockGetDefaultTeamSettings.mockResolvedValue(mockSettingsResponse); + + renderWithProviders(); + + await waitFor(() => { + expect(screen.getByText("gpt-4")).toBeInTheDocument(); + }); + }); + + it("should display permissions as tags in view mode", async () => { + mockGetDefaultTeamSettings.mockResolvedValue(mockSettingsResponse); + + renderWithProviders(); + + await waitFor(() => { + expect(screen.getByText("/key/generate")).toBeInTheDocument(); + expect(screen.getByText("/key/update")).toBeInTheDocument(); + }); }); it("should display 'Not set' for null values", async () => { - const settingsWithNulls = { - ...mockSettings, + mockGetDefaultTeamSettings.mockResolvedValue({ values: { - ...mockSettings.values, max_budget: null, + budget_duration: null, + tpm_limit: null, + rpm_limit: null, + models: [], + team_member_permissions: [], }, - }; - mockGetDefaultTeamSettings.mockResolvedValue(settingsWithNulls); + }); renderWithProviders(); await waitFor(() => { - expect(screen.getByText("Not set")).toBeInTheDocument(); + const notSetElements = screen.getAllByText("Not set"); + // max_budget, budget_duration, tpm_limit, rpm_limit, models (empty), permissions (empty) + expect(notSetElements.length).toBeGreaterThanOrEqual(4); }); }); - it("should toggle edit mode when edit button is clicked", async () => { - mockGetDefaultTeamSettings.mockResolvedValue(mockSettings); + // --- Edit Mode Toggle --- + + it("should toggle to edit mode when Edit Settings is clicked", async () => { + mockGetDefaultTeamSettings.mockResolvedValue(mockSettingsResponse); renderWithProviders(); await waitFor(() => { - expect(screen.getByRole("button", { name: "Edit Settings" })).toBeInTheDocument(); + expect(screen.getByRole("button", { name: /Edit Settings/i })).toBeInTheDocument(); }); - const editButton = screen.getByRole("button", { name: "Edit Settings" }); - await userEvent.click(editButton); + await userEvent.click(screen.getByRole("button", { name: /Edit Settings/i })); - expect(screen.getByRole("button", { name: "Cancel" })).toBeInTheDocument(); - expect(screen.getByRole("button", { name: "Save Changes" })).toBeInTheDocument(); - expect(screen.queryByRole("button", { name: "Edit Settings" })).not.toBeInTheDocument(); + expect(screen.getByRole("button", { name: /Cancel/i })).toBeInTheDocument(); + expect(screen.getByRole("button", { name: /Save Changes/i })).toBeInTheDocument(); + expect(screen.queryByRole("button", { name: /Edit Settings/i })).not.toBeInTheDocument(); }); it("should cancel edit mode and reset values", async () => { - mockGetDefaultTeamSettings.mockResolvedValue(mockSettings); + mockGetDefaultTeamSettings.mockResolvedValue(mockSettingsResponse); renderWithProviders(); await waitFor(() => { - expect(screen.getByRole("button", { name: "Edit Settings" })).toBeInTheDocument(); + expect(screen.getByRole("button", { name: /Edit Settings/i })).toBeInTheDocument(); }); - const editButton = screen.getByRole("button", { name: "Edit Settings" }); - await userEvent.click(editButton); + await userEvent.click(screen.getByRole("button", { name: /Edit Settings/i })); + await userEvent.click(screen.getByRole("button", { name: /Cancel/i })); - const cancelButton = screen.getByRole("button", { name: "Cancel" }); - await userEvent.click(cancelButton); - - expect(screen.getByRole("button", { name: "Edit Settings" })).toBeInTheDocument(); - expect(screen.queryByRole("button", { name: "Cancel" })).not.toBeInTheDocument(); + expect(screen.getByRole("button", { name: /Edit Settings/i })).toBeInTheDocument(); + expect(screen.queryByRole("button", { name: /Cancel/i })).not.toBeInTheDocument(); }); - it("should save settings when save button is clicked", async () => { - mockGetDefaultTeamSettings.mockResolvedValue(mockSettings); - mockUpdateDefaultTeamSettings.mockResolvedValue({ - settings: mockSettings.values, - }); + // --- Edit Mode Fields --- + + it("should show budget duration dropdown in edit mode", async () => { + mockGetDefaultTeamSettings.mockResolvedValue(mockSettingsResponse); renderWithProviders(); await waitFor(() => { - expect(screen.getByRole("button", { name: "Edit Settings" })).toBeInTheDocument(); + expect(screen.getByRole("button", { name: /Edit Settings/i })).toBeInTheDocument(); }); - const editButton = screen.getByRole("button", { name: "Edit Settings" }); - await userEvent.click(editButton); + await userEvent.click(screen.getByRole("button", { name: /Edit Settings/i })); await waitFor(() => { - expect(screen.getByRole("button", { name: "Save Changes" })).toBeInTheDocument(); - }); - - const saveButton = screen.getByRole("button", { name: "Save Changes" }); - await userEvent.click(saveButton); - - await waitFor(() => { - expect(mockUpdateDefaultTeamSettings).toHaveBeenCalledWith("test-token", mockSettings.values); - }); - - expect(mockNotificationsManager.success).toHaveBeenCalledWith("Default team settings updated successfully"); - }); - - it("should show error notification when save fails", async () => { - mockGetDefaultTeamSettings.mockResolvedValue(mockSettings); - mockUpdateDefaultTeamSettings.mockRejectedValue(new Error("Save failed")); - - renderWithProviders(); - - await waitFor(() => { - expect(screen.getByRole("button", { name: "Edit Settings" })).toBeInTheDocument(); - }); - - const editButton = screen.getByRole("button", { name: "Edit Settings" }); - await userEvent.click(editButton); - - await waitFor(() => { - expect(screen.getByRole("button", { name: "Save Changes" })).toBeInTheDocument(); - }); - - const saveButton = screen.getByRole("button", { name: "Save Changes" }); - await userEvent.click(saveButton); - - await waitFor(() => { - expect(mockNotificationsManager.fromBackend).toHaveBeenCalledWith("Failed to update team settings"); + expect(screen.getByTestId("budget-duration-dropdown")).toBeInTheDocument(); }); }); - it("should render boolean field as switch in edit mode", async () => { - mockGetDefaultTeamSettings.mockResolvedValue(mockSettings); + it("should show ModelSelect in edit mode", async () => { + mockGetDefaultTeamSettings.mockResolvedValue(mockSettingsResponse); renderWithProviders(); await waitFor(() => { - expect(screen.getByRole("button", { name: "Edit Settings" })).toBeInTheDocument(); + expect(screen.getByRole("button", { name: /Edit Settings/i })).toBeInTheDocument(); }); - const editButton = screen.getByRole("button", { name: "Edit Settings" }); - await userEvent.click(editButton); - - await waitFor(() => { - const switchElement = screen.getByRole("switch"); - expect(switchElement).toBeInTheDocument(); - expect(switchElement).toBeChecked(); - }); - }); - - it("should update boolean value when switch is toggled", async () => { - mockGetDefaultTeamSettings.mockResolvedValue(mockSettings); - - renderWithProviders(); - - await waitFor(() => { - expect(screen.getByRole("button", { name: "Edit Settings" })).toBeInTheDocument(); - }); - - const editButton = screen.getByRole("button", { name: "Edit Settings" }); - await userEvent.click(editButton); - - await waitFor(() => { - expect(screen.getByRole("switch")).toBeInTheDocument(); - }); - - const switchElement = screen.getByRole("switch"); - await userEvent.click(switchElement); - - expect(switchElement).not.toBeChecked(); - }); - - it("should render budget duration dropdown in edit mode", async () => { - mockGetDefaultTeamSettings.mockResolvedValue(mockSettings); - - renderWithProviders(); - - await waitFor(() => { - expect(screen.getByRole("button", { name: "Edit Settings" })).toBeInTheDocument(); - }); - - const editButton = screen.getByRole("button", { name: "Edit Settings" }); - await userEvent.click(editButton); - - await waitFor(() => { - expect(screen.getByLabelText("Budget duration")).toBeInTheDocument(); - }); - }); - - it("should update budget duration when dropdown value changes", async () => { - mockGetDefaultTeamSettings.mockResolvedValue(mockSettings); - - renderWithProviders(); - - await waitFor(() => { - expect(screen.getByRole("button", { name: "Edit Settings" })).toBeInTheDocument(); - }); - - const editButton = screen.getByRole("button", { name: "Edit Settings" }); - await userEvent.click(editButton); - - await waitFor(() => { - expect(screen.getByLabelText("Budget duration")).toBeInTheDocument(); - }); - - const dropdown = screen.getByLabelText("Budget duration"); - await userEvent.selectOptions(dropdown, "daily"); - - expect(dropdown).toHaveValue("daily"); - }); - - it("should render text input for string fields in edit mode", async () => { - const settingsWithString = { - ...mockSettings, - field_schema: { - ...mockSettings.field_schema, - properties: { - ...mockSettings.field_schema.properties, - team_name: { - type: "string", - description: "Team name", - }, - }, - }, - values: { - ...mockSettings.values, - team_name: "Test Team", - }, - }; - mockGetDefaultTeamSettings.mockResolvedValue(settingsWithString); - - renderWithProviders(); - - await waitFor(() => { - expect(screen.getByRole("button", { name: "Edit Settings" })).toBeInTheDocument(); - }); - - const editButton = screen.getByRole("button", { name: "Edit Settings" }); - await userEvent.click(editButton); - - await waitFor(() => { - const textInput = screen.getByDisplayValue("Test Team"); - expect(textInput).toBeInTheDocument(); - }); - }); - - it("should render enum select for string enum fields in edit mode", async () => { - mockGetDefaultTeamSettings.mockResolvedValue(mockSettings); - - renderWithProviders(); - - await waitFor(() => { - expect(screen.getByRole("button", { name: "Edit Settings" })).toBeInTheDocument(); - }); - - const editButton = screen.getByRole("button", { name: "Edit Settings" }); - await userEvent.click(editButton); - - await waitFor(() => { - const statusSelect = screen.getAllByRole("listbox")[0]; - expect(statusSelect).toBeInTheDocument(); - }); - }); - - it("should render multi-select for array enum fields in edit mode", async () => { - mockGetDefaultTeamSettings.mockResolvedValue(mockSettings); - - renderWithProviders(); - - await waitFor(() => { - expect(screen.getByRole("button", { name: "Edit Settings" })).toBeInTheDocument(); - }); - - const editButton = screen.getByRole("button", { name: "Edit Settings" }); - await userEvent.click(editButton); - - await waitFor(() => { - const multiSelects = screen.getAllByRole("listbox"); - expect(multiSelects.length).toBeGreaterThan(0); - }); - }); - - it("should render ModelSelect for models field in edit mode", async () => { - mockGetDefaultTeamSettings.mockResolvedValue(mockSettings); - - renderWithProviders(); - - await waitFor(() => { - expect(screen.getByRole("button", { name: "Edit Settings" })).toBeInTheDocument(); - }); - - const editButton = screen.getByRole("button", { name: "Edit Settings" }); - await userEvent.click(editButton); + await userEvent.click(screen.getByRole("button", { name: /Edit Settings/i })); await waitFor(() => { expect(screen.getByTestId("model-select")).toBeInTheDocument(); }); }); - it("should display models as badges in view mode", async () => { - mockGetDefaultTeamSettings.mockResolvedValue(mockSettings); + it("should show number inputs for budget and rate limits in edit mode", async () => { + mockGetDefaultTeamSettings.mockResolvedValue(mockSettingsResponse); renderWithProviders(); await waitFor(() => { - const gpt4Elements = screen.getAllByText("gpt-4"); - expect(gpt4Elements.length).toBeGreaterThan(0); + expect(screen.getByRole("button", { name: /Edit Settings/i })).toBeInTheDocument(); + }); + + await userEvent.click(screen.getByRole("button", { name: /Edit Settings/i })); + + await waitFor(() => { + const numberInputs = screen.getAllByLabelText("number input"); + // max_budget, tpm_limit, rpm_limit + expect(numberInputs.length).toBe(3); }); }); - it("should display 'None' for empty arrays in view mode", async () => { - const settingsWithEmptyArray = { - ...mockSettings, - values: { - ...mockSettings.values, - models: [], - }, - }; - mockGetDefaultTeamSettings.mockResolvedValue(settingsWithEmptyArray); + it("should show permissions multi-select in edit mode", async () => { + mockGetDefaultTeamSettings.mockResolvedValue(mockSettingsResponse); renderWithProviders(); await waitFor(() => { - const noneTexts = screen.getAllByText("None"); - expect(noneTexts.length).toBeGreaterThan(0); + expect(screen.getByRole("button", { name: /Edit Settings/i })).toBeInTheDocument(); + }); + + await userEvent.click(screen.getByRole("button", { name: /Edit Settings/i })); + + await waitFor(() => { + const listboxes = screen.getAllByRole("listbox"); + expect(listboxes.length).toBeGreaterThan(0); }); }); - it("should display schema description when available", async () => { - mockGetDefaultTeamSettings.mockResolvedValue(mockSettings); + // --- Save --- + + it("should save settings and show success notification", async () => { + mockGetDefaultTeamSettings.mockResolvedValue(mockSettingsResponse); + mockUpdateDefaultTeamSettings.mockResolvedValue({ + settings: mockSettingsResponse.values, + }); renderWithProviders(); await waitFor(() => { - expect(screen.getByText("Default team settings schema")).toBeInTheDocument(); + expect(screen.getByRole("button", { name: /Edit Settings/i })).toBeInTheDocument(); + }); + + await userEvent.click(screen.getByRole("button", { name: /Edit Settings/i })); + await userEvent.click(screen.getByRole("button", { name: /Save Changes/i })); + + await waitFor(() => { + expect(mockUpdateDefaultTeamSettings).toHaveBeenCalledWith("test-token", expect.any(Object)); + }); + + expect(mockNotificationsManager.success).toHaveBeenCalledWith("Default team settings updated successfully"); + + // Should exit edit mode after save + await waitFor(() => { + expect(screen.getByRole("button", { name: /Edit Settings/i })).toBeInTheDocument(); }); }); - it("should show error notification when fetching settings fails", async () => { - mockGetDefaultTeamSettings.mockRejectedValue(new Error("Fetch failed")); + it("should show error notification when save fails", async () => { + mockGetDefaultTeamSettings.mockResolvedValue(mockSettingsResponse); + mockUpdateDefaultTeamSettings.mockRejectedValue(new Error("Save failed")); renderWithProviders(); await waitFor(() => { - expect(mockNotificationsManager.fromBackend).toHaveBeenCalledWith("Failed to fetch team settings"); + expect(screen.getByRole("button", { name: /Edit Settings/i })).toBeInTheDocument(); }); - }); - it("should handle model fetch error gracefully", async () => { - mockGetDefaultTeamSettings.mockResolvedValue(mockSettings); - mockModelAvailableCall.mockRejectedValue(new Error("Model fetch failed")); - - renderWithProviders(); + await userEvent.click(screen.getByRole("button", { name: /Edit Settings/i })); + await userEvent.click(screen.getByRole("button", { name: /Save Changes/i })); await waitFor(() => { - expect(screen.getByText("Default Team Settings")).toBeInTheDocument(); + expect(mockNotificationsManager.fromBackend).toHaveBeenCalledWith("Failed to update team settings"); }); }); it("should disable cancel button while saving", async () => { - mockGetDefaultTeamSettings.mockResolvedValue(mockSettings); + mockGetDefaultTeamSettings.mockResolvedValue(mockSettingsResponse); mockUpdateDefaultTeamSettings.mockImplementation( - () => new Promise((resolve) => setTimeout(() => resolve({ settings: mockSettings.values }), 100)), + () => new Promise((resolve) => setTimeout(() => resolve({ settings: mockSettingsResponse.values }), 100)), ); renderWithProviders(); await waitFor(() => { - expect(screen.getByRole("button", { name: "Edit Settings" })).toBeInTheDocument(); + expect(screen.getByRole("button", { name: /Edit Settings/i })).toBeInTheDocument(); }); - const editButton = screen.getByRole("button", { name: "Edit Settings" }); - await userEvent.click(editButton); + await userEvent.click(screen.getByRole("button", { name: /Edit Settings/i })); + await userEvent.click(screen.getByRole("button", { name: /Save Changes/i })); - await waitFor(() => { - expect(screen.getByRole("button", { name: "Save Changes" })).toBeInTheDocument(); - }); - - const saveButton = screen.getByRole("button", { name: "Save Changes" }); - await userEvent.click(saveButton); - - const cancelButton = screen.getByRole("button", { name: "Cancel" }); - expect(cancelButton).toBeDisabled(); - }); - - it("should display field descriptions", async () => { - mockGetDefaultTeamSettings.mockResolvedValue(mockSettings); - - renderWithProviders(); - - await waitFor(() => { - expect(screen.getByText("Budget duration setting")).toBeInTheDocument(); - expect(screen.getByText("Maximum budget amount")).toBeInTheDocument(); - }); - }); - - it("should format field names by replacing underscores and capitalizing", async () => { - const settingsWithUnderscores = { - ...mockSettings, - field_schema: { - ...mockSettings.field_schema, - properties: { - ...mockSettings.field_schema.properties, - max_budget_per_user: { - type: "number", - description: "Max budget per user", - }, - }, - }, - values: { - ...mockSettings.values, - max_budget_per_user: 500, - }, - }; - mockGetDefaultTeamSettings.mockResolvedValue(settingsWithUnderscores); - - renderWithProviders(); - - await waitFor(() => { - expect(screen.getByText("Max Budget Per User")).toBeInTheDocument(); - }); - }); - - it("should display 'No schema information available' when schema is missing", async () => { - const settingsWithoutSchema = { - values: {}, - field_schema: null, - }; - mockGetDefaultTeamSettings.mockResolvedValue(settingsWithoutSchema); - - renderWithProviders(); - - await waitFor(() => { - expect(screen.getByText("No schema information available")).toBeInTheDocument(); - }); + expect(screen.getByRole("button", { name: /Cancel/i })).toBeDisabled(); }); }); diff --git a/ui/litellm-dashboard/src/components/TeamSSOSettings.tsx b/ui/litellm-dashboard/src/components/TeamSSOSettings.tsx index 33bfc783afd..98745d388a2 100644 --- a/ui/litellm-dashboard/src/components/TeamSSOSettings.tsx +++ b/ui/litellm-dashboard/src/components/TeamSSOSettings.tsx @@ -1,30 +1,100 @@ import React, { useState, useEffect } from "react"; -import { Card, Title, Text, Divider, Button, TextInput } from "@tremor/react"; -import { Typography, Spin, Switch, Select } from "antd"; -import { getDefaultTeamSettings, updateDefaultTeamSettings, modelAvailableCall } from "./networking"; +import { Card, Button, InputNumber, Typography, Spin, Select, Tag, Row, Col } from "antd"; +import { EditOutlined, SaveOutlined } from "@ant-design/icons"; +import { getDefaultTeamSettings, updateDefaultTeamSettings } from "./networking"; import BudgetDurationDropdown, { getBudgetDurationLabel } from "./common_components/budget_duration_dropdown"; import { getModelDisplayName } from "./key_team_helpers/fetch_available_models_team_key"; import NotificationsManager from "./molecules/notifications_manager"; import { ModelSelect } from "./ModelSelect/ModelSelect"; +const { Title, Text } = Typography; + interface TeamSSOSettingsProps { accessToken: string | null; userID: string; userRole: string; } -const TeamSSOSettings: React.FC = ({ accessToken, userID, userRole }) => { +const PERMISSION_OPTIONS = [ + "/key/generate", + "/key/update", + "/key/delete", + "/key/regenerate", + "/key/service-account/generate", + "/key/{key_id}/regenerate", + "/key/block", + "/key/unblock", + "/key/bulk_update", + "/key/{key_id}/reset_spend", + "/key/info", + "/key/list", + "/key/aliases", + "/team/daily/activity", +]; + +interface SettingRowProps { + label: string; + description: string; + isEditing: boolean; + viewContent: React.ReactNode; + editContent: React.ReactNode; +} + +const SettingRow: React.FC = ({ label, description, isEditing, viewContent, editContent }) => ( + + +
{label}
+
{description}
+ + +
{isEditing ? editContent : viewContent}
+ + +); + +const NotSet = () => Not set; + +const renderTags = (values: string[], displayFn?: (v: string) => string) => { + if (!values || values.length === 0) return ; + return ( +
+ {values.map((v) => ( + + {displayFn ? displayFn(v) : v} + + ))} +
+ ); +}; + +interface SettingsValues { + max_budget: number | null; + budget_duration: string | null; + tpm_limit: number | null; + rpm_limit: number | null; + models: string[]; + team_member_permissions: string[]; +} + +const DEFAULT_VALUES: SettingsValues = { + max_budget: null, + budget_duration: null, + tpm_limit: null, + rpm_limit: null, + models: [], + team_member_permissions: [], +}; + +const TeamSSOSettings: React.FC = ({ accessToken }) => { const [loading, setLoading] = useState(true); - const [settings, setSettings] = useState(null); + const [values, setValues] = useState(DEFAULT_VALUES); const [isEditing, setIsEditing] = useState(false); - const [editedValues, setEditedValues] = useState({}); + const [editedValues, setEditedValues] = useState(DEFAULT_VALUES); const [saving, setSaving] = useState(false); - const [availableModels, setAvailableModels] = useState([]); - const { Paragraph } = Typography; - const { Option } = Select; + const [fetchError, setFetchError] = useState(false); useEffect(() => { - const fetchTeamSSOSettings = async () => { + const fetchSettings = async () => { if (!accessToken) { setLoading(false); return; @@ -32,39 +102,30 @@ const TeamSSOSettings: React.FC = ({ accessToken, userID, try { const data = await getDefaultTeamSettings(accessToken); - setSettings(data); - setEditedValues(data.values || {}); - - // Fetch available models - if (accessToken) { - try { - const modelResponse = await modelAvailableCall(accessToken, userID, userRole); - if (modelResponse && modelResponse.data) { - const modelNames = modelResponse.data.map((model: { id: string }) => model.id); - setAvailableModels(modelNames); - } - } catch (error) { - console.error("Error fetching available models:", error); - } - } + const fetched = { ...DEFAULT_VALUES, ...(data.values || {}) }; + setValues(fetched); + setEditedValues(fetched); } catch (error) { console.error("Error fetching team SSO settings:", error); + setFetchError(true); NotificationsManager.fromBackend("Failed to fetch team settings"); } finally { setLoading(false); } }; - fetchTeamSSOSettings(); + fetchSettings(); }, [accessToken]); - const handleSaveSettings = async () => { + const handleSave = async () => { if (!accessToken) return; setSaving(true); try { const updatedSettings = await updateDefaultTeamSettings(accessToken, editedValues); - setSettings({ ...settings, values: updatedSettings.settings }); + const newValues = { ...DEFAULT_VALUES, ...(updatedSettings.settings || {}) }; + setValues(newValues); + setEditedValues(newValues); setIsEditing(false); NotificationsManager.success("Default team settings updated successfully"); } catch (error) { @@ -75,129 +136,13 @@ const TeamSSOSettings: React.FC = ({ accessToken, userID, } }; - const handleTextInputChange = (key: string, value: any) => { - setEditedValues((prev: Record) => ({ - ...prev, - [key]: value, - })); + const handleCancel = () => { + setIsEditing(false); + setEditedValues(values); }; - const renderEditableField = (key: string, property: any, value: any) => { - const type = property.type; - - if (key === "budget_duration") { - return ( - handleTextInputChange(key, value)} - className="mt-2" - /> - ); - } else if (type === "boolean") { - return ( -
- handleTextInputChange(key, checked)} /> -
- ); - } else if (type === "array" && property.items?.enum) { - return ( - - ); - } else if (key === "models") { - return ( - handleTextInputChange(key, value)} - context="global" - style={{ width: "100%" }} - options={{ - includeSpecialOptions: true, - }} - /> - ); - } else if (type === "string" && property.enum) { - return ( - - ); - } else { - return ( - handleTextInputChange(key, e.target.value)} - placeholder={property.description || ""} - className="mt-2" - /> - ); - } - }; - - const renderValue = (key: string, value: any): JSX.Element => { - if (value === null || value === undefined) return Not set; - - if (key === "budget_duration") { - return {getBudgetDurationLabel(value)}; - } - - if (typeof value === "boolean") { - return {value ? "Enabled" : "Disabled"}; - } - - if (key === "models" && Array.isArray(value)) { - if (value.length === 0) return None; - - return ( -
- {value.map((model, index) => ( - - {getModelDisplayName(model)} - - ))} -
- ); - } - - if (typeof value === "object") { - if (Array.isArray(value)) { - if (value.length === 0) return None; - - return ( -
- {value.map((item, index) => ( - - {typeof item === "object" ? JSON.stringify(item) : String(item)} - - ))} -
- ); - } - - return 
{JSON.stringify(value, null, 2)}
; - } - - return {String(value)}; + const update = (key: K, value: SettingsValues[K]) => { + setEditedValues((prev) => ({ ...prev, [key]: value })); }; if (loading) { @@ -208,7 +153,7 @@ const TeamSSOSettings: React.FC = ({ accessToken, userID, ); } - if (!settings) { + if (fetchError) { return ( No team settings available or you do not have permission to view them. @@ -216,70 +161,166 @@ const TeamSSOSettings: React.FC = ({ accessToken, userID, ); } - // Dynamically render settings based on the schema - const renderSettings = () => { - const { values, field_schema } = settings; - - if (!field_schema || !field_schema.properties) { - return No schema information available; - } - - return Object.entries(field_schema.properties).map(([key, property]: [string, any]) => { - const value = values[key]; - const displayName = key.replace(/_/g, " ").replace(/\b\w/g, (l) => l.toUpperCase()); - - return ( -
- {displayName} - - {property.description || "No description available"} - - - {isEditing ? ( -
{renderEditableField(key, property, value)}
- ) : ( -
{renderValue(key, value)}
- )} -
- ); - }); - }; - return ( - -
- Default Team Settings - {!loading && - settings && - (isEditing ? ( -
- -
) : ( - - ))} + + )} +
- These settings will be applied by default when creating new teams. +
+ {/* Budget & Rate Limits */} +
+
Budget & Rate Limits
+
+ ${Number(values.max_budget).toLocaleString()} : + } + editContent={ + update("max_budget", v)} + placeholder="Not set" + prefix="$" + min={0} + /> + } + /> - {settings?.field_schema?.description && ( - {settings.field_schema.description} - )} - + {getBudgetDurationLabel(values.budget_duration)} : + } + editContent={ + update("budget_duration", v)} + style={{ maxWidth: 320 }} + /> + } + /> -
{renderSettings()}
+ {values.tpm_limit.toLocaleString()} : + } + editContent={ + update("tpm_limit", v)} + placeholder="Not set" + min={0} + /> + } + /> + + {values.rpm_limit.toLocaleString()} : + } + editContent={ + update("rpm_limit", v)} + placeholder="Not set" + min={0} + /> + } + /> +
+
+ + {/* Access & Permissions */} +
+
Access & Permissions
+
+ update("models", v)} + context="global" + style={{ width: "100%" }} + options={{ includeSpecialOptions: true }} + /> + } + /> + + update("team_member_permissions", v)} + placeholder="Select permissions" + tagRender={({ label, closable, onClose }) => ( + + {label} + + )} + > + {PERMISSION_OPTIONS.map((option) => ( + + {option} + + ))} + + } + /> +
+
+
); }; diff --git a/ui/litellm-dashboard/src/components/ToolPolicies/PolicySelect.test.tsx b/ui/litellm-dashboard/src/components/ToolPolicies/PolicySelect.test.tsx new file mode 100644 index 00000000000..cde59fe9c6d --- /dev/null +++ b/ui/litellm-dashboard/src/components/ToolPolicies/PolicySelect.test.tsx @@ -0,0 +1,84 @@ +import { renderWithProviders, screen } from "../../../tests/test-utils"; +import { vi } from "vitest"; +import { + PolicySelect, + policyStyle, + INPUT_POLICY_OPTIONS, + OUTPUT_POLICY_OPTIONS, +} from "./PolicySelect"; + +describe("policyStyle", () => { + it("should return the matching option for a known policy", () => { + expect(policyStyle("trusted")).toEqual(INPUT_POLICY_OPTIONS[1]); + }); + + it("should return the matching option for blocked", () => { + expect(policyStyle("blocked")).toEqual(INPUT_POLICY_OPTIONS[2]); + }); + + it("should return the first option as fallback for unknown policy", () => { + expect(policyStyle("unknown")).toEqual(INPUT_POLICY_OPTIONS[0]); + }); +}); + +describe("PolicySelect", () => { + it("should render", () => { + renderWithProviders( + + ); + expect(screen.getByText("untrusted")).toBeInTheDocument(); + }); + + it("should show the current policy value", () => { + renderWithProviders( + + ); + expect(screen.getByText("trusted")).toBeInTheDocument(); + }); + + it("should be disabled when saving is true", () => { + renderWithProviders( + + ); + expect(screen.getByRole("combobox")).toHaveAttribute("aria-expanded", "false"); + expect(screen.getByRole("combobox").closest(".ant-select")).toHaveClass("ant-select-disabled"); + }); + + it("should not be disabled when saving is false", () => { + renderWithProviders( + + ); + expect(screen.getByRole("combobox").closest(".ant-select")).not.toHaveClass("ant-select-disabled"); + }); +}); + +describe("Policy option constants", () => { + it("should have 3 input policy options", () => { + expect(INPUT_POLICY_OPTIONS).toHaveLength(3); + }); + + it("should have 2 output policy options (no blocked)", () => { + expect(OUTPUT_POLICY_OPTIONS).toHaveLength(2); + expect(OUTPUT_POLICY_OPTIONS.map((o) => o.value)).toEqual(["untrusted", "trusted"]); + }); +}); diff --git a/ui/litellm-dashboard/src/components/ToolPoliciesView.test.tsx b/ui/litellm-dashboard/src/components/ToolPoliciesView.test.tsx new file mode 100644 index 00000000000..8b2b1d0e4b7 --- /dev/null +++ b/ui/litellm-dashboard/src/components/ToolPoliciesView.test.tsx @@ -0,0 +1,53 @@ +import React from "react"; +import { describe, it, expect, vi } from "vitest"; +import { screen } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import { renderWithProviders } from "../../tests/test-utils"; +import ToolPoliciesView from "./ToolPoliciesView"; + +vi.mock("@/components/ToolDetail", () => ({ + ToolDetail: ({ toolName, onBack }: { toolName: string; onBack: () => void }) => ( +
+ Detail: {toolName} + +
+ ), +})); + +vi.mock("@/components/ToolPolicies", () => ({ + ToolPolicies: ({ onSelectTool }: { onSelectTool: (name: string) => void }) => ( +
+ Tool Policies Overview + +
+ ), +})); + +describe("ToolPoliciesView", () => { + it("should render the overview by default", () => { + renderWithProviders(); + + expect(screen.getByText("Tool Policies Overview")).toBeInTheDocument(); + }); + + it("should navigate to tool detail when a tool is selected", async () => { + const user = userEvent.setup(); + renderWithProviders(); + + await user.click(screen.getByRole("button", { name: /select tool/i })); + + expect(screen.getByText("Detail: my-tool")).toBeInTheDocument(); + expect(screen.queryByText("Tool Policies Overview")).not.toBeInTheDocument(); + }); + + it("should navigate back to overview when back is clicked", async () => { + const user = userEvent.setup(); + renderWithProviders(); + + await user.click(screen.getByRole("button", { name: /select tool/i })); + await user.click(screen.getByRole("button", { name: /back/i })); + + expect(screen.getByText("Tool Policies Overview")).toBeInTheDocument(); + expect(screen.queryByText("Detail: my-tool")).not.toBeInTheDocument(); + }); +}); diff --git a/ui/litellm-dashboard/src/components/UsagePage/components/EntityUsage/EntityUsage.test.tsx b/ui/litellm-dashboard/src/components/UsagePage/components/EntityUsage/EntityUsage.test.tsx index c29ade5d653..5c23cf71ab4 100644 --- a/ui/litellm-dashboard/src/components/UsagePage/components/EntityUsage/EntityUsage.test.tsx +++ b/ui/litellm-dashboard/src/components/UsagePage/components/EntityUsage/EntityUsage.test.tsx @@ -125,6 +125,208 @@ describe("EntityUsage", () => { }, }; + const mockAgentSpendData = { + results: [ + { + date: "2025-01-01", + metrics: { + spend: 245.8, + api_requests: 3200, + successful_requests: 3100, + failed_requests: 100, + total_tokens: 1250000, + prompt_tokens: 850000, + completion_tokens: 400000, + cache_read_input_tokens: 50000, + cache_creation_input_tokens: 10000, + }, + breakdown: { + entities: { + "agent-code-review": { + metrics: { + spend: 120.4, + api_requests: 1500, + successful_requests: 1450, + failed_requests: 50, + total_tokens: 620000, + prompt_tokens: 420000, + completion_tokens: 200000, + cache_read_input_tokens: 30000, + cache_creation_input_tokens: 5000, + }, + metadata: { agent_name: "Code Review Agent" }, + api_key_breakdown: {}, + }, + "agent-customer-support": { + metrics: { + spend: 85.2, + api_requests: 1200, + successful_requests: 1170, + failed_requests: 30, + total_tokens: 430000, + prompt_tokens: 290000, + completion_tokens: 140000, + cache_read_input_tokens: 15000, + cache_creation_input_tokens: 3000, + }, + metadata: { agent_name: "Customer Support Agent" }, + api_key_breakdown: {}, + }, + "agent-data-analyst": { + metrics: { + spend: 40.2, + api_requests: 500, + successful_requests: 480, + failed_requests: 20, + total_tokens: 200000, + prompt_tokens: 140000, + completion_tokens: 60000, + cache_read_input_tokens: 5000, + cache_creation_input_tokens: 2000, + }, + metadata: { agent_name: "Data Analyst Agent" }, + api_key_breakdown: {}, + }, + }, + models: { + "gpt-4o": { + metrics: { + spend: 180.0, + api_requests: 2000, + successful_requests: 1950, + failed_requests: 50, + total_tokens: 900000, + prompt_tokens: 600000, + completion_tokens: 300000, + cache_read_input_tokens: 40000, + cache_creation_input_tokens: 8000, + }, + metadata: {}, + api_key_breakdown: {}, + }, + "claude-sonnet-4-20250514": { + metrics: { + spend: 65.8, + api_requests: 1200, + successful_requests: 1150, + failed_requests: 50, + total_tokens: 350000, + prompt_tokens: 250000, + completion_tokens: 100000, + cache_read_input_tokens: 10000, + cache_creation_input_tokens: 2000, + }, + metadata: {}, + api_key_breakdown: {}, + }, + }, + api_keys: {}, + providers: { + openai: { + metrics: { + spend: 180.0, + api_requests: 2000, + successful_requests: 1950, + failed_requests: 50, + total_tokens: 900000, + prompt_tokens: 600000, + completion_tokens: 300000, + cache_read_input_tokens: 40000, + cache_creation_input_tokens: 8000, + }, + }, + anthropic: { + metrics: { + spend: 65.8, + api_requests: 1200, + successful_requests: 1150, + failed_requests: 50, + total_tokens: 350000, + prompt_tokens: 250000, + completion_tokens: 100000, + cache_read_input_tokens: 10000, + cache_creation_input_tokens: 2000, + }, + }, + }, + }, + }, + { + date: "2025-01-02", + metrics: { + spend: 198.5, + api_requests: 2800, + successful_requests: 2720, + failed_requests: 80, + total_tokens: 980000, + prompt_tokens: 670000, + completion_tokens: 310000, + cache_read_input_tokens: 42000, + cache_creation_input_tokens: 9000, + }, + breakdown: { + entities: { + "agent-code-review": { + metrics: { + spend: 95.3, + api_requests: 1300, + successful_requests: 1270, + failed_requests: 30, + total_tokens: 510000, + prompt_tokens: 350000, + completion_tokens: 160000, + cache_read_input_tokens: 25000, + cache_creation_input_tokens: 4000, + }, + metadata: { agent_name: "Code Review Agent" }, + api_key_breakdown: {}, + }, + "agent-customer-support": { + metrics: { + spend: 68.7, + api_requests: 1000, + successful_requests: 970, + failed_requests: 30, + total_tokens: 320000, + prompt_tokens: 220000, + completion_tokens: 100000, + cache_read_input_tokens: 12000, + cache_creation_input_tokens: 3000, + }, + metadata: { agent_name: "Customer Support Agent" }, + api_key_breakdown: {}, + }, + "agent-data-analyst": { + metrics: { + spend: 34.5, + api_requests: 500, + successful_requests: 480, + failed_requests: 20, + total_tokens: 150000, + prompt_tokens: 100000, + completion_tokens: 50000, + cache_read_input_tokens: 5000, + cache_creation_input_tokens: 2000, + }, + metadata: { agent_name: "Data Analyst Agent" }, + api_key_breakdown: {}, + }, + }, + models: {}, + api_keys: {}, + providers: {}, + }, + }, + ], + metadata: { + total_spend: 444.3, + total_api_requests: 6000, + total_successful_requests: 5820, + total_failed_requests: 180, + total_tokens: 2230000, + }, + }; + const defaultProps = { accessToken: "test-token", entityType: "tag" as const, @@ -153,7 +355,7 @@ describe("EntityUsage", () => { mockTeamDailyActivityCall.mockResolvedValue(mockSpendData); mockOrganizationDailyActivityCall.mockResolvedValue(mockSpendData); mockCustomerDailyActivityCall.mockResolvedValue(mockSpendData); - mockAgentDailyActivityCall.mockResolvedValue(mockSpendData); + mockAgentDailyActivityCall.mockResolvedValue(mockAgentSpendData); mockUserDailyActivityCall.mockResolvedValue(mockSpendData); }); @@ -231,7 +433,7 @@ describe("EntityUsage", () => { expect(screen.getByText("Agent Spend Overview")).toBeInTheDocument(); await waitFor(() => { - const spendElements = screen.getAllByText("$100.50"); + const spendElements = screen.getAllByText("$444.30"); expect(spendElements.length).toBeGreaterThan(0); }); }); @@ -385,6 +587,87 @@ describe("EntityUsage", () => { }); }); + it("should display Agent Activity tab for team entity type", async () => { + render(); + + await waitFor(() => { + expect(mockTeamDailyActivityCall).toHaveBeenCalled(); + }); + + expect(screen.getByText("Agent Activity")).toBeInTheDocument(); + }); + + it("should not display Agent Activity tab for non-team entity types", async () => { + render(); + + await waitFor(() => { + expect(mockTagDailyActivityCall).toHaveBeenCalled(); + }); + + expect(screen.queryByText("Agent Activity")).not.toBeInTheDocument(); + }); + + it("should display Top Agents Driving Spend card for team entity type", async () => { + render(); + + await waitFor(() => { + expect(mockTeamDailyActivityCall).toHaveBeenCalled(); + }); + + expect(screen.getByText("Top Agents Driving Spend")).toBeInTheDocument(); + }); + + it("should not display Top Agents Driving Spend card for non-team entity types", async () => { + render(); + + await waitFor(() => { + expect(mockTagDailyActivityCall).toHaveBeenCalled(); + }); + + expect(screen.queryByText("Top Agents Driving Spend")).not.toBeInTheDocument(); + }); + + it("should fetch agent activity data when entity type is team", async () => { + render(); + + await waitFor(() => { + expect(mockAgentDailyActivityCall).toHaveBeenCalledWith( + "test-token", + expect.any(Date), + expect.any(Date), + 1, + null, + ); + }); + }); + + it("should not fetch agent activity data for non-team entity types", async () => { + render(); + + await waitFor(() => { + expect(mockTagDailyActivityCall).toHaveBeenCalled(); + }); + + expect(mockAgentDailyActivityCall).not.toHaveBeenCalled(); + }); + + it("should switch to Agent Activity tab for team entity type", async () => { + render(); + + await waitFor(() => { + expect(mockTeamDailyActivityCall).toHaveBeenCalled(); + }); + + const agentActivityTab = screen.getByText("Agent Activity"); + act(() => { + fireEvent.click(agentActivityTab); + }); + + await waitFor(() => { + expect(screen.getAllByText("Activity Metrics").length).toBeGreaterThan(0); + }); + }); + it("should fallback to entity value when no entityList and no team_alias", async () => { const spendDataWithoutAlias = { ...mockSpendData, diff --git a/ui/litellm-dashboard/src/components/UsagePage/components/EntityUsage/EntityUsage.tsx b/ui/litellm-dashboard/src/components/UsagePage/components/EntityUsage/EntityUsage.tsx index a106910cff7..aaeb8ebb4be 100644 --- a/ui/litellm-dashboard/src/components/UsagePage/components/EntityUsage/EntityUsage.tsx +++ b/ui/litellm-dashboard/src/components/UsagePage/components/EntityUsage/EntityUsage.tsx @@ -22,7 +22,9 @@ import { Text, Title, } from "@tremor/react"; -import React, { useEffect, useState } from "react"; +import { ExportOutlined, LoadingOutlined } from "@ant-design/icons"; +import { Alert, Button } from "antd"; +import React, { useMemo, useState } from "react"; import { ActivityMetrics, processActivityData } from "../../../activity_metrics"; import { UsageExportHeader } from "../../../EntityUsageExport"; import type { EntityType } from "../../../EntityUsageExport/types"; @@ -35,6 +37,7 @@ import { userDailyActivityCall, } from "../../../networking"; import { getProviderLogoAndName } from "../../../provider_info_helpers"; +import { usePaginatedDailyActivity } from "../../hooks/usePaginatedDailyActivity"; import { BreakdownMetrics, DailyData, EntityMetricWithMetadata, KeyMetricWithMetadata, TagUsage } from "../../types"; import { valueFormatterSpend } from "../../utils/value_formatters"; import EndpointUsage from "../EndpointUsage/EndpointUsage"; @@ -87,93 +90,64 @@ interface EntityUsageProps { dateValue: DateRangePickerValue; } -const EntityUsage: React.FC = ({ accessToken, entityType, entityId, entityList, dateValue }) => { - const [spendData, setSpendData] = useState({ - results: [], - metadata: { - total_spend: 0, - total_api_requests: 0, - total_successful_requests: 0, - total_failed_requests: 0, - total_tokens: 0, - }, - }); - const { teams } = useTeams(); +const ENTITY_FETCH_FNS: Record Promise> = { + tag: tagDailyActivityCall, + team: teamDailyActivityCall, + organization: organizationDailyActivityCall, + customer: customerDailyActivityCall, + agent: agentDailyActivityCall, + user: userDailyActivityCall, +}; - const modelMetrics = processActivityData(spendData, "models", teams || []); - const keyMetrics = processActivityData(spendData, "api_keys", teams || []); +const EntityUsage: React.FC = ({ accessToken, entityType, entityId, entityList, dateValue }) => { + const { teams } = useTeams(); const [selectedTags, setSelectedTags] = useState([]); const [topKeysLimit, setTopKeysLimit] = useState(5); const [topModelsLimit, setTopModelsLimit] = useState(5); + const [topAgentsLimit, setTopAgentsLimit] = useState(5); - const fetchSpendData = async () => { - if (!accessToken || !dateValue.from || !dateValue.to) return; - // Create new Date objects to avoid mutating the original dates - const startTime = new Date(dateValue.from); - const endTime = new Date(dateValue.to); + const startTime = useMemo(() => (dateValue.from ? new Date(dateValue.from) : null), [dateValue.from]); + const endTime = useMemo(() => (dateValue.to ? new Date(dateValue.to) : null), [dateValue.to]); - if (entityType === "tag") { - const data = await tagDailyActivityCall( - accessToken, - startTime, - endTime, - 1, - selectedTags.length > 0 ? selectedTags : null, - ); - setSpendData(data); - } else if (entityType === "team") { - const data = await teamDailyActivityCall( - accessToken, - startTime, - endTime, - 1, - selectedTags.length > 0 ? selectedTags : null, - ); - setSpendData(data); - } else if (entityType === "organization") { - const data = await organizationDailyActivityCall( - accessToken, - startTime, - endTime, - 1, - selectedTags.length > 0 ? selectedTags : null, - ); - setSpendData(data); - } else if (entityType === "customer") { - const data = await customerDailyActivityCall( - accessToken, - startTime, - endTime, - 1, - selectedTags.length > 0 ? selectedTags : null, - ); - setSpendData(data); - } else if (entityType === "agent") { - const data = await agentDailyActivityCall( - accessToken, - startTime, - endTime, - 1, - selectedTags.length > 0 ? selectedTags : null, - ); - setSpendData(data); - } else if (entityType === "user") { - const data = await userDailyActivityCall( - accessToken, - startTime, - endTime, - 1, - selectedTags.length > 0 ? selectedTags[0] : null, - ); - setSpendData(data); - } else { - throw new Error("Invalid entity type"); - } - }; + const entityFilterArg = useMemo(() => { + if (entityType === "user") return selectedTags.length > 0 ? selectedTags[0] : null; + return selectedTags.length > 0 ? selectedTags : null; + }, [entityType, selectedTags]); - useEffect(() => { - fetchSpendData(); - }, [accessToken, dateValue, entityId, selectedTags]); + const fetchFn = ENTITY_FETCH_FNS[entityType]; + const enabled = !!accessToken && !!startTime && !!endTime; + + const { + data: spendDataRaw, + isFetchingMore, + progress, + cancelled, + cancel, + } = usePaginatedDailyActivity({ + fetchFn, + args: [accessToken, startTime, endTime, entityFilterArg], + enabled, + }); + + const spendData = spendDataRaw as unknown as EntitySpendData; + + const { + data: agentSpendDataRaw, + isFetchingMore: agentIsFetchingMore, + progress: agentProgress, + cancelled: agentCancelled, + cancel: agentCancel, + } = usePaginatedDailyActivity({ + fetchFn: agentDailyActivityCall, + args: [accessToken, startTime, endTime, null], + enabled: enabled && entityType === "team", + }); + + const agentSpendData = agentSpendDataRaw as unknown as EntitySpendData; + + const modelMetrics = processActivityData(spendData, "models", teams || []); + const keyMetrics = processActivityData(spendData, "api_keys", teams || []); + const agentMetrics = entityType === "team" ? processActivityData(agentSpendData, "entities", teams || []) : {}; const getTopModels = () => { const modelSpend: { [key: string]: any } = {}; @@ -209,6 +183,37 @@ const EntityUsage: React.FC = ({ accessToken, entityType, enti .slice(0, topModelsLimit); }; + const getTopAgents = () => { + const agentSpend: { [key: string]: any } = {}; + agentSpendData.results.forEach((day) => { + Object.entries(day.breakdown.entities || {}).forEach(([agentId, data]) => { + if (!agentSpend[agentId]) { + agentSpend[agentId] = { + spend: 0, + requests: 0, + successful_requests: 0, + failed_requests: 0, + tokens: 0, + agent_name: (data.metadata as any)?.agent_name || agentId, + }; + } + agentSpend[agentId].spend += data.metrics.spend; + agentSpend[agentId].requests += data.metrics.api_requests; + agentSpend[agentId].successful_requests += data.metrics.successful_requests; + agentSpend[agentId].failed_requests += data.metrics.failed_requests; + agentSpend[agentId].tokens += data.metrics.total_tokens; + }); + }); + + return Object.entries(agentSpend) + .map(([agentId, metrics]) => ({ + key: metrics.agent_name, + ...metrics, + })) + .sort((a, b) => b.spend - a.spend) + .slice(0, topAgentsLimit); + }; + const getTopAPIKeys = () => { console.log("debugTags", { spendData }); const keySpend: { [key: string]: KeyMetricWithMetadata } = {}; @@ -391,6 +396,78 @@ const EntityUsage: React.FC = ({ accessToken, entityType, enti return (
+ {isFetchingMore && ( + + + + Currently fetching spend data: fetched {progress.currentPage} / {progress.totalPages} pages. Charts will + update periodically as data loads. Moving off of this page will stop and reset this. To continue using + the UI in the meantime,{" "} + + open a new tab + + . + + +
+ } + /> + )} + {cancelled && ( + + Showing partial data ({progress.currentPage}/{progress.totalPages} pages loaded) + + } + /> + )} + {agentIsFetchingMore && entityType === "team" && ( + + + + Currently fetching agent data: fetched {agentProgress.currentPage} / {agentProgress.totalPages} pages. + Charts will update periodically as data loads. Moving off of this page will stop and reset this. To + continue using the UI in the meantime,{" "} + + open a new tab + + . + + + + } + /> + )} + {agentCancelled && entityType === "team" && ( + + Showing partial agent data ({agentProgress.currentPage}/{agentProgress.totalPages} pages loaded) + + } + /> + )} = ({ accessToken, entityType, enti Cost {entityType === "agent" ? "Request / Token Consumption" : "Model Activity"} + {entityType === "team" ? Agent Activity : <>} Key Activity Endpoint Activity @@ -621,6 +699,20 @@ const EntityUsage: React.FC = ({ accessToken, entityType, enti + {/* Top Agents - only for team entity type */} + {entityType === "team" && ( + + + Top Agents Driving Spend + + + + )} + {/* Spend by Provider */} @@ -696,6 +788,13 @@ const EntityUsage: React.FC = ({ accessToken, entityType, enti + {entityType === "team" ? ( + + + + ) : ( + <> + )} diff --git a/ui/litellm-dashboard/src/components/UsagePage/components/UsagePageView.test.tsx b/ui/litellm-dashboard/src/components/UsagePage/components/UsagePageView.test.tsx index b9fe1687e6c..bbcddd572cd 100644 --- a/ui/litellm-dashboard/src/components/UsagePage/components/UsagePageView.test.tsx +++ b/ui/litellm-dashboard/src/components/UsagePage/components/UsagePageView.test.tsx @@ -251,6 +251,7 @@ vi.mock("@ant-design/icons", async () => { UserOutlined: Icon, DownOutlined: Icon, RightOutlined: Icon, + ExportOutlined: Icon, LoadingOutlined, }; }); diff --git a/ui/litellm-dashboard/src/components/UsagePage/components/UsagePageView.tsx b/ui/litellm-dashboard/src/components/UsagePage/components/UsagePageView.tsx index e81e4ceaa36..1495c7d3e5b 100644 --- a/ui/litellm-dashboard/src/components/UsagePage/components/UsagePageView.tsx +++ b/ui/litellm-dashboard/src/components/UsagePage/components/UsagePageView.tsx @@ -6,7 +6,8 @@ * Works at 1m+ spend logs, by querying an aggregate table instead. */ -import { DownOutlined, InfoCircleOutlined, LoadingOutlined, RightOutlined } from "@ant-design/icons"; +import { DownOutlined, ExportOutlined, InfoCircleOutlined, LoadingOutlined, RightOutlined } from "@ant-design/icons"; +import { useDebouncedState } from "@tanstack/react-pacer/debouncer"; import { BarChart, Card, @@ -19,11 +20,10 @@ import { TabPanel, TabPanels, Text, - Title + Title, } from "@tremor/react"; -import { Alert, Segmented, Select, Tooltip, Typography } from "antd"; -import { useDebouncedState } from "@tanstack/react-pacer/debouncer"; -import React, { useCallback, useEffect, useMemo, useState, type UIEvent } from "react"; +import { Alert, Button, Segmented, Select, Tooltip, Typography } from "antd"; +import React, { useCallback, useEffect, useMemo, useRef, useState, type UIEvent } from "react"; import { useAgents } from "@/app/(dashboard)/hooks/agents/useAgents"; import { useCustomers } from "@/app/(dashboard)/hooks/customers/useCustomers"; @@ -31,7 +31,6 @@ import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized"; import { useCurrentUser } from "@/app/(dashboard)/hooks/users/useCurrentUser"; import { useInfiniteUsers } from "@/app/(dashboard)/hooks/users/useUsers"; import { formatNumberWithCommas } from "@/utils/dataUtils"; -import { Button } from "@tremor/react"; import { all_admin_roles } from "../../../utils/roles"; import { ActivityMetrics, processActivityData } from "../../activity_metrics"; import CloudZeroExportModal from "../../cloudzero_export_modal"; @@ -43,14 +42,15 @@ import { ChartLoader } from "../../shared/chart_loader"; import { Tag } from "../../tag_management/types"; import UserAgentActivity from "../../user_agent_activity"; import ViewUserSpend from "../../view_user_spend"; +import { usePaginatedDailyActivity } from "../hooks/usePaginatedDailyActivity"; import { DailyData, KeyMetricWithMetadata, MetricWithMetadata } from "../types"; import { valueFormatterSpend } from "../utils/value_formatters"; import EndpointUsage from "./EndpointUsage/EndpointUsage"; import EntityUsage, { EntityList } from "./EntityUsage/EntityUsage"; import SpendByProvider from "./EntityUsage/SpendByProvider"; import TopKeyView from "./EntityUsage/TopKeyView"; -import { UsageOption, UsageViewSelect } from "./UsageViewSelect/UsageViewSelect"; import UsageAIChatPanel from "./UsageAIChatPanel"; +import { UsageOption, UsageViewSelect } from "./UsageViewSelect/UsageViewSelect"; interface UsagePageProps { teams: Team[]; @@ -59,13 +59,12 @@ interface UsagePageProps { const UsagePage: React.FC = ({ teams, organizations }) => { const { accessToken, userRole, userId: userID, premiumUser } = useAuthorized(); - const [userSpendData, setUserSpendData] = useState<{ - results: DailyData[]; - metadata: any; - }>({ results: [], metadata: {} }); + // Aggregated endpoint: try first, fall back to paginated if unavailable + const [aggregatedData, setAggregatedData] = useState<{ results: DailyData[]; metadata: any } | null>(null); + const [aggregatedFailed, setAggregatedFailed] = useState(false); + const [aggregatedLoading, setAggregatedLoading] = useState(false); // Separate loading states for better UX - const [loading, setLoading] = useState(false); const [isDateChanging, setIsDateChanging] = useState(false); // Create initial dates outside of state to prevent recreation @@ -128,8 +127,7 @@ const UsagePage: React.FC = ({ teams, organizations }) => { const handleUserPopupScroll = (e: UIEvent) => { const target = e.currentTarget; - const scrollRatio = - (target.scrollTop + target.clientHeight) / target.scrollHeight; + const scrollRatio = (target.scrollTop + target.clientHeight) / target.scrollHeight; if (scrollRatio >= 0.8 && hasNextUsersPage && !isFetchingNextUsersPage) { fetchNextUsersPage(); } @@ -137,9 +135,7 @@ const UsagePage: React.FC = ({ teams, organizations }) => { // For admins: null means global view (all users), a string means filter by that user // For non-admins: always set to their own user ID - const [selectedUserId, setSelectedUserId] = useState( - isAdmin ? null : (userID || null) - ); + const [selectedUserId, setSelectedUserId] = useState(isAdmin ? null : userID || null); const [modelViewType, setModelViewType] = useState<"groups" | "individual">("groups"); const [isCloudZeroModalOpen, setIsCloudZeroModalOpen] = useState(false); const [isGlobalExportModalOpen, setIsGlobalExportModalOpen] = useState(false); @@ -173,6 +169,67 @@ const UsagePage: React.FC = ({ teams, organizations }) => { } }, [isAdmin, userID]); + // For non-admins, always pass their own user_id + const effectiveUserId = isAdmin ? selectedUserId : userID || null; + + const startTime = useMemo(() => (dateValue.from ? new Date(dateValue.from) : null), [dateValue.from]); + const endTime = useMemo(() => (dateValue.to ? new Date(dateValue.to) : null), [dateValue.to]); + + // Try aggregated endpoint first, fall back to paginated on failure + const aggregatedFetchIdRef = useRef(0); + useEffect(() => { + if (!accessToken || !startTime || !endTime) return; + const fetchId = ++aggregatedFetchIdRef.current; + setAggregatedLoading(true); + setAggregatedFailed(false); + setAggregatedData(null); + + userDailyActivityAggregatedCall(accessToken, startTime, endTime, effectiveUserId) + .then((data) => { + if (aggregatedFetchIdRef.current !== fetchId) return; + setAggregatedData(data); + setAggregatedLoading(false); + setIsDateChanging(false); + }) + .catch(() => { + if (aggregatedFetchIdRef.current !== fetchId) return; + setAggregatedFailed(true); + setAggregatedLoading(false); + }); + }, [accessToken, startTime, endTime, effectiveUserId]); + + // Paginated fallback — only enabled when aggregated endpoint fails + const paginatedResult = usePaginatedDailyActivity({ + fetchFn: userDailyActivityCall, + args: [accessToken, startTime, endTime, effectiveUserId], + enabled: aggregatedFailed && !!accessToken && !!startTime && !!endTime, + }); + + // Derive userSpendData from whichever source is active + const userSpendData = useMemo(() => { + if (aggregatedData) return aggregatedData; + if (aggregatedFailed) return paginatedResult.data; + return { results: [] as DailyData[], metadata: {} as any }; + }, [aggregatedData, aggregatedFailed, paginatedResult.data]); + + const loading = aggregatedLoading || paginatedResult.loading; + + // Clear isDateChanging when paginated data starts arriving + useEffect(() => { + if (aggregatedFailed && !paginatedResult.loading && paginatedResult.data.results.length > 0) { + setIsDateChanging(false); + } + }, [aggregatedFailed, paginatedResult.loading, paginatedResult.data.results.length]); + + // Super responsive date change handler + const handleDateChange = useCallback((newValue: DateRangePickerValue) => { + // Instant visual feedback + setIsDateChanging(true); + + // Update date immediately for UI responsiveness + setDateValue(newValue); + }, []); + // Derived states from userSpendData const totalSpend = userSpendData.metadata?.total_spend || 0; @@ -300,7 +357,8 @@ const UsagePage: React.FC = ({ teams, organizations }) => { providerSpendMap[provider].metrics.successful_requests += metrics.metrics.successful_requests || 0; providerSpendMap[provider].metrics.failed_requests += metrics.metrics.failed_requests || 0; providerSpendMap[provider].metrics.cache_read_input_tokens += metrics.metrics.cache_read_input_tokens || 0; - providerSpendMap[provider].metrics.cache_creation_input_tokens += metrics.metrics.cache_creation_input_tokens || 0; + providerSpendMap[provider].metrics.cache_creation_input_tokens += + metrics.metrics.cache_creation_input_tokens || 0; }); }); @@ -362,494 +420,415 @@ const UsagePage: React.FC = ({ teams, organizations }) => { .slice(0, topKeysLimit); }, [userSpendData.results, topKeysLimit]); - const fetchUserSpendData = useCallback(async () => { - if (!accessToken || !dateValue.from || !dateValue.to) return; - - // For non-admins, always pass their own user_id - const effectiveUserId = isAdmin ? selectedUserId : (userID || null); - - setLoading(true); - - // Create new Date objects to avoid mutating the original dates - const startTime = new Date(dateValue.from); - const endTime = new Date(dateValue.to); - - try { - // Prefer aggregated endpoint to avoid many page requests - try { - const aggregated = await userDailyActivityAggregatedCall(accessToken, startTime, endTime, effectiveUserId); - setUserSpendData(aggregated); - return; - } catch (e) { - // Fallback to paginated calls if aggregated endpoint is unavailable - } - - const firstPageData = await userDailyActivityCall(accessToken, startTime, endTime, 1, effectiveUserId); - - if (firstPageData.metadata.total_pages <= 1) { - setUserSpendData(firstPageData); - return; - } - - const allResults = [...firstPageData.results]; - const aggregatedMetadata = { ...firstPageData.metadata }; - - for (let page = 2; page <= firstPageData.metadata.total_pages; page++) { - const pageData = await userDailyActivityCall(accessToken, startTime, endTime, page, effectiveUserId); - allResults.push(...pageData.results); - if (pageData.metadata) { - aggregatedMetadata.total_spend = (aggregatedMetadata.total_spend || 0) + (pageData.metadata.total_spend || 0); - aggregatedMetadata.total_api_requests = (aggregatedMetadata.total_api_requests || 0) + (pageData.metadata.total_api_requests || 0); - aggregatedMetadata.total_successful_requests = (aggregatedMetadata.total_successful_requests || 0) + (pageData.metadata.total_successful_requests || 0); - aggregatedMetadata.total_failed_requests = (aggregatedMetadata.total_failed_requests || 0) + (pageData.metadata.total_failed_requests || 0); - aggregatedMetadata.total_tokens = (aggregatedMetadata.total_tokens || 0) + (pageData.metadata.total_tokens || 0); - aggregatedMetadata.total_prompt_tokens = (aggregatedMetadata.total_prompt_tokens || 0) + (pageData.metadata.total_prompt_tokens || 0); - aggregatedMetadata.total_completion_tokens = (aggregatedMetadata.total_completion_tokens || 0) + (pageData.metadata.total_completion_tokens || 0); - aggregatedMetadata.total_cache_read_input_tokens = (aggregatedMetadata.total_cache_read_input_tokens || 0) + (pageData.metadata.total_cache_read_input_tokens || 0); - aggregatedMetadata.total_cache_creation_input_tokens = (aggregatedMetadata.total_cache_creation_input_tokens || 0) + (pageData.metadata.total_cache_creation_input_tokens || 0); - } - } - - setUserSpendData({ - results: allResults, - metadata: aggregatedMetadata, - }); - } catch (error) { - console.error("Error fetching user spend data:", error); - } finally { - setLoading(false); - setIsDateChanging(false); - } - }, [accessToken, dateValue.from, dateValue.to, selectedUserId, isAdmin, userID]); - - // Super responsive date change handler - const handleDateChange = useCallback((newValue: DateRangePickerValue) => { - // Instant visual feedback - setIsDateChanging(true); - setLoading(true); - - // Update date immediately for UI responsiveness - setDateValue(newValue); - }, []); - - // Debounced effect for data fetching with shorter delay - useEffect(() => { - if (!dateValue.from || !dateValue.to) return; - - const timeoutId = setTimeout(() => { - fetchUserSpendData(); - }, 50); // Very short debounce - - return () => clearTimeout(timeoutId); - }, [fetchUserSpendData]); - const sortedDailyResults = useMemo( () => [...userSpendData.results].sort((a, b) => new Date(a.date).getTime() - new Date(b.date).getTime()), [userSpendData.results], ); const modelMetrics = useMemo(() => processActivityData(userSpendData, "models", teams), [userSpendData, teams]); const keyMetrics = useMemo(() => processActivityData(userSpendData, "api_keys", teams), [userSpendData, teams]); - const mcpServerMetrics = useMemo(() => processActivityData(userSpendData, "mcp_servers", teams), [userSpendData, teams]); + const mcpServerMetrics = useMemo( + () => processActivityData(userSpendData, "mcp_servers", teams), + [userSpendData, teams], + ); return (
- {/* Export Data Button - Positioned in top right corner */} - {/* {all_admin_roles.includes(userRole || "") && ( -
- -
- )} */} - {/* Global Date Picker and Tabs - Single Row */}
- setUsageView(value)} - isAdmin={isAdmin} - /> + setUsageView(value)} isAdmin={isAdmin} />
+ {paginatedResult.isFetchingMore && ( + + + + Currently fetching spend data: fetched {paginatedResult.progress.currentPage} /{" "} + {paginatedResult.progress.totalPages} pages. Charts will update periodically as data loads. Moving + off of this page will stop and reset this. To continue using the UI in the meantime,{" "} + + open a new tab + + . + + +
+ } + /> + )} + {paginatedResult.cancelled && ( + + Showing partial data ({paginatedResult.progress.currentPage}/{paginatedResult.progress.totalPages}{" "} + pages loaded) + + } + /> + )} {/* Your Usage Panel */} {usageView === "global" && ( <> - {isAdmin && ( -
- Filter by user - setSelectedUserId(value ?? null)} + filterOption={false} + onSearch={handleUserSearchChange} + searchValue={userSearchInput} + onPopupScroll={handleUserPopupScroll} + loading={isLoadingUsers} + notFoundContent={isLoadingUsers ? : "No users found"} + options={userOptions} + popupRender={(menu) => ( + <> + {menu} + {isFetchingNextUsersPage && ( +
+ +
+ )} + )} - > - Ask AI - - + />
-
- - {/* Cost Panel */} - - - {/* Total Spend Card */} - -
- - Project Spend{" "} - {dateValue.from && dateValue.to && ( - <> - {dateValue.from.toLocaleDateString("en-US", { - month: "short", - day: "numeric", - year: dateValue.from.getFullYear() !== dateValue.to.getFullYear() ? "numeric" : undefined, - })} - {" - "} - {dateValue.to.toLocaleDateString("en-US", { - month: "short", - day: "numeric", - year: "numeric", - })} - - )} - -
+ )} + +
+ + Cost + Model Activity + Key Activity + MCP Server Activity + Endpoint Activity + +
+ + +
+
+ + {/* Cost Panel */} + + + {/* Total Spend Card */} + +
+ + Project Spend{" "} + {dateValue.from && dateValue.to && ( + <> + {dateValue.from.toLocaleDateString("en-US", { + month: "short", + day: "numeric", + year: + dateValue.from.getFullYear() !== dateValue.to.getFullYear() ? "numeric" : undefined, + })} + {" - "} + {dateValue.to.toLocaleDateString("en-US", { + month: "short", + day: "numeric", + year: "numeric", + })} + + )} + +
- - + + - - - Usage Metrics - - - Total Requests - - {userSpendData.metadata?.total_api_requests?.toLocaleString() || 0} - - - - Successful Requests - - {userSpendData.metadata?.total_successful_requests?.toLocaleString() || 0} - - - -
- Failed Requests - - - -
- - {userSpendData.metadata?.total_failed_requests?.toLocaleString() || 0} - - - - Average Cost per Request - - $ - {formatNumberWithCommas( - (totalSpend || 0) / (userSpendData.metadata?.total_api_requests || 1), - 4, - )} - - - setShowTokenBreakdown(!showTokenBreakdown)} - > -
- Total Tokens - {showTokenBreakdown ? ( - - ) : ( - - )} -
- - {userSpendData.metadata?.total_tokens?.toLocaleString() || 0} - - - - {showTokenBreakdown && ( - + + + Usage Metrics + - Input Tokens - - {userSpendData.metadata?.total_prompt_tokens?.toLocaleString() || 0} + Total Requests + + {userSpendData.metadata?.total_api_requests?.toLocaleString() || 0} - Output Tokens - - {userSpendData.metadata?.total_completion_tokens?.toLocaleString() || 0} - - - - Cache Read Tokens + Successful Requests - {userSpendData.metadata?.total_cache_read_input_tokens?.toLocaleString() || 0} + {userSpendData.metadata?.total_successful_requests?.toLocaleString() || 0} - Cache Write Tokens - - {userSpendData.metadata?.total_cache_creation_input_tokens?.toLocaleString() || 0} +
+ Failed Requests + + + +
+ + {userSpendData.metadata?.total_failed_requests?.toLocaleString() || 0} + + + + Average Cost per Request + + $ + {formatNumberWithCommas( + (totalSpend || 0) / (userSpendData.metadata?.total_api_requests || 1), + 4, + )} + + + setShowTokenBreakdown(!showTokenBreakdown)} + > +
+ Total Tokens + {showTokenBreakdown ? ( + + ) : ( + + )} +
+ + {userSpendData.metadata?.total_tokens?.toLocaleString() || 0} - )} - - + {showTokenBreakdown && ( + + + Input Tokens + + {userSpendData.metadata?.total_prompt_tokens?.toLocaleString() || 0} + + + + Output Tokens + + {userSpendData.metadata?.total_completion_tokens?.toLocaleString() || 0} + + + + Cache Read Tokens + + {userSpendData.metadata?.total_cache_read_input_tokens?.toLocaleString() || 0} + + + + Cache Write Tokens + + {userSpendData.metadata?.total_cache_creation_input_tokens?.toLocaleString() || 0} + + + + )} + + - {/* Daily Spend Chart */} - - - Daily Spend - {loading ? ( - - ) : ( - { - if (!active || !payload?.[0]) return null; - const data = payload[0].payload; - return ( -
-

{data.date}

-

- Spend: ${formatNumberWithCommas(data.metrics.spend, 2)} -

-

Requests: {data.metrics.api_requests}

-

Successful: {data.metrics.successful_requests}

-

Failed: {data.metrics.failed_requests}

-

Tokens: {data.metrics.total_tokens}

-
- ); - }} + {/* Daily Spend Chart */} + + + Daily Spend + {loading ? ( + + ) : ( + { + if (!active || !payload?.[0]) return null; + const data = payload[0].payload; + return ( +
+

{data.date}

+

+ Spend: ${formatNumberWithCommas(data.metrics.spend, 2)} +

+

Requests: {data.metrics.api_requests}

+

Successful: {data.metrics.successful_requests}

+

Failed: {data.metrics.failed_requests}

+

Tokens: {data.metrics.total_tokens}

+
+ ); + }} + /> + )} + + + {/* Top API Keys */} + + + Top Virtual Keys + - )} - - - {/* Top API Keys */} - - - Top Virtual Keys - + + + {/* Top Models */} + + + {modelViewType === "groups" ? "Top Public Model Names" : "Top Litellm Models"} +
+ setTopModelsLimit(value as number)} + /> +
+ + +
+
+ {loading ? ( + + ) : ( +
+ {(() => { + const modelData = modelViewType === "groups" ? topModelGroups : topModels; + return ( + { + if (!active || !payload?.[0]) return null; + const data = payload[0].payload; + return ( +
+

{data.key}

+

+ Spend: ${formatNumberWithCommas(data.spend, 2)} +

+

+ Total Requests: {data.requests.toLocaleString()} +

+

+ Successful: {data.successful_requests.toLocaleString()} +

+

+ Failed: {data.failed_requests.toLocaleString()} +

+

Tokens: {data.tokens.toLocaleString()}

+
+ ); + }} + /> + ); + })()} +
+ )} + + + + {/* Spend by Provider */} + + - - + - {/* Top Models */} - - - {modelViewType === "groups" ? "Top Public Model Names" : "Top Litellm Models"} -
- setTopModelsLimit(value as number)} - /> -
- - -
-
- {loading ? ( - - ) : ( -
- {(() => { - const modelData = - modelViewType === "groups" - ? topModelGroups - : topModels; - return ( - { - if (!active || !payload?.[0]) return null; - const data = payload[0].payload; - return ( -
-

{data.key}

-

Spend: ${formatNumberWithCommas(data.spend, 2)}

-

- Total Requests: {data.requests.toLocaleString()} -

-

- Successful: {data.successful_requests.toLocaleString()} -

-

Failed: {data.failed_requests.toLocaleString()}

-

Tokens: {data.tokens.toLocaleString()}

-
- ); - }} - /> - ); - })()} -
- )} - - + {/* Usage Metrics */} + + - {/* Spend by Provider */} - - - - - {/* Usage Metrics */} - - - - {/* Activity Panel */} - - - - - - - - - - - - - - + {/* Activity Panel */} + + + + + + + + + + + + + + )} {/* Organization Usage Panel */} @@ -991,11 +970,7 @@ const UsagePage: React.FC = ({ teams, organizations }) => { /> {/* AI Chat Panel */} - setIsAiChatOpen(false)} - accessToken={accessToken} - /> + setIsAiChatOpen(false)} accessToken={accessToken} />
); }; diff --git a/ui/litellm-dashboard/src/components/UsagePage/hooks/usePaginatedDailyActivity.ts b/ui/litellm-dashboard/src/components/UsagePage/hooks/usePaginatedDailyActivity.ts new file mode 100644 index 00000000000..9a7ab22c9af --- /dev/null +++ b/ui/litellm-dashboard/src/components/UsagePage/hooks/usePaginatedDailyActivity.ts @@ -0,0 +1,246 @@ +import { useCallback, useEffect, useRef, useState } from "react"; +import { DailyData } from "../types"; + +export interface PaginationProgress { + currentPage: number; + totalPages: number; +} + +/** Delay between sequential page fetches (ms) to avoid overloading the backend. */ +const PAGE_FETCH_DELAY_MS = 300; + +/** Number of pages to accumulate before flushing to React state (reduces re-renders). */ +const RENDER_BATCH_SIZE = 3; + +/** The metadata fields returned by the daily activity API that should be summed across pages. */ +const SUMMABLE_METADATA_KEYS = [ + "total_spend", + "total_prompt_tokens", + "total_completion_tokens", + "total_tokens", + "total_api_requests", + "total_successful_requests", + "total_failed_requests", + "total_cache_read_input_tokens", + "total_cache_creation_input_tokens", +] as const; + +interface DailyActivityResponse { + results: DailyData[]; + metadata: Record; +} + +type FetchPageFn = (...args: any[]) => Promise; + +interface UsePaginatedDailyActivityParams { + /** The API call function (e.g., userDailyActivityCall). */ + fetchFn: FetchPageFn; + /** Arguments to pass to fetchFn: [accessToken, startTime, endTime, ...extraArgs]. Page is injected by the hook at index 3. */ + args: any[]; + /** Whether the hook should fetch. Set to false to disable. */ + enabled: boolean; +} + +interface UsePaginatedDailyActivityReturn { + data: DailyActivityResponse; + loading: boolean; + isFetchingMore: boolean; + progress: PaginationProgress; + cancelled: boolean; + cancel: () => void; +} + +const EMPTY_DATA: DailyActivityResponse = { + results: [], + metadata: { + total_spend: 0, + total_prompt_tokens: 0, + total_completion_tokens: 0, + total_tokens: 0, + total_api_requests: 0, + total_successful_requests: 0, + total_failed_requests: 0, + total_cache_read_input_tokens: 0, + total_cache_creation_input_tokens: 0, + total_pages: 1, + has_more: false, + page: 1, + }, +}; + +function sumMetadata( + a: Record, + b: Record, +): Record { + const result = { ...a }; + for (const key of SUMMABLE_METADATA_KEYS) { + result[key] = (a[key] || 0) + (b[key] || 0); + } + return result; +} + +/** + * Hook that auto-paginates daily activity endpoints, updating state in batches + * so charts render progressively. Cancels on unmount, param changes, or + * manual cancel(). + * + * The `args` array should contain every argument the fetchFn expects EXCEPT + * the `page` parameter. The hook injects `page` as the 4th argument (index 3), + * matching the signature of all daily activity calls: + * (accessToken, startTime, endTime, page, ...rest) + */ +export function usePaginatedDailyActivity({ + fetchFn, + args, + enabled, +}: UsePaginatedDailyActivityParams): UsePaginatedDailyActivityReturn { + const [data, setData] = useState(EMPTY_DATA); + const [loading, setLoading] = useState(false); + const [isFetchingMore, setIsFetchingMore] = useState(false); + const [progress, setProgress] = useState({ + currentPage: 0, + totalPages: 0, + }); + const [cancelled, setCancelled] = useState(false); + + const fetchIdRef = useRef(0); + const cancelledRef = useRef(false); + const delayTimerRef = useRef | null>(null); + + // Keep args in a ref so the effect can always read the latest values + // without needing them in the dependency array. + const argsRef = useRef(args); + argsRef.current = args; + + // Stable serialised key so the effect only re-runs when the arg *values* change. + const argsKey = JSON.stringify(args); + + const cancel = useCallback(() => { + cancelledRef.current = true; + setCancelled(true); + setIsFetchingMore(false); + if (delayTimerRef.current !== null) { + clearTimeout(delayTimerRef.current); + delayTimerRef.current = null; + } + }, []); + + useEffect(() => { + if (!enabled) { + setData(EMPTY_DATA); + setLoading(false); + setIsFetchingMore(false); + setProgress({ currentPage: 0, totalPages: 0 }); + setCancelled(false); + return; + } + + const currentFetchId = ++fetchIdRef.current; + cancelledRef.current = false; + setCancelled(false); + + const isStale = () => + fetchIdRef.current !== currentFetchId || cancelledRef.current; + + /** Cancellable delay that clears itself on cleanup. */ + const delay = (ms: number) => + new Promise((resolve) => { + delayTimerRef.current = setTimeout(() => { + delayTimerRef.current = null; + resolve(); + }, ms); + }); + + const run = async () => { + const currentArgs = argsRef.current; + setLoading(true); + setIsFetchingMore(false); + setProgress({ currentPage: 1, totalPages: 1 }); + + try { + // Inject page=1 as the 4th argument. + const argsWithPage = [...currentArgs.slice(0, 3), 1, ...currentArgs.slice(3)]; + const firstPage = await fetchFn(...argsWithPage); + + if (isStale()) return; + + setData(firstPage); + + const totalPages = firstPage.metadata?.total_pages || 1; + + setProgress({ currentPage: 1, totalPages }); + + if (totalPages <= 1) { + setLoading(false); + return; + } + + // More pages — start fetching sequentially. + setLoading(false); + setIsFetchingMore(true); + + let accumulatedResults = [...firstPage.results]; + let accumulatedMetadata = { ...firstPage.metadata }; + + for (let page = 2; page <= totalPages; page++) { + if (isStale()) return; + + // Small delay to avoid overwhelming the backend. + await delay(PAGE_FETCH_DELAY_MS); + + if (isStale()) return; + + const argsForPage = [...currentArgs.slice(0, 3), page, ...currentArgs.slice(3)]; + const pageData = await fetchFn(...argsForPage); + + if (isStale()) return; + + accumulatedResults = [...accumulatedResults, ...pageData.results]; + accumulatedMetadata = sumMetadata( + accumulatedMetadata, + pageData.metadata, + ); + accumulatedMetadata.total_pages = totalPages; + accumulatedMetadata.has_more = page < totalPages; + accumulatedMetadata.page = page; + + // Flush accumulated data and progress to React state every + // RENDER_BATCH_SIZE pages (or on the final page) to avoid + // expensive per-page re-renders. Progress and data are updated + // together so the counter never appears to decrement. + const isLastPage = page === totalPages; + const isBatchBoundary = (page - 1) % RENDER_BATCH_SIZE === 0; + if (isLastPage || isBatchBoundary) { + setData({ + results: accumulatedResults, + metadata: accumulatedMetadata, + }); + setProgress({ currentPage: page, totalPages }); + } + } + + setIsFetchingMore(false); + } catch (error) { + if (!isStale()) { + console.error("Error fetching daily activity:", error); + setLoading(false); + setIsFetchingMore(false); + } + } + }; + + run(); + + return () => { + fetchIdRef.current++; + if (delayTimerRef.current !== null) { + clearTimeout(delayTimerRef.current); + delayTimerRef.current = null; + } + }; + // argsKey is a stable JSON string so the effect only re-fires when arg values change. + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [enabled, fetchFn, argsKey]); + + return { data, loading, isFetchingMore, progress, cancelled, cancel }; +} diff --git a/ui/litellm-dashboard/src/components/VirtualKeysPage/VirtualKeysTable.test.tsx b/ui/litellm-dashboard/src/components/VirtualKeysPage/VirtualKeysTable.test.tsx index d63154f0cef..e6d144df65a 100644 --- a/ui/litellm-dashboard/src/components/VirtualKeysPage/VirtualKeysTable.test.tsx +++ b/ui/litellm-dashboard/src/components/VirtualKeysPage/VirtualKeysTable.test.tsx @@ -275,8 +275,8 @@ it("should display user email correctly", async () => { }); }); -it("should show skeleton loaders when isLoading is true", () => { - // Mock loading state +it("should show loading message only on initial load (isPending)", () => { + // Mock initial loading state mockUseKeys.mockReturnValue({ data: null, isPending: true, @@ -296,7 +296,7 @@ it("should show skeleton loaders when isLoading is true", () => { renderWithProviders(); - // Check that loading message is shown + // Check that loading message is shown on initial load expect(screen.getByText("🚅 Loading keys...")).toBeInTheDocument(); // Check that actual key data is not shown @@ -898,3 +898,79 @@ describe("pagination display – total count and page count", () => { }); }); }); + +describe("refetch button", () => { + it("should show Fetch button in normal state", () => { + renderWithProviders(); + + const fetchButton = screen.getByTitle("Fetch data"); + expect(fetchButton).toBeInTheDocument(); + expect(fetchButton).not.toBeDisabled(); + expect(screen.getByText("Fetch")).toBeInTheDocument(); + }); + + it("should show Fetching state and keep table data visible during refetch", () => { + mockUseKeys.mockReturnValue({ + data: { + keys: [mockKey], + total_count: 1, + current_page: 1, + total_pages: 1, + } as KeysResponse, + isPending: false, + isFetching: true, + refetch: vi.fn(), + } as any); + + renderWithProviders(); + + // Button should show "Fetching" and be disabled + expect(screen.getByText("Fetching")).toBeInTheDocument(); + const fetchButton = screen.getByTitle("Fetch data"); + expect(fetchButton).toBeDisabled(); + + // Table data should still be visible (stale data) + expect(screen.getByText("Test Key Alias")).toBeInTheDocument(); + + // "Loading keys..." should NOT appear during refetch + expect(screen.queryByText("🚅 Loading keys...")).not.toBeInTheDocument(); + }); + + it("should call refetch when Fetch button is clicked", () => { + const mockRefetch = vi.fn(); + mockUseKeys.mockReturnValue({ + data: { + keys: [mockKey], + total_count: 1, + current_page: 1, + total_pages: 1, + } as KeysResponse, + isPending: false, + isFetching: false, + refetch: mockRefetch, + } as any); + + renderWithProviders(); + + const fetchButton = screen.getByTitle("Fetch data"); + fireEvent.click(fetchButton); + + expect(mockRefetch).toHaveBeenCalledTimes(1); + }); + + it("should show Fetch button enabled on error so user can retry", () => { + mockUseKeys.mockReturnValue({ + data: null, + isPending: false, + isFetching: false, + isError: true, + refetch: vi.fn(), + } as any); + + renderWithProviders(); + + const fetchButton = screen.getByTitle("Fetch data"); + expect(fetchButton).not.toBeDisabled(); + expect(screen.getByText("Fetch")).toBeInTheDocument(); + }); +}); diff --git a/ui/litellm-dashboard/src/components/VirtualKeysPage/VirtualKeysTable.tsx b/ui/litellm-dashboard/src/components/VirtualKeysPage/VirtualKeysTable.tsx index 60917941704..b30d4b6ce5b 100644 --- a/ui/litellm-dashboard/src/components/VirtualKeysPage/VirtualKeysTable.tsx +++ b/ui/litellm-dashboard/src/components/VirtualKeysPage/VirtualKeysTable.tsx @@ -85,6 +85,7 @@ export function VirtualKeysTable({ teams, organizations, onSortChange, currentSo data: keys, isPending: isLoading, isFetching, + isError, refetch, } = useKeys(tablePagination.pageIndex + 1, tablePagination.pageSize, { sortBy: sortBy || undefined, @@ -102,6 +103,15 @@ export function VirtualKeysTable({ teams, organizations, onSortChange, currentSo organizations, }); + // Defer the transition so the button stays in loading state until the table + // has rendered with the new data (mirrors the spend-logs pattern) + const isFetchingDeferred = useDeferredValue(isFetching); + const isButtonLoading = (isFetching || isFetchingDeferred) && !isError; + + const handleRefresh = () => { + refetch(); + }; + const totalCount = filteredTotalCount ?? keys?.total_count ?? 0; // Add a useEffect to call refresh when a key is created @@ -684,16 +694,28 @@ export function VirtualKeysTable({ teams, organizations, onSortChange, currentSo
- {isLoading || isFetching ? ( - - ) : ( - - Showing {rangeLabel} of {totalCount} results - - )} +
+ {isLoading ? ( + + ) : ( + + Showing {rangeLabel} of {totalCount} results + + )} + + } + onClick={handleRefresh} + disabled={isButtonLoading} + title="Fetch data" + > + {isButtonLoading ? "Fetching" : "Fetch"} + +
- {isLoading || isFetching ? ( + {isLoading ? ( ) : ( @@ -701,24 +723,24 @@ export function VirtualKeysTable({ teams, organizations, onSortChange, currentSo )} - {isLoading || isFetching ? ( + {isLoading ? ( ) : ( )} - {isLoading || isFetching ? ( + {isLoading ? ( ) : ( + + +
+ +
+ +
+
Session Info
+
+ {[['token', tokenPreview], ['ice', iceState], ['conn', connState], ['data ch.', dcState]].map(([k, v]) => ( +
{k}{v}
+ ))} +
+
+
+ + {/* Right panel */} +
+
+ WEBRTC REALTIME TESTER +
+
+ {status} +
+
+ +
+ {['logs','sdp','audio'].map(t => ( +
setActiveTab(t)}> + {t.toUpperCase()} +
+ ))} +
+ + {/* Logs */} +
+
+ {entries.length === 0 + ?
📡
Hit "Start Session" to begin
+ : entries.map(e => ( +
+ {e.time} + [{e.tag}] + {e.msg} +
+ )) + } +
+
+ + {/* SDP */} +
+
+
+
SDP OFFER
+