diff --git a/.circleci/config.yml b/.circleci/config.yml index 12e3cb1f6b6..d4943b59c9a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -42,7 +42,7 @@ commands: "pydantic==2.11.0" "mcp==1.25.0" "requests-mock>=1.12.1" \ "responses==0.25.7" "pytest-xdist==3.6.1" "pytest-timeout==2.2.0" \ "pytest-cov==5.0.0" "semantic_router==0.1.10" "fastapi-offline==1.7.3" \ - "a2a" + "a2a" "parameterized>=0.9.0" - setup_litellm_enterprise_pip - save_cache: paths: @@ -406,119 +406,6 @@ jobs: # Store test results - store_test_results: path: test-results - caching_unit_tests: - docker: - - image: cimg/python:3.11 - auth: - username: ${DOCKERHUB_USERNAME} - password: ${DOCKERHUB_PASSWORD} - resource_class: large - working_directory: ~/project - parallelism: 2 - - steps: - - checkout - - setup_google_dns - - run: - name: DNS lookup for Redis host - command: | - sudo apt-get update - sudo apt-get install -y dnsutils - dig redis-19899.c239.us-east-1-2.ec2.redns.redis-cloud.com +short - - run: - name: Show git commit hash - command: | - echo "Git commit hash: $CIRCLE_SHA1" - - - restore_cache: - keys: - - v2-caching-deps-{{ checksum ".circleci/requirements.txt" }} - - v2-caching-deps- - - run: - name: Install Dependencies - command: | - python -m pip install --upgrade pip - python -m pip install -r .circleci/requirements.txt - pip install "pytest==7.3.1" - pip install "pytest-retry==1.6.3" - pip install "pytest-asyncio==0.21.1" - pip install "pytest-cov==5.0.0" - pip install "mypy==1.18.2" - pip install "google-generativeai==0.3.2" - pip install "google-cloud-aiplatform==1.43.0" - pip install pyarrow - pip install "boto3==1.36.0" - pip install "aioboto3==13.4.0" - pip install langchain - pip install lunary==0.2.5 - pip install "azure-identity==1.16.1" - pip install "langfuse==2.59.7" - pip install "logfire==0.29.0" - pip install numpydoc - pip install traceloop-sdk==0.21.1 - pip install opentelemetry-api==1.25.0 - pip install opentelemetry-sdk==1.25.0 - pip install opentelemetry-exporter-otlp==1.25.0 - pip install openai==1.100.1 - pip install prisma==0.11.0 - pip install "detect_secrets==1.5.0" - pip install "httpx==0.24.1" - pip install "respx==0.22.0" - pip install fastapi - pip install "gunicorn==21.2.0" - pip install "anyio==4.2.0" - pip install "aiodynamo==23.10.1" - pip install "asyncio==3.4.3" - pip install "apscheduler==3.10.4" - pip install "PyGithub==1.59.1" - pip install argon2-cffi - pip install "pytest-mock==3.12.0" - pip install python-multipart - pip install google-cloud-aiplatform - pip install prometheus-client==0.20.0 - pip install "pydantic==2.10.2" - pip install "diskcache==5.6.1" - pip install "Pillow==10.3.0" - pip install "jsonschema==4.22.0" - pip install "websockets==13.1.0" - pip install "pytest-xdist==3.6.1" - - setup_litellm_enterprise_pip - - save_cache: - paths: - - /home/circleci/.pyenv/versions - - /home/circleci/.local - key: v2-caching-deps-{{ checksum ".circleci/requirements.txt" }} - - run: - name: Run prisma ./docker/entrypoint.sh - command: | - set +e - chmod +x docker/entrypoint.sh - ./docker/entrypoint.sh - set -e - - # Run pytest and generate JUnit XML report - - run: - name: Run tests - command: | - pwd - ls - mkdir -p test-results - - TEST_FILES=$(circleci tests glob "tests/local_testing/**/test_*.py") - - echo "$TEST_FILES" | circleci tests run \ - --split-by=timings \ - --verbose \ - --command="xargs python -m pytest \ - -v \ - --junitxml=test-results/junit.xml \ - --durations=5 \ - -k 'caching or cache'" - no_output_timeout: 15m - - # Store test results - - store_test_results: - path: test-results auth_ui_unit_tests: docker: - image: cimg/python:3.11 @@ -664,376 +551,6 @@ jobs: # Store test results - store_test_results: path: test-results - litellm_security_tests: - docker: - - image: cimg/python:3.13 - auth: - username: ${DOCKERHUB_USERNAME} - password: ${DOCKERHUB_PASSWORD} - - image: cimg/postgres:14.0 - environment: - POSTGRES_USER: postgres - POSTGRES_PASSWORD: postgres - POSTGRES_DB: circle_test - resource_class: xlarge - working_directory: ~/project - environment: - DATABASE_URL: "postgresql://postgres:postgres@localhost:5432/circle_test" - steps: - - checkout - - setup_google_dns - - run: - name: Show git commit hash - command: | - echo "Git commit hash: $CIRCLE_SHA1" - - setup_remote_docker: - docker_layer_caching: true - - restore_cache: - keys: - - v3-litellm-uv-deps-{{ checksum "requirements.txt" }}-{{ checksum ".circleci/config.yml" }} - - run: - name: Install Dependencies - command: | - python -m pip install --upgrade pip uv - uv pip install --system -r requirements.txt - pip install "pytest==7.3.1" "pytest-retry==1.6.3" "pytest-mock==3.12.0" \ - "pytest-asyncio==0.21.1" "pytest-cov==5.0.0" - - save_cache: - paths: - - ~/.local/lib - - ~/.local/bin - - ~/.cache/uv - key: v3-litellm-uv-deps-{{ checksum "requirements.txt" }}-{{ checksum ".circleci/config.yml" }} - - run: - name: Install dockerize - command: | - wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz - sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz - rm dockerize-linux-amd64-v0.6.1.tar.gz - - run: - name: Wait for PostgreSQL to be ready - command: dockerize -wait tcp://localhost:5432 -timeout 1m - - run: - name: Run Security Scans - command: | - chmod +x ci_cd/security_scans.sh - ./ci_cd/security_scans.sh - - run: - name: Run prisma ./docker/entrypoint.sh - command: | - set +e - chmod +x docker/entrypoint.sh - ./docker/entrypoint.sh - set -e - # Run pytest and generate JUnit XML report - - run: - name: Run tests - command: | - python -m pytest tests/proxy_security_tests -v -x --junitxml=test-results/junit.xml --durations=5 - no_output_timeout: 15m - # Store test results - - store_test_results: - path: test-results - # Split proxy unit tests into 3 jobs for faster execution and better debugging - # test_key_generate_prisma runs separately without parallel execution to avoid event loop issues with logging worker - litellm_proxy_unit_testing_key_generation: - docker: - - image: cimg/python:3.11 - auth: - username: ${DOCKERHUB_USERNAME} - password: ${DOCKERHUB_PASSWORD} - working_directory: ~/project - resource_class: medium - steps: - - checkout - - setup_google_dns - - run: - name: Show git commit hash - command: | - echo "Git commit hash: $CIRCLE_SHA1" - - run: - name: Install PostgreSQL - command: | - sudo apt-get update - sudo apt-get install -y postgresql-14 postgresql-contrib-14 - - restore_cache: - keys: - - v1-dependencies-{{ checksum ".circleci/requirements.txt" }} - - run: - name: Install Dependencies - command: | - python -m pip install --upgrade pip - python -m pip install -r .circleci/requirements.txt - pip install "pytest==7.3.1" - pip install "pytest-retry==1.6.3" - pip install "pytest-asyncio==0.21.1" - pip install "pytest-cov==5.0.0" - pip install "pytest-timeout==2.2.0" - pip install "pytest-forked==1.6.0" - pip install "mypy==1.18.2" - pip install "google-generativeai==0.3.2" - pip install "google-cloud-aiplatform==1.43.0" - pip install "google-genai==1.22.0" - pip install pyarrow - pip install "boto3==1.36.0" - pip install "aioboto3==13.4.0" - pip install langchain - pip install lunary==0.2.5 - pip install "azure-identity==1.16.1" - pip install "langfuse==2.59.7" - pip install "logfire==0.29.0" - pip install numpydoc - pip install traceloop-sdk==0.21.1 - pip install opentelemetry-api==1.25.0 - pip install opentelemetry-sdk==1.25.0 - pip install opentelemetry-exporter-otlp==1.25.0 - pip install openai==1.100.1 - pip install prisma==0.11.0 - pip install "detect_secrets==1.5.0" - pip install "httpx==0.24.1" - pip install "respx==0.22.0" - pip install fastapi - pip install "gunicorn==21.2.0" - pip install "anyio==4.2.0" - pip install "aiodynamo==23.10.1" - pip install "asyncio==3.4.3" - pip install "apscheduler==3.10.4" - pip install "PyGithub==1.59.1" - pip install argon2-cffi - pip install "pytest-mock==3.12.0" - pip install python-multipart - pip install google-cloud-aiplatform - pip install prometheus-client==0.20.0 - pip install "pydantic==2.10.2" - pip install "diskcache==5.6.1" - pip install "Pillow==10.3.0" - pip install "jsonschema==4.22.0" - pip install "pytest-postgresql==7.0.1" - pip install "fakeredis==2.28.1" - - setup_litellm_enterprise_pip - - save_cache: - paths: - - ./venv - key: v1-dependencies-{{ checksum ".circleci/requirements.txt" }} - - run: - name: Run prisma ./docker/entrypoint.sh - command: | - set +e - chmod +x docker/entrypoint.sh - ./docker/entrypoint.sh - set -e - - run: - name: Run key generation tests (no parallel execution to avoid event loop issues) - command: | - pwd - ls - # Run without -n flag to avoid pytest-xdist event loop conflicts with logging worker - python -m pytest tests/proxy_unit_tests/test_key_generate_prisma.py --cov=litellm --cov-report=xml --junitxml=test-results/junit-key-generation.xml --durations=10 --timeout=300 -vv --log-cli-level=INFO - no_output_timeout: 15m - - run: - name: Rename the coverage files - command: | - mv coverage.xml litellm_proxy_unit_tests_key_generation_coverage.xml - mv .coverage litellm_proxy_unit_tests_key_generation_coverage - - store_test_results: - path: test-results - - persist_to_workspace: - root: . - paths: - - litellm_proxy_unit_tests_key_generation_coverage.xml - - litellm_proxy_unit_tests_key_generation_coverage - litellm_proxy_unit_testing_part1: - docker: - - image: cimg/python:3.11 - auth: - username: ${DOCKERHUB_USERNAME} - password: ${DOCKERHUB_PASSWORD} - working_directory: ~/project - resource_class: xlarge - steps: - - checkout - - setup_google_dns - - run: - name: Show git commit hash - command: | - echo "Git commit hash: $CIRCLE_SHA1" - - run: - name: Install PostgreSQL - command: | - sudo apt-get update - sudo apt-get install -y postgresql-14 postgresql-contrib-14 - - restore_cache: - keys: - - v1-dependencies-{{ checksum ".circleci/requirements.txt" }} - - run: - name: Install Dependencies - command: | - python -m pip install --upgrade pip - python -m pip install -r .circleci/requirements.txt - pip install "pytest==7.3.1" - pip install "pytest-retry==1.6.3" - pip install "pytest-asyncio==0.21.1" - pip install "pytest-cov==5.0.0" - pip install "pytest-timeout==2.2.0" - pip install "pytest-forked==1.6.0" - pip install "mypy==1.18.2" - pip install "google-generativeai==0.3.2" - pip install "google-cloud-aiplatform==1.43.0" - pip install "google-genai==1.22.0" - pip install pyarrow - pip install "boto3==1.36.0" - pip install "aioboto3==13.4.0" - pip install langchain - pip install lunary==0.2.5 - pip install "azure-identity==1.16.1" - pip install "langfuse==2.59.7" - pip install "logfire==0.29.0" - pip install numpydoc - pip install traceloop-sdk==0.21.1 - pip install opentelemetry-api==1.25.0 - pip install opentelemetry-sdk==1.25.0 - pip install opentelemetry-exporter-otlp==1.25.0 - pip install openai==1.100.1 - pip install prisma==0.11.0 - pip install "detect_secrets==1.5.0" - pip install "httpx==0.24.1" - pip install "respx==0.22.0" - pip install fastapi - pip install "gunicorn==21.2.0" - pip install "anyio==4.2.0" - pip install "aiodynamo==23.10.1" - pip install "asyncio==3.4.3" - pip install "apscheduler==3.10.4" - pip install "PyGithub==1.59.1" - pip install argon2-cffi - pip install "pytest-mock==3.12.0" - pip install python-multipart - pip install google-cloud-aiplatform - pip install prometheus-client==0.20.0 - pip install "pydantic==2.10.2" - pip install "diskcache==5.6.1" - pip install "Pillow==10.3.0" - pip install "jsonschema==4.22.0" - pip install "pytest-postgresql==7.0.1" - pip install "fakeredis==2.28.1" - pip install "pytest-xdist==3.6.1" - - setup_litellm_enterprise_pip - - save_cache: - paths: - - ./venv - key: v1-dependencies-{{ checksum ".circleci/requirements.txt" }} - - run: - name: Run prisma ./docker/entrypoint.sh - command: | - set +e - chmod +x docker/entrypoint.sh - ./docker/entrypoint.sh - set -e - - run: - name: Run proxy unit tests (part 1 - auth checks) - command: | - pwd - ls - python -m pytest tests/proxy_unit_tests/test_auth_checks.py tests/proxy_unit_tests/test_user_api_key_auth.py --junitxml=test-results/junit-part1.xml --durations=10 -n 8 --timeout=300 -v - no_output_timeout: 15m - - store_test_results: - path: test-results - litellm_proxy_unit_testing_part2: - docker: - - image: cimg/python:3.11 - auth: - username: ${DOCKERHUB_USERNAME} - password: ${DOCKERHUB_PASSWORD} - working_directory: ~/project - resource_class: xlarge - steps: - - checkout - - setup_google_dns - - run: - name: Show git commit hash - command: | - echo "Git commit hash: $CIRCLE_SHA1" - - run: - name: Install PostgreSQL - command: | - sudo apt-get update - sudo apt-get install -y postgresql-14 postgresql-contrib-14 - - restore_cache: - keys: - - v1-dependencies-{{ checksum ".circleci/requirements.txt" }} - - run: - name: Install Dependencies - command: | - python -m pip install --upgrade pip - python -m pip install -r .circleci/requirements.txt - pip install "pytest==7.3.1" - pip install "pytest-retry==1.6.3" - pip install "pytest-asyncio==0.21.1" - pip install "pytest-cov==5.0.0" - pip install "pytest-timeout==2.2.0" - pip install "pytest-forked==1.6.0" - pip install "mypy==1.18.2" - pip install "google-generativeai==0.3.2" - pip install "google-cloud-aiplatform==1.43.0" - pip install "google-genai==1.22.0" - pip install pyarrow - pip install "boto3==1.36.0" - pip install "aioboto3==13.4.0" - pip install langchain - pip install lunary==0.2.5 - pip install "azure-identity==1.16.1" - pip install "langfuse==2.59.7" - pip install "logfire==0.29.0" - pip install numpydoc - pip install traceloop-sdk==0.21.1 - pip install opentelemetry-api==1.25.0 - pip install opentelemetry-sdk==1.25.0 - pip install opentelemetry-exporter-otlp==1.25.0 - pip install openai==1.100.1 - pip install prisma==0.11.0 - pip install "detect_secrets==1.5.0" - pip install "httpx==0.24.1" - pip install "respx==0.22.0" - pip install fastapi - pip install "gunicorn==21.2.0" - pip install "anyio==4.2.0" - pip install "aiodynamo==23.10.1" - pip install "asyncio==3.4.3" - pip install "apscheduler==3.10.4" - pip install "PyGithub==1.59.1" - pip install argon2-cffi - pip install "pytest-mock==3.12.0" - pip install python-multipart - pip install google-cloud-aiplatform - pip install prometheus-client==0.20.0 - pip install "pydantic==2.10.2" - pip install "diskcache==5.6.1" - pip install "Pillow==10.3.0" - pip install "jsonschema==4.22.0" - pip install "pytest-postgresql==7.0.1" - pip install "fakeredis==2.28.1" - pip install "pytest-xdist==3.6.1" - - setup_litellm_enterprise_pip - - save_cache: - paths: - - ./venv - key: v1-dependencies-{{ checksum ".circleci/requirements.txt" }} - - run: - name: Run prisma ./docker/entrypoint.sh - command: | - set +e - chmod +x docker/entrypoint.sh - ./docker/entrypoint.sh - set -e - - run: - name: Run proxy unit tests (part 2 - remaining tests) - command: | - pwd - ls - python -m pytest tests/proxy_unit_tests --ignore=tests/proxy_unit_tests/test_key_generate_prisma.py --ignore=tests/proxy_unit_tests/test_auth_checks.py --ignore=tests/proxy_unit_tests/test_user_api_key_auth.py --junitxml=test-results/junit-part2.xml --durations=10 -n 8 --timeout=300 -v - no_output_timeout: 15m - - store_test_results: - path: test-results litellm_assistants_api_testing: # Runs all tests with the "assistants" keyword docker: - image: cimg/python:3.13.1 @@ -1115,7 +632,7 @@ jobs: for dir in "${IGNORE_DIRS[@]}"; do IGNORE_ARGS="$IGNORE_ARGS --ignore=$dir" done - python -m pytest -v tests/llm_translation $IGNORE_ARGS --junitxml=test-results/junit.xml --durations=20 -n 8 --timeout=120 --timeout_method=thread + python -m pytest -v tests/llm_translation $IGNORE_ARGS --junitxml=test-results/junit.xml --durations=20 -n 8 --timeout=120 --timeout_method=thread --retries 2 --retry-delay 5 no_output_timeout: 15m # Store test results @@ -1331,7 +848,7 @@ jobs: command: | pwd ls - python -m pytest -vv tests/unified_google_tests --cov=litellm --cov-report=xml -x -s -v --junitxml=test-results/junit.xml --durations=5 + python -m pytest -vv tests/unified_google_tests --cov=litellm --cov-report=xml -x -s -v --junitxml=test-results/junit.xml --durations=5 --retries 3 --retry-delay 5 no_output_timeout: 15m - run: name: Rename the coverage files @@ -1507,101 +1024,6 @@ jobs: no_output_timeout: 15m - store_test_results: path: test-results - litellm_mapped_tests_llms: - docker: - - image: cimg/python:3.11 - auth: - username: ${DOCKERHUB_USERNAME} - password: ${DOCKERHUB_PASSWORD} - working_directory: ~/project - resource_class: large - steps: - - setup_litellm_test_deps - - run: - name: Run LLM provider tests - command: | - python -m pytest tests/test_litellm/llms --junitxml=test-results/junit-llms.xml --durations=10 -n 4 --maxfail=5 --timeout=300 -vv --log-cli-level=WARNING - no_output_timeout: 15m - - store_test_results: - path: test-results - litellm_mapped_tests_core: - docker: - - image: cimg/python:3.11 - auth: - username: ${DOCKERHUB_USERNAME} - password: ${DOCKERHUB_PASSWORD} - working_directory: ~/project - resource_class: large - steps: - - setup_litellm_test_deps - - run: - name: Run core tests - command: | - python -m pytest tests/test_litellm --ignore=tests/test_litellm/proxy --ignore=tests/test_litellm/llms --ignore=tests/test_litellm/integrations --ignore=tests/test_litellm/litellm_core_utils --ignore=tests/test_litellm/experimental_mcp_client --junitxml=test-results/junit-core.xml --durations=10 -n 4 --maxfail=5 --timeout=300 -vv --log-cli-level=WARNING - no_output_timeout: 15m - - store_test_results: - path: test-results - litellm_mapped_tests_litellm_core_utils: - docker: - - image: cimg/python:3.11 - auth: - username: ${DOCKERHUB_USERNAME} - password: ${DOCKERHUB_PASSWORD} - working_directory: ~/project - resource_class: large - steps: - - setup_litellm_test_deps - - run: - name: Run litellm_core_utils tests - command: | - python -m pytest tests/test_litellm/litellm_core_utils --junitxml=test-results/junit-litellm-core-utils.xml --durations=10 -n 4 --maxfail=5 --timeout=300 -vv --log-cli-level=WARNING - no_output_timeout: 15m - - store_test_results: - path: test-results - litellm_mapped_tests_mcps: - docker: - - image: cimg/python:3.11 - auth: - username: ${DOCKERHUB_USERNAME} - password: ${DOCKERHUB_PASSWORD} - working_directory: ~/project - resource_class: medium - steps: - - setup_litellm_test_deps - - run: - name: Run MCP client tests - command: | - python -m pytest tests/test_litellm/experimental_mcp_client --cov=litellm --cov-report=xml --junitxml=test-results/junit-mcps.xml --durations=10 -n 2 --maxfail=5 --timeout=300 -vv --log-cli-level=WARNING - no_output_timeout: 15m - - run: - name: Rename the coverage files - command: | - mv coverage.xml litellm_mcps_tests_coverage.xml - mv .coverage litellm_mcps_tests_coverage - - store_test_results: - path: test-results - - persist_to_workspace: - root: . - paths: - - litellm_mcps_tests_coverage.xml - - litellm_mcps_tests_coverage - litellm_mapped_tests_integrations: - docker: - - image: cimg/python:3.11 - auth: - username: ${DOCKERHUB_USERNAME} - password: ${DOCKERHUB_PASSWORD} - working_directory: ~/project - resource_class: large - steps: - - setup_litellm_test_deps - - run: - name: Run integrations tests - command: | - python -m pytest tests/test_litellm/integrations --junitxml=test-results/junit-integrations.xml --durations=10 -n 4 --maxfail=5 --timeout=300 -vv --log-cli-level=WARNING - no_output_timeout: 15m - - store_test_results: - path: test-results litellm_mapped_enterprise_tests: docker: - image: cimg/python:3.11 @@ -2137,6 +1559,25 @@ jobs: pip install "pytest-asyncio==0.21.1" pip install aiohttp pip install apscheduler + - run: + name: Install dockerize + command: | + sudo wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz + sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz + sudo rm dockerize-linux-amd64-v0.6.1.tar.gz + - run: + name: Start PostgreSQL Database + command: | + docker run -d \ + --name postgres-db \ + -e POSTGRES_USER=postgres \ + -e POSTGRES_PASSWORD=postgres \ + -e POSTGRES_DB=litellm_test \ + -p 5432:5432 \ + postgres:14 + - run: + name: Wait for PostgreSQL to be ready + command: dockerize -wait tcp://localhost:5432 -timeout 1m - attach_workspace: at: ~/project - run: @@ -2145,29 +1586,41 @@ jobs: zstd -d litellm-docker-database.tar.zst --stdout | docker load docker images | grep litellm-docker-database - run: - name: Run Docker container + name: Seed database with real schema + command: | + docker run -d \ + -p 4001:4000 \ + -e DATABASE_URL="postgresql://postgres:postgres@host.docker.internal:5432/litellm_test" \ + -e LITELLM_MASTER_KEY="sk-1234" \ + --name schema-seed \ + --add-host=host.docker.internal:host-gateway \ + -v $(pwd)/litellm/proxy/example_config_yaml/simple_config.yaml:/app/config.yaml \ + litellm-docker-database:ci \ + --config /app/config.yaml \ + --port 4000 \ + --use_prisma_db_push + - run: + name: Wait for schema seed to complete + command: dockerize -wait http://localhost:4001 -timeout 5m + - run: + name: Stop schema seed container + command: docker stop schema-seed && docker rm schema-seed + - run: + name: Run Docker container with bad schema and disabled updates command: | docker run -d \ -p 4000:4000 \ - -e DATABASE_URL=$PROXY_DATABASE_URL \ + -e DATABASE_URL="postgresql://postgres:postgres@host.docker.internal:5432/litellm_test" \ -e DEFAULT_NUM_WORKERS_LITELLM_PROXY=1 \ -e DISABLE_SCHEMA_UPDATE="True" \ + --name my-app \ + --add-host=host.docker.internal:host-gateway \ -v $(pwd)/litellm/proxy/example_config_yaml/bad_schema.prisma:/app/schema.prisma \ -v $(pwd)/litellm/proxy/example_config_yaml/bad_schema.prisma:/app/litellm/proxy/schema.prisma \ -v $(pwd)/litellm/proxy/example_config_yaml/disable_schema_update.yaml:/app/config.yaml \ - --name my-app \ litellm-docker-database:ci \ --config /app/config.yaml \ --port 4000 - - run: - name: Install curl and dockerize - command: | - sudo apt-get update - sudo apt-get install -y curl - sudo wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz - sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz - sudo rm dockerize-linux-amd64-v0.6.1.tar.gz - - run: name: Wait for container to be ready command: dockerize -wait http://localhost:4000 -timeout 1m @@ -2575,9 +2028,6 @@ jobs: -e OPENAI_API_KEY=$OPENAI_API_KEY \ -e LITELLM_LICENSE=$LITELLM_LICENSE \ -e OTEL_EXPORTER="in_memory" \ - -e APORIA_API_BASE_2=$APORIA_API_BASE_2 \ - -e APORIA_API_KEY_2=$APORIA_API_KEY_2 \ - -e APORIA_API_BASE_1=$APORIA_API_BASE_1 \ -e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID \ -e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY \ -e DEFAULT_NUM_WORKERS_LITELLM_PROXY=1 \ @@ -2585,7 +2035,6 @@ jobs: -e DD_API_KEY=$DD_API_KEY \ -e DD_SITE=$DD_SITE \ -e AWS_REGION_NAME=$AWS_REGION_NAME \ - -e APORIA_API_KEY_1=$APORIA_API_KEY_1 \ -e COHERE_API_KEY=$COHERE_API_KEY \ -e GCS_FLUSH_INTERVAL="1" \ --add-host host.docker.internal:host-gateway \ @@ -3061,6 +2510,20 @@ jobs: name: Build Docker image command: | docker build -t my-app:latest -f docker/build_from_pip/Dockerfile.build_from_pip . + - run: + name: Start PostgreSQL Database + command: | + docker run -d \ + --name postgres-db \ + -e POSTGRES_USER=postgres \ + -e POSTGRES_PASSWORD=postgres \ + -e POSTGRES_DB=circle_test \ + -p 5432:5432 \ + postgres:14 + - run: + name: Wait for PostgreSQL to be ready + command: | + timeout 60s bash -c 'until docker exec postgres-db pg_isready -U postgres -d circle_test; do sleep 2; done' - run: name: Run Docker container # intentionally give bad redis credentials here @@ -3068,7 +2531,7 @@ jobs: command: | docker run -d \ -p 4000:4000 \ - -e DATABASE_URL=$PROXY_DATABASE_URL \ + -e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \ -e REDIS_HOST=$REDIS_HOST \ -e REDIS_PASSWORD=$REDIS_PASSWORD \ -e REDIS_PORT=$REDIS_PORT \ @@ -3076,18 +2539,15 @@ jobs: -e OPENAI_API_KEY=$OPENAI_API_KEY \ -e LITELLM_LICENSE=$LITELLM_LICENSE \ -e OTEL_EXPORTER="in_memory" \ - -e APORIA_API_BASE_2=$APORIA_API_BASE_2 \ - -e APORIA_API_KEY_2=$APORIA_API_KEY_2 \ - -e APORIA_API_BASE_1=$APORIA_API_BASE_1 \ -e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID \ -e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY \ -e AWS_REGION_NAME=$AWS_REGION_NAME \ - -e APORIA_API_KEY_1=$APORIA_API_KEY_1 \ -e COHERE_API_KEY=$COHERE_API_KEY \ -e USE_DDTRACE=True \ -e DD_API_KEY=$DD_API_KEY \ -e DD_SITE=$DD_SITE \ -e GCS_FLUSH_INTERVAL="1" \ + --add-host host.docker.internal:host-gateway \ --name my-app \ -v $(pwd)/docker/build_from_pip/litellm_config.yaml:/app/config.yaml \ my-app:latest \ @@ -3118,8 +2578,11 @@ jobs: - run: name: Stop and remove first container command: | - docker stop my-app - docker rm my-app + docker stop my-app || true + docker rm my-app || true + docker stop postgres-db || true + docker rm postgres-db || true + when: always proxy_pass_through_endpoint_tests: machine: image: ubuntu-2204:2023.10.1 @@ -3543,93 +3006,6 @@ jobs: - codecov/upload: file: ./coverage.xml - publish_to_pypi: - docker: - - image: cimg/python:3.8 - working_directory: ~/project - - environment: - TWINE_USERNAME: __token__ - - steps: - - checkout - - - run: - name: Copy model_prices_and_context_window File to model_prices_and_context_window_backup - command: | - cp model_prices_and_context_window.json litellm/model_prices_and_context_window_backup.json - - - run: - name: Checkout code - command: git checkout $CIRCLE_SHA1 - - # Check if setup.py is modified and publish to PyPI - - run: - name: PyPI publish - command: | - echo "Install TOML package." - python -m pip install toml - VERSION=$(python -c "import toml; print(toml.load('pyproject.toml')['tool']['poetry']['version'])") - PACKAGE_NAME=$(python -c "import toml; print(toml.load('pyproject.toml')['tool']['poetry']['name'])") - if ! pip show -v $PACKAGE_NAME | grep -q "Version: ${VERSION}"; then - echo "pyproject.toml modified" - echo -e "[pypi]\nusername = $PYPI_PUBLISH_USERNAME\npassword = $PYPI_PUBLISH_PASSWORD" > ~/.pypirc - python -m pip install --upgrade pip - pip install build - pip install wheel - pip install --upgrade twine setuptools - rm -rf build dist - - echo "Building package" - python -m build - - echo "Twine upload to dist" - echo "Contents of dist directory:" - ls dist/ - twine upload --verbose dist/* - else - echo "Version ${VERSION} of package is already published on PyPI." - - # Check if corresponding Docker nightly image exists - NIGHTLY_TAG="v${VERSION}-nightly" - echo "Checking for Docker nightly image: litellm/litellm:${NIGHTLY_TAG}" - - # Check Docker Hub for the nightly image - if curl -s "https://hub.docker.com/v2/repositories/litellm/litellm/tags/${NIGHTLY_TAG}" | grep -q "name"; then - echo "Docker nightly image ${NIGHTLY_TAG} exists. This release was already completed successfully." - echo "Skipping PyPI publish and continuing to ensure Docker images are up to date." - circleci step halt - else - echo "ERROR: PyPI package ${VERSION} exists but Docker nightly image ${NIGHTLY_TAG} does not exist!" - echo "This indicates an incomplete release. Please investigate." - exit 1 - fi - fi - - run: - name: Trigger Github Action for new Docker Container + Trigger Load Testing - command: | - echo "Install TOML package." - python3 -m pip install toml - VERSION=$(python3 -c "import toml; print(toml.load('pyproject.toml')['tool']['poetry']['version'])") - echo "LiteLLM Version ${VERSION}" - - # Determine which branch to use for Docker build - if [[ "$CIRCLE_BRANCH" =~ ^litellm_release_day_.* ]]; then - BUILD_BRANCH="$CIRCLE_BRANCH" - echo "Using release branch: $BUILD_BRANCH" - else - BUILD_BRANCH="main" - echo "Using default branch: $BUILD_BRANCH" - fi - - curl -X POST \ - -H "Accept: application/vnd.github.v3+json" \ - -H "Authorization: Bearer $GITHUB_TOKEN" \ - "https://api.github.com/repos/BerriAI/litellm/actions/workflows/ghcr_deploy.yml/dispatches" \ - -d "{\"ref\":\"${BUILD_BRANCH}\", \"inputs\":{\"tag\":\"v${VERSION}-nightly\", \"commit_hash\":\"$CIRCLE_SHA1\"}}" - echo "triggering load testing server for version ${VERSION} and commit ${CIRCLE_SHA1}" - curl -X POST "https://proxyloadtester-production.up.railway.app/start/load/test?version=${VERSION}&commit_hash=${CIRCLE_SHA1}&release_type=nightly" - publish_proxy_extras: docker: - image: cimg/python:3.8 @@ -3942,37 +3318,39 @@ jobs: - setup_google_dns - attach_workspace: at: ~/project + - run: + name: Install dockerize + command: | + sudo wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz + sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz + sudo rm dockerize-linux-amd64-v0.6.1.tar.gz + - run: + name: Start PostgreSQL Database + command: | + docker run -d \ + --name postgres-db \ + -e POSTGRES_USER=postgres \ + -e POSTGRES_PASSWORD=postgres \ + -e POSTGRES_DB=litellm_schema_sync \ + -p 5432:5432 \ + postgres:14 + - run: + name: Wait for PostgreSQL to be ready + command: dockerize -wait tcp://localhost:5432 -timeout 1m - run: name: Load Docker Database Image command: | zstd -d litellm-docker-database.tar.zst --stdout | docker load docker images | grep litellm-docker-database - run: - name: Install Neon CLI + name: Run schema sync via prisma db push command: | - npm i -g neonctl - - run: - name: Install curl and dockerize - command: | - sudo apt-get update - sudo apt-get install -y curl - sudo wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz - sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz - sudo rm dockerize-linux-amd64-v0.6.1.tar.gz - - run: - name: Sync schema on base e2e database - command: | - BASE_DATABASE_URL=$(neon connection-string \ - --project-id $NEON_PROJECT_ID \ - --api-key $NEON_API_KEY \ - --branch br-fancy-paper-ad1olsb3 \ - --database-name yuneng-trial-db \ - --role neondb_owner) docker run -d \ -p 4000:4000 \ - -e DATABASE_URL=$BASE_DATABASE_URL \ + -e DATABASE_URL="postgresql://postgres:postgres@host.docker.internal:5432/litellm_schema_sync" \ -e LITELLM_MASTER_KEY="sk-1234" \ --name schema-sync \ + --add-host=host.docker.internal:host-gateway \ -v $(pwd)/litellm/proxy/example_config_yaml/simple_config.yaml:/app/config.yaml \ litellm-docker-database:ci \ --config /app/config.yaml \ @@ -4089,34 +3467,14 @@ workflows: only: - main - /litellm_.*/ - - caching_unit_tests: - filters: - branches: - only: - main - /litellm_.*/ - - litellm_proxy_unit_testing_key_generation: - filters: - branches: - only: - main - /litellm_.*/ - - litellm_proxy_unit_testing_part1: - filters: - branches: - only: - main - /litellm_.*/ - - litellm_proxy_unit_testing_part2: - filters: - branches: - only: - main - /litellm_.*/ - - litellm_security_tests: - filters: - branches: - only: - main - /litellm_.*/ - litellm_assistants_api_testing: @@ -4170,7 +3528,6 @@ workflows: - main - /litellm_.*/ - prisma_schema_sync: - context: e2e_ui_tests requires: - build_docker_database_image filters: @@ -4178,32 +3535,32 @@ workflows: only: - main - /litellm_.*/ - - e2e_ui_testing: - name: e2e_ui_testing_chromium - browser: chromium - context: e2e_ui_tests - requires: - - ui_build - - build_docker_database_image - - prisma_schema_sync - filters: - branches: - only: - - main - - /litellm_.*/ - - e2e_ui_testing: - name: e2e_ui_testing_firefox - browser: firefox - context: e2e_ui_tests - requires: - - ui_build - - build_docker_database_image - - prisma_schema_sync - filters: - branches: - only: - - main - - /litellm_.*/ + # - e2e_ui_testing: # migrate to dynamic db - currently requires neon cli + # name: e2e_ui_testing_chromium + # browser: chromium + # context: e2e_ui_tests + # requires: + # - ui_build + # - build_docker_database_image + # - prisma_schema_sync + # filters: + # branches: + # only: + # - main + # - /litellm_.*/ + # - e2e_ui_testing: + # name: e2e_ui_testing_firefox + # browser: firefox + # context: e2e_ui_tests + # requires: + # - ui_build + # - build_docker_database_image + # - prisma_schema_sync + # filters: + # branches: + # only: + # - main + # - /litellm_.*/ - build_and_test: requires: - build_docker_database_image @@ -4352,34 +3709,14 @@ workflows: only: - main - /litellm_.*/ - - litellm_mapped_tests_llms: - filters: - branches: - only: - main - /litellm_.*/ - - litellm_mapped_tests_core: - filters: - branches: - only: - main - /litellm_.*/ - - litellm_mapped_tests_mcps: - filters: - branches: - only: - main - /litellm_.*/ - - litellm_mapped_tests_integrations: - filters: - branches: - only: - main - /litellm_.*/ - - litellm_mapped_tests_litellm_core_utils: - filters: - branches: - only: - main - /litellm_.*/ - batches_testing: @@ -4429,11 +3766,6 @@ workflows: - search_testing - litellm_mapped_tests_proxy_part1 - litellm_mapped_tests_proxy_part2 - - litellm_mapped_tests_llms - - litellm_mapped_tests_core - - litellm_mapped_tests_mcps - - litellm_mapped_tests_integrations - - litellm_mapped_tests_litellm_core_utils - litellm_mapped_enterprise_tests - batches_testing - litellm_utils_testing @@ -4441,8 +3773,6 @@ workflows: - image_gen_testing - logging_testing - audio_testing - - caching_unit_tests - - litellm_proxy_unit_testing_key_generation - langfuse_logging_unit_tests - local_testing_part1 - local_testing_part2 @@ -4489,59 +3819,3 @@ workflows: only: - main - /litellm_release_day_.*/ - - publish_to_pypi: - requires: - - mypy_linting - - semgrep - - local_testing_part1 - - local_testing_part2 - - build_and_test - - e2e_openai_endpoints - - test_bad_database_url - - llm_translation_testing - - realtime_translation_testing - - mcp_testing - - agent_testing - - google_generate_content_endpoint_testing - - llm_responses_api_testing - - ocr_testing - - search_testing - - litellm_mapped_tests_proxy_part1 - - litellm_mapped_tests_proxy_part2 - - litellm_mapped_tests_llms - - litellm_mapped_tests_core - - litellm_mapped_tests_mcps - - litellm_mapped_tests_integrations - - litellm_mapped_tests_litellm_core_utils - - litellm_mapped_enterprise_tests - - batches_testing - - litellm_utils_testing - - pass_through_unit_testing - - image_gen_testing - - logging_testing - - audio_testing - - litellm_router_testing - - litellm_router_unit_testing - - caching_unit_tests - - langfuse_logging_unit_tests - - litellm_assistants_api_testing - - auth_ui_unit_tests - - ui_unit_tests - - db_migration_disable_update_check - - e2e_ui_testing_chromium - - e2e_ui_testing_firefox - - litellm_proxy_unit_testing_key_generation - - litellm_proxy_unit_testing_part1 - - litellm_proxy_unit_testing_part2 - - litellm_security_tests - - installing_litellm_on_python - - installing_litellm_on_python_3_13 - - proxy_logging_guardrails_model_info_tests - - proxy_spend_accuracy_tests - - proxy_multi_instance_tests - - proxy_store_model_in_db_tests - - proxy_build_from_pip_tests - - proxy_pass_through_endpoint_tests - - check_code_and_doc_quality - - publish_proxy_extras - - guardrails_testing diff --git a/.claude/settings.json b/.claude/settings.json deleted file mode 100644 index 8c1d85f96e0..00000000000 --- a/.claude/settings.json +++ /dev/null @@ -1,36 +0,0 @@ -{ - "permissions": { - "allow": [ - "Bash(git show:*)", - "Bash(git worktree add:*)", - "Read(//Users/krrishdholakia/Documents/litellm/**)", - "Read(//Users/krrishdholakia/Documents/litellm-claude-code-guardrails/litellm/types/**)", - "Read(//Users/krrishdholakia/Documents/litellm-claude-code-guardrails/**)", - "Read(//Users/krrishdholakia/Documents/litellm-claude-code-guardrails/litellm/**)", - "Bash(python:*)", - "Bash(python -c \"\nimport sys; sys.path.insert\\(0, ''.''\\)\nfrom litellm.proxy.guardrails.guardrail_hooks.claude_code.guardrail import ClaudeCodeGuardrail, HOSTED_TOOL_PREFIXES\nprint\\(''HOSTED_TOOL_PREFIXES:'', HOSTED_TOOL_PREFIXES\\)\nprint\\(''ClaudeCodeGuardrail imported OK''\\)\n\")", - "Read(//Users/krrishdholakia/Documents/litellm-mcp-jwt-groups/litellm/proxy/**)", - "Read(//Users/krrishdholakia/Documents/litellm-mcp-jwt-groups/**)", - "Bash(poetry run pytest:*)", - "Bash(git add:*)", - "Bash(git commit:*)", - "Bash(poetry run python:*)", - "Bash(poetry run pip:*)", - "Bash(git reset:*)", - "Bash(git cherry-pick:*)", - "Bash(git checkout:*)", - "Read(//Users/krrishdholakia/Documents/litellm/litellm/proxy/guardrails/guardrail_hooks/**)", - "Read(//Users/krrishdholakia/Documents/**)", - "Bash(git -C /Users/krrishdholakia/Documents/litellm-mcp-user-permissions worktree list)", - "Bash(ls:*)" - ], - "additionalDirectories": [ - "/Users/krrishdholakia/Documents/litellm-mcp-group-plan/plan", - "/Users/krrishdholakia/Documents/litellm-claude-code-guardrails/litellm/proxy/guardrails/guardrail_hooks/claude_code", - "/Users/krrishdholakia/Documents/litellm-claude-code-guardrails/litellm/types", - "/Users/krrishdholakia/Documents/litellm-claude-code-guardrails", - "/Users/krrishdholakia/Documents/litellm-mcp-jwt-groups/litellm/proxy", - "/Users/krrishdholakia/Documents/litellm-mcp-jwt-groups/tests/test_litellm/proxy/auth" - ] - } -} diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml index 4744ab048c7..cbf380bac01 100644 --- a/.github/ISSUE_TEMPLATE/config.yml +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -1,7 +1,7 @@ blank_issues_enabled: true contact_links: - name: Schedule Demo - url: https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions + url: https://enterprise.litellm.ai/demo about: Speak directly with Krrish and Ishaan, the founders, to discuss issues, share feedback, or explore improvements for LiteLLM - name: Discord url: https://discord.com/invite/wuPM9dRgDw diff --git a/.github/actions/helm-oci-chart-releaser/action.yml b/.github/actions/helm-oci-chart-releaser/action.yml index 1823e262832..454c591d436 100644 --- a/.github/actions/helm-oci-chart-releaser/action.yml +++ b/.github/actions/helm-oci-chart-releaser/action.yml @@ -41,32 +41,54 @@ runs: using: composite steps: - name: Helm | Setup - uses: azure/setup-helm@v4 + uses: azure/setup-helm@1a275c3b69536ee54be43f2070a358922e12c8d4 # v4.3.1 with: version: v3.20.0 - name: Helm | Login shell: bash - run: echo ${{ inputs.registry_password }} | helm registry login -u ${{ inputs.registry_username }} --password-stdin ${{ inputs.registry }} + env: + REGISTRY_PASSWORD: ${{ inputs.registry_password }} + REGISTRY_USERNAME: ${{ inputs.registry_username }} + REGISTRY: ${{ inputs.registry }} + run: echo "$REGISTRY_PASSWORD" | helm registry login -u "$REGISTRY_USERNAME" --password-stdin "$REGISTRY" - name: Helm | Dependency if: inputs.update_dependencies == 'true' shell: bash - run: helm dependency update ${{ inputs.path == null && format('{0}/{1}', 'charts', inputs.name) || inputs.path }} + env: + CHART_PATH: ${{ inputs.path == null && format('{0}/{1}', 'charts', inputs.name) || inputs.path }} + run: helm dependency update "$CHART_PATH" - name: Helm | Package shell: bash - run: helm package ${{ inputs.path == null && format('{0}/{1}', 'charts', inputs.name) || inputs.path }} --version ${{ inputs.tag }} --app-version ${{ inputs.app_version }} + env: + CHART_PATH: ${{ inputs.path == null && format('{0}/{1}', 'charts', inputs.name) || inputs.path }} + TAG: ${{ inputs.tag }} + APP_VERSION: ${{ inputs.app_version }} + run: helm package "$CHART_PATH" --version "$TAG" --app-version "$APP_VERSION" - name: Helm | Push shell: bash - run: helm push ${{ inputs.name }}-${{ inputs.tag }}.tgz oci://${{ inputs.registry }}/${{ inputs.repository }} + env: + NAME: ${{ inputs.name }} + TAG: ${{ inputs.tag }} + REGISTRY: ${{ inputs.registry }} + REPOSITORY: ${{ inputs.repository }} + run: helm push "${NAME}-${TAG}.tgz" "oci://${REGISTRY}/${REPOSITORY}" - name: Helm | Logout shell: bash - run: helm registry logout ${{ inputs.registry }} + env: + REGISTRY: ${{ inputs.registry }} + run: helm registry logout "$REGISTRY" - name: Helm | Output id: output shell: bash - run: echo "image=${{ inputs.registry }}/${{ inputs.repository }}/${{ inputs.name }}:${{ inputs.tag }}" >> $GITHUB_OUTPUT + env: + REGISTRY: ${{ inputs.registry }} + REPOSITORY: ${{ inputs.repository }} + NAME: ${{ inputs.name }} + TAG: ${{ inputs.tag }} + run: echo "image=${REGISTRY}/${REPOSITORY}/${NAME}:${TAG}" >> $GITHUB_OUTPUT diff --git a/.github/codeql/codeql-config.yml b/.github/codeql/codeql-config.yml index 20807685e12..36d70c1d746 100644 --- a/.github/codeql/codeql-config.yml +++ b/.github/codeql/codeql-config.yml @@ -1,22 +1,21 @@ name: "LiteLLM CodeQL config" -# Use security-extended suite instead of security-and-quality to avoid -# result sets > 2 GiB on this codebase that cause fatal OOM failures. queries: - - uses: security-extended + - uses: security-and-quality -# These two queries are security queries included in security-extended that -# individually produce result sets > 2 GiB on this codebase, causing fatal -# OOM failures. Exclude them as a safety net until CI confirms they no longer -# OOM; drop these exclusions in a follow-up once verified. +# Known OOM queries on large Python codebases: +# CodeQL builds a full data flow graph in memory. These two queries trace +# sensitive data through every log call / regex pattern, causing combinatorial +# path explosion on codebases with extensive logging like LiteLLM (>2 GiB +# result sets). This is a known CodeQL scaling limitation, not a code issue. +# Re-test periodically as CodeQL improves or the codebase refactors logging. query-filters: - exclude: - id: py/clear-text-logging-sensitive-data # CWE-312 — > 2 GiB result set + id: py/clear-text-logging-sensitive-data # CWE-312 - exclude: - id: py/polynomial-redos # CWE-730 — > 2 GiB result set + id: py/polynomial-redos # CWE-730 paths-ignore: - tests - docs - "**/*.md" - - litellm/proxy/_experimental/out diff --git a/.github/dependabot.yaml b/.github/dependabot.yaml index 58e7cfe10da..c49882a8d62 100644 --- a/.github/dependabot.yaml +++ b/.github/dependabot.yaml @@ -4,6 +4,9 @@ updates: directory: "/" schedule: interval: "daily" + cooldown: + default-days: 7 + semver-major-days: 14 groups: github-actions: patterns: diff --git a/.github/workflows/_test-unit-base.yml b/.github/workflows/_test-unit-base.yml new file mode 100644 index 00000000000..f1ae30e67d7 --- /dev/null +++ b/.github/workflows/_test-unit-base.yml @@ -0,0 +1,96 @@ +name: _Unit Test Base (Reusable) + +on: + workflow_call: + inputs: + test-path: + description: "Pytest path(s) to run" + required: true + type: string + workers: + description: "Number of pytest-xdist workers" + required: false + type: number + default: 2 + reruns: + description: "Number of reruns for flaky tests" + required: false + type: number + default: 2 + timeout-minutes: + description: "Job timeout in minutes" + required: false + type: number + default: 20 + max-failures: + description: "Stop after this many failures" + required: false + type: number + default: 10 + +permissions: + contents: read + +jobs: + run: + name: Run tests + runs-on: ubuntu-latest + timeout-minutes: ${{ inputs.timeout-minutes }} + + steps: + - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + with: + persist-credentials: false + + - name: Set up Python + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: "3.12" + + - name: Install Poetry + run: pip install 'poetry==2.3.2' + + - name: Cache Poetry dependencies + uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 + with: + path: | + ~/.cache/pypoetry + ~/.cache/pip + .venv + key: ${{ runner.os }}-poetry-${{ hashFiles('poetry.lock') }} + restore-keys: | + ${{ runner.os }}-poetry- + + - name: Install dependencies + run: | + poetry config virtualenvs.in-project true + poetry install --with dev,proxy-dev --extras "proxy semantic-router" + poetry run pip install google-genai==1.22.0 \ + google-cloud-aiplatform==1.115.0 fastapi-offline==1.7.3 python-multipart==0.0.22 openapi-core==0.23.0 + + - name: Setup litellm-enterprise + run: | + poetry run pip install --force-reinstall --no-deps -e enterprise/ + + - name: Generate Prisma client + env: + PRISMA_BINARY_CACHE_DIR: ${{ runner.temp }}/prisma-cache + run: | + poetry run pip install nodejs-wheel-binaries==24.13.1 + poetry run prisma generate --schema litellm/proxy/schema.prisma + + - name: Run tests + env: + TEST_PATH: ${{ inputs.test-path }} + MAX_FAILURES: ${{ inputs.max-failures }} + WORKERS: ${{ inputs.workers }} + RERUNS: ${{ inputs.reruns }} + run: | + poetry run pytest ${TEST_PATH:?} \ + --tb=short -vv \ + --maxfail="${MAX_FAILURES}" \ + -n "${WORKERS}" \ + --reruns "${RERUNS}" \ + --reruns-delay 1 \ + --dist=loadscope \ + --durations=20 diff --git a/.github/workflows/_test-unit-services-base.yml b/.github/workflows/_test-unit-services-base.yml new file mode 100644 index 00000000000..d53a9e8822a --- /dev/null +++ b/.github/workflows/_test-unit-services-base.yml @@ -0,0 +1,164 @@ +name: _Unit Test Services Base (Reusable) + +on: + workflow_call: + inputs: + test-path: + description: "Pytest path(s) to run" + required: true + type: string + workers: + description: "Number of pytest-xdist workers (0 = no parallelism)" + required: false + type: number + default: 2 + reruns: + description: "Number of reruns for flaky tests" + required: false + type: number + default: 2 + timeout-minutes: + description: "Job timeout in minutes" + required: false + type: number + default: 20 + max-failures: + description: "Stop after this many failures" + required: false + type: number + default: 10 + enable-redis: + description: "Pass Redis Cloud credentials to tests via REDIS_HOST/PORT/PASSWORD env vars" + required: false + type: boolean + default: false + enable-postgres: + description: "Start a local Postgres service container and run Prisma migrations" + required: false + type: boolean + default: false + secrets: + REDIS_HOST: + required: false + REDIS_PORT: + required: false + REDIS_PASSWORD: + required: false + DATABASE_URL: + required: false + POSTGRES_USER: + required: false + POSTGRES_PASSWORD: + required: false + +permissions: + contents: read + +jobs: + run: + name: Run tests + runs-on: ubuntu-latest + timeout-minutes: ${{ inputs.timeout-minutes }} + # Environment is derived from the enable-* flags, not caller-controllable. + # This prevents callers from passing arbitrary environment names to bypass secret scoping. + # Note: Postgres service container always starts (GHA limitation), so any Redis job + # also needs Postgres secrets → uses integration-redis-postgres, not integration-redis. + environment: >- + ${{ + inputs.enable-redis && 'integration-redis-postgres' || + inputs.enable-postgres && 'integration-postgres' || + '' + }} + + services: + postgres: + image: postgres@sha256:705a5d5b5836f3fcba0d02c4d281e6a7dd9ed2dd4078640f08a1e1e9896e097d # postgres:14 + env: + POSTGRES_USER: ${{ secrets.POSTGRES_USER }} + POSTGRES_PASSWORD: ${{ secrets.POSTGRES_PASSWORD }} + POSTGRES_DB: litellm_test + ports: + - 5432:5432 + options: >- + --health-cmd "pg_isready" + --health-interval 10s + --health-timeout 5s + --health-retries 5 + + steps: + - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + with: + persist-credentials: false + + - name: Set up Python + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: "3.12" + + - name: Install Poetry + run: pip install 'poetry==2.3.2' + + - name: Cache Poetry dependencies + uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 + with: + path: | + ~/.cache/pypoetry + ~/.cache/pip + .venv + key: ${{ runner.os }}-poetry-services-${{ hashFiles('poetry.lock') }} + restore-keys: | + ${{ runner.os }}-poetry-services- + + - name: Install dependencies + run: | + poetry config virtualenvs.in-project true + poetry install --with dev,proxy-dev --extras "proxy semantic-router" + poetry run pip install google-genai==1.22.0 \ + google-cloud-aiplatform==1.115.0 fastapi-offline==1.7.3 python-multipart==0.0.22 openapi-core==0.23.0 + + - name: Setup litellm-enterprise + run: | + poetry run pip install --force-reinstall --no-deps -e enterprise/ + + - name: Generate Prisma client + env: + PRISMA_BINARY_CACHE_DIR: ${{ runner.temp }}/prisma-cache + run: | + poetry run pip install nodejs-wheel-binaries==24.13.1 + poetry run prisma generate --schema litellm/proxy/schema.prisma + + - name: Run Prisma migrations + if: ${{ inputs.enable-postgres }} + env: + DATABASE_URL: ${{ secrets.DATABASE_URL }} + run: | + poetry run prisma db push --schema litellm/proxy/schema.prisma --accept-data-loss + + - name: Run tests + env: + TEST_PATH: ${{ inputs.test-path }} + MAX_FAILURES: ${{ inputs.max-failures }} + WORKERS: ${{ inputs.workers }} + RERUNS: ${{ inputs.reruns }} + DATABASE_URL: ${{ inputs.enable-postgres && secrets.DATABASE_URL || '' }} + REDIS_HOST: ${{ inputs.enable-redis && secrets.REDIS_HOST || '' }} + REDIS_PORT: ${{ inputs.enable-redis && secrets.REDIS_PORT || '' }} + REDIS_PASSWORD: ${{ inputs.enable-redis && secrets.REDIS_PASSWORD || '' }} + run: | + if [ "${WORKERS}" = "0" ]; then + poetry run pytest ${TEST_PATH:?} \ + --tb=short -vv \ + --maxfail="${MAX_FAILURES}" \ + --reruns "${RERUNS}" \ + --reruns-delay 1 \ + --durations=20 + else + poetry run pytest ${TEST_PATH:?} \ + --tb=short -vv \ + --maxfail="${MAX_FAILURES}" \ + -n "${WORKERS}" \ + --reruns "${RERUNS}" \ + --reruns-delay 1 \ + --dist=loadscope \ + --durations=20 + fi diff --git a/.github/workflows/auto_update_price_and_context_window.yml b/.github/workflows/auto_update_price_and_context_window.yml index 98b9d868e68..60e89936219 100644 --- a/.github/workflows/auto_update_price_and_context_window.yml +++ b/.github/workflows/auto_update_price_and_context_window.yml @@ -2,18 +2,24 @@ name: Updates model_prices_and_context_window.json and Create Pull Request on: schedule: - - cron: "0 0 * * 0" # Run every Sundays at midnight + - cron: "0 0 * * 0" # Run every Sundays at midnight #- cron: "0 0 * * *" # Run daily at midnight +permissions: + contents: write + pull-requests: write + jobs: auto_update_price_and_context_window: if: github.repository == 'BerriAI/litellm' runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + with: + persist-credentials: false - name: Install Dependencies run: | - pip install aiohttp + pip install 'aiohttp==3.13.3' - name: Update JSON Data run: | python ".github/workflows/auto_update_price_and_context_window_file.py" @@ -26,4 +32,4 @@ jobs: --head auto-update-price-and-context-window-$(date +'%Y-%m-%d') \ --base main env: - GH_TOKEN: ${{ secrets.GH_TOKEN }} \ No newline at end of file + GH_TOKEN: ${{ secrets.GH_TOKEN }} diff --git a/.github/workflows/check-schema-sync.yml b/.github/workflows/check-schema-sync.yml new file mode 100644 index 00000000000..0e5e2804e60 --- /dev/null +++ b/.github/workflows/check-schema-sync.yml @@ -0,0 +1,58 @@ +name: Check Schema Sync + +on: + pull_request: + paths: + - 'schema.prisma' + - 'litellm/proxy/schema.prisma' + - 'litellm-proxy-extras/litellm_proxy_extras/schema.prisma' + +permissions: + contents: read + +jobs: + check-sync: + name: Verify schema.prisma copies match root + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - name: Checkout PR + uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + with: + persist-credentials: false + + - name: Reject symlinked schema files + run: | + for f in schema.prisma litellm/proxy/schema.prisma litellm-proxy-extras/litellm_proxy_extras/schema.prisma; do + if [ -L "$f" ]; then + echo "::error file=$f::$f is a symlink, which is not allowed" + exit 1 + fi + done + + - name: Check all schemas match root + run: | + EXIT=0 + + diff schema.prisma litellm/proxy/schema.prisma || { + echo "::error file=litellm/proxy/schema.prisma::litellm/proxy/schema.prisma differs from root schema.prisma" + EXIT=1 + } + + diff schema.prisma litellm-proxy-extras/litellm_proxy_extras/schema.prisma || { + echo "::error file=litellm-proxy-extras/litellm_proxy_extras/schema.prisma::litellm-proxy-extras/litellm_proxy_extras/schema.prisma differs from root schema.prisma" + EXIT=1 + } + + if [ "$EXIT" -ne 0 ]; then + echo "" + echo "Schema files are out of sync." + echo "The root schema.prisma is the source of truth." + echo "" + echo "To fix, run from the repo root:" + echo " cp schema.prisma litellm/proxy/schema.prisma" + echo " cp schema.prisma litellm-proxy-extras/litellm_proxy_extras/schema.prisma" + exit 1 + fi + + echo "All schema copies are in sync with root." diff --git a/.github/workflows/check_duplicate_issues.yml b/.github/workflows/check_duplicate_issues.yml index 6d11ce573eb..289d78880ad 100644 --- a/.github/workflows/check_duplicate_issues.yml +++ b/.github/workflows/check_duplicate_issues.yml @@ -12,7 +12,7 @@ jobs: contents: read steps: - name: Check for potential duplicates - uses: wow-actions/potential-duplicates@v1 + uses: wow-actions/potential-duplicates@4d4ea0352e0383859279938e255179dd1dbb67b5 # v1.1.0 with: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} label: potential-duplicate @@ -30,13 +30,14 @@ jobs: - name: Checkout close script if: github.event.action == 'opened' - uses: actions/checkout@v4 + uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 with: sparse-checkout: .github/scripts + persist-credentials: false - name: Set up Python if: github.event.action == 'opened' - uses: actions/setup-python@v5 + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version: "3.11" diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 0b7cce2e4be..e86fca17c7a 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -6,8 +6,8 @@ on: pull_request: branches: [main] schedule: - # Run weekly on Sundays at 04:00 UTC - - cron: "0 4 * * 0" + # Run daily at 04:00 UTC + - cron: "0 4 * * *" concurrency: group: ${{ github.workflow }}-${{ github.ref }} @@ -15,6 +15,7 @@ concurrency: jobs: analyze: + if: github.event_name != 'schedule' || github.repository == 'BerriAI/litellm' name: Analyze (${{ matrix.language }}) runs-on: ubuntu-latest timeout-minutes: 30 @@ -37,16 +38,18 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v4 + uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + with: + persist-credentials: false - name: Initialize CodeQL - uses: github/codeql-action/init@v3 + uses: github/codeql-action/init@ebcb5b36ded6beda4ceefea6a8bc4cc885255bb3 # v3 with: languages: ${{ matrix.language }} build-mode: ${{ matrix.build-mode }} config-file: ./.github/codeql/codeql-config.yml - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v3 + uses: github/codeql-action/analyze@ebcb5b36ded6beda4ceefea6a8bc4cc885255bb3 # v3 with: category: "/language:${{ matrix.language }}" diff --git a/.github/workflows/codspeed.yml b/.github/workflows/codspeed.yml index 385b95fdaf5..52d64addea9 100644 --- a/.github/workflows/codspeed.yml +++ b/.github/workflows/codspeed.yml @@ -25,10 +25,12 @@ jobs: timeout-minutes: 15 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + with: + persist-credentials: false - name: Set up Python - uses: actions/setup-python@v5 + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version: "3.12" @@ -38,7 +40,7 @@ jobs: pip install pytest pytest-codspeed==4.3.0 - name: Run benchmarks - uses: CodSpeedHQ/action@v4 + uses: CodSpeedHQ/action@1c8ae4843586d3ba879736b7f6b7b0c990757fab # v4.12.1 with: mode: simulation run: pytest tests/benchmarks/ --codspeed diff --git a/.github/workflows/create_daily_staging_branch.yml b/.github/workflows/create_daily_staging_branch.yml index 08aebd7d04c..424d8de0a41 100644 --- a/.github/workflows/create_daily_staging_branch.yml +++ b/.github/workflows/create_daily_staging_branch.yml @@ -2,18 +2,22 @@ name: Create Daily Staging Branch on: schedule: - - cron: '0 0,12 * * *' # Runs every 12 hours at midnight and noon UTC - workflow_dispatch: # Allow manual trigger + - cron: "0 0,12 * * *" # Runs every 12 hours at midnight and noon UTC + workflow_dispatch: # Allow manual trigger jobs: create-staging-branch: + if: github.repository == 'BerriAI/litellm' runs-on: ubuntu-latest + permissions: + contents: write steps: - name: Checkout repository - uses: actions/checkout@v3 + uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 with: fetch-depth: 0 + persist-credentials: false - name: Create daily staging branch env: @@ -43,13 +47,17 @@ jobs: fi create-internal-dev-branch: + if: github.repository == 'BerriAI/litellm' runs-on: ubuntu-latest + permissions: + contents: write steps: - name: Checkout repository - uses: actions/checkout@v3 + uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 with: fetch-depth: 0 + persist-credentials: false - name: Create internal dev branch env: diff --git a/.github/workflows/ghcr_deploy.yml b/.github/workflows/ghcr_deploy.yml deleted file mode 100644 index 344b0ec48ee..00000000000 --- a/.github/workflows/ghcr_deploy.yml +++ /dev/null @@ -1,444 +0,0 @@ -# this workflow is triggered by an API call when there is a new PyPI release of LiteLLM -name: Build, Publish LiteLLM Docker Image. New Release -on: - workflow_dispatch: - inputs: - tag: - description: "The tag version you want to build" - required: true - release_type: - description: "The release type you want to build. Can be 'latest', 'stable', 'dev', 'rc'" - type: string - default: "latest" - commit_hash: - description: "Commit hash" - required: true - -# Defines two custom environment variables for the workflow. Used for the Container registry domain, and a name for the Docker image that this workflow builds. -env: - REGISTRY: ghcr.io - IMAGE_NAME: ${{ github.repository }} - CHART_NAME: litellm-helm - -# There is a single job in this workflow. It's configured to run on the latest available version of Ubuntu. -jobs: - # print commit hash, tag, and release type - print: - runs-on: ubuntu-latest - steps: - - run: | - echo "Commit hash: ${{ github.event.inputs.commit_hash }}" - echo "Tag: ${{ github.event.inputs.tag }}" - echo "Release type: ${{ github.event.inputs.release_type }}" - docker-hub-deploy: - if: github.repository == 'BerriAI/litellm' - runs-on: ubuntu-latest - steps: - - - name: Checkout - uses: actions/checkout@v4 - with: - ref: ${{ github.event.inputs.commit_hash }} - - - name: Set up QEMU - uses: docker/setup-qemu-action@v3 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - - name: Login to Docker Hub - uses: docker/login-action@v3 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - - name: Build and push - uses: docker/build-push-action@v5 - with: - context: . - push: true - tags: litellm/litellm:${{ github.event.inputs.tag || 'latest' }} - - - name: Build and push litellm-database image - uses: docker/build-push-action@v5 - with: - context: . - push: true - file: ./docker/Dockerfile.database - tags: litellm/litellm-database:${{ github.event.inputs.tag || 'latest' }} - - - name: Build and push litellm-spend-logs image - uses: docker/build-push-action@v5 - with: - context: . - push: true - file: ./litellm-js/spend-logs/Dockerfile - tags: litellm/litellm-spend_logs:${{ github.event.inputs.tag || 'latest' }} - - - name: Build and push litellm-non_root image - uses: docker/build-push-action@v5 - with: - context: . - push: true - file: ./docker/Dockerfile.non_root - tags: litellm/litellm-non_root:${{ github.event.inputs.tag || 'latest' }} - build-and-push-image: - runs-on: ubuntu-latest - # Sets the permissions granted to the `GITHUB_TOKEN` for the actions in this job. - permissions: - contents: read - packages: write - steps: - - name: Checkout repository - uses: actions/checkout@v4 - with: - ref: ${{ github.event.inputs.commit_hash }} - # Uses the `docker/login-action` action to log in to the Container registry registry using the account and password that will publish the packages. Once published, the packages are scoped to the account defined here. - - name: Log in to the Container registry - uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1 - with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - # This step uses [docker/metadata-action](https://github.com/docker/metadata-action#about) to extract tags and labels that will be applied to the specified image. The `id` "meta" allows the output of this step to be referenced in a subsequent step. The `images` value provides the base name for the tags and labels. - - name: Extract metadata (tags, labels) for Docker - id: meta - uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7 - with: - images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} - # Configure multi platform Docker builds - - name: Set up QEMU - uses: docker/setup-qemu-action@e0e4588fad221d38ee467c0bffd91115366dc0c5 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@edfb0fe6204400c56fbfd3feba3fe9ad1adfa345 - # This step uses the `docker/build-push-action` action to build the image, based on your repository's `Dockerfile`. If the build succeeds, it pushes the image to GitHub Packages. - # It uses the `context` parameter to define the build's context as the set of files located in the specified path. For more information, see "[Usage](https://github.com/docker/build-push-action#usage)" in the README of the `docker/build-push-action` repository. - # It uses the `tags` and `labels` parameters to tag and label the image with the output from the "meta" step. - - name: Build and push Docker image - uses: docker/build-push-action@4976231911ebf5f32aad765192d35f942aa48cb8 - with: - context: . - push: true - tags: | - ${{ steps.meta.outputs.tags }}-${{ github.event.inputs.tag || 'latest' }}, - ${{ steps.meta.outputs.tags }}-${{ github.event.inputs.release_type }} - ${{ (github.event.inputs.release_type == 'stable' || github.event.inputs.release_type == 'rc') && format('{0}/berriai/litellm:main-{1}', env.REGISTRY, github.event.inputs.tag) || '' }}, - ${{ github.event.inputs.release_type == 'stable' && format('{0}/berriai/litellm:main-stable', env.REGISTRY) || '' }}, - ${{ (github.event.inputs.release_type == 'stable' || github.event.inputs.release_type == 'rc') && format('{0}/berriai/litellm:{1}', env.REGISTRY, github.event.inputs.tag) || '' }}, - labels: ${{ steps.meta.outputs.labels }} - platforms: local,linux/amd64,linux/arm64,linux/arm64/v8 - - build-and-push-image-ee: - runs-on: ubuntu-latest - permissions: - contents: read - packages: write - steps: - - name: Checkout repository - uses: actions/checkout@v4 - with: - ref: ${{ github.event.inputs.commit_hash }} - - - name: Log in to the Container registry - uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1 - with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Extract metadata (tags, labels) for EE Dockerfile - id: meta-ee - uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7 - with: - images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee - # Configure multi platform Docker builds - - name: Set up QEMU - uses: docker/setup-qemu-action@e0e4588fad221d38ee467c0bffd91115366dc0c5 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@edfb0fe6204400c56fbfd3feba3fe9ad1adfa345 - - - name: Build and push EE Docker image - uses: docker/build-push-action@f2a1d5e99d037542a71f64918e516c093c6f3fc4 - with: - context: . - file: Dockerfile - push: true - tags: | - ${{ steps.meta-ee.outputs.tags }}-${{ github.event.inputs.tag || 'latest' }}, - ${{ steps.meta-ee.outputs.tags }}-${{ github.event.inputs.release_type }} - ${{ (github.event.inputs.release_type == 'stable' || github.event.inputs.release_type == 'rc') && format('{0}/berriai/litellm-ee:main-{1}', env.REGISTRY, github.event.inputs.tag) || '' }}, - ${{ github.event.inputs.release_type == 'stable' && format('{0}/berriai/litellm-ee:main-stable', env.REGISTRY) || '' }} - labels: ${{ steps.meta-ee.outputs.labels }} - platforms: local,linux/amd64,linux/arm64,linux/arm64/v8 - - build-and-push-image-database: - runs-on: ubuntu-latest - permissions: - contents: read - packages: write - steps: - - name: Checkout repository - uses: actions/checkout@v4 - with: - ref: ${{ github.event.inputs.commit_hash }} - - - name: Log in to the Container registry - uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1 - with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Extract metadata (tags, labels) for database Dockerfile - id: meta-database - uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7 - with: - images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-database - # Configure multi platform Docker builds - - name: Set up QEMU - uses: docker/setup-qemu-action@e0e4588fad221d38ee467c0bffd91115366dc0c5 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@edfb0fe6204400c56fbfd3feba3fe9ad1adfa345 - - - name: Build and push Database Docker image - uses: docker/build-push-action@f2a1d5e99d037542a71f64918e516c093c6f3fc4 - with: - context: . - file: ./docker/Dockerfile.database - push: true - tags: | - ${{ steps.meta-database.outputs.tags }}-${{ github.event.inputs.tag || 'latest' }}, - ${{ steps.meta-database.outputs.tags }}-${{ github.event.inputs.release_type }} - ${{ (github.event.inputs.release_type == 'stable' || github.event.inputs.release_type == 'rc') && format('{0}/berriai/litellm-database:main-{1}', env.REGISTRY, github.event.inputs.tag) || '' }}, - ${{ github.event.inputs.release_type == 'stable' && format('{0}/berriai/litellm-database:main-stable', env.REGISTRY) || '' }} - labels: ${{ steps.meta-database.outputs.labels }} - platforms: local,linux/amd64,linux/arm64,linux/arm64/v8 - - build-and-push-image-non_root: - runs-on: ubuntu-latest - permissions: - contents: read - packages: write - steps: - - name: Checkout repository - uses: actions/checkout@v4 - with: - ref: ${{ github.event.inputs.commit_hash }} - - - name: Log in to the Container registry - uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1 - with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Extract metadata (tags, labels) for non_root Dockerfile - id: meta-non_root - uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7 - with: - images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-non_root - # Configure multi platform Docker builds - - name: Set up QEMU - uses: docker/setup-qemu-action@e0e4588fad221d38ee467c0bffd91115366dc0c5 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@edfb0fe6204400c56fbfd3feba3fe9ad1adfa345 - - - name: Build and push non_root Docker image - uses: docker/build-push-action@f2a1d5e99d037542a71f64918e516c093c6f3fc4 - with: - context: . - file: ./docker/Dockerfile.non_root - push: true - tags: | - ${{ steps.meta-non_root.outputs.tags }}-${{ github.event.inputs.tag || 'latest' }}, - ${{ steps.meta-non_root.outputs.tags }}-${{ github.event.inputs.release_type }} - ${{ (github.event.inputs.release_type == 'stable' || github.event.inputs.release_type == 'rc') && format('{0}/berriai/litellm-non_root:main-{1}', env.REGISTRY, github.event.inputs.tag) || '' }}, - ${{ github.event.inputs.release_type == 'stable' && format('{0}/berriai/litellm-non_root:main-stable', env.REGISTRY) || '' }} - labels: ${{ steps.meta-non_root.outputs.labels }} - platforms: local,linux/amd64,linux/arm64,linux/arm64/v8 - - build-and-push-image-spend-logs: - runs-on: ubuntu-latest - permissions: - contents: read - packages: write - steps: - - name: Checkout repository - uses: actions/checkout@v4 - with: - ref: ${{ github.event.inputs.commit_hash }} - - - name: Log in to the Container registry - uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1 - with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Extract metadata (tags, labels) for spend-logs Dockerfile - id: meta-spend-logs - uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7 - with: - images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-spend_logs - # Configure multi platform Docker builds - - name: Set up QEMU - uses: docker/setup-qemu-action@e0e4588fad221d38ee467c0bffd91115366dc0c5 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@edfb0fe6204400c56fbfd3feba3fe9ad1adfa345 - - - name: Build and push Database Docker image - uses: docker/build-push-action@f2a1d5e99d037542a71f64918e516c093c6f3fc4 - with: - context: . - file: ./litellm-js/spend-logs/Dockerfile - push: true - tags: | - ${{ steps.meta-spend-logs.outputs.tags }}-${{ github.event.inputs.tag || 'latest' }}, - ${{ steps.meta-spend-logs.outputs.tags }}-${{ github.event.inputs.release_type }} - ${{ (github.event.inputs.release_type == 'stable' || github.event.inputs.release_type == 'rc') && format('{0}/berriai/litellm-spend_logs:main-{1}', env.REGISTRY, github.event.inputs.tag) || '' }}, - ${{ github.event.inputs.release_type == 'stable' && format('{0}/berriai/litellm-spend_logs:main-stable', env.REGISTRY) || '' }} - platforms: local,linux/amd64,linux/arm64,linux/arm64/v8 - - run-observatory-tests: - if: github.event.inputs.release_type == 'rc' || github.event.inputs.release_type == 'stable' - needs: [docker-hub-deploy] - uses: ./.github/workflows/run_observatory_tests.yml - with: - tag: ${{ github.event.inputs.tag }} - commit_hash: ${{ github.event.inputs.commit_hash }} - secrets: inherit - - build-and-push-helm-chart: - if: github.event.inputs.release_type != 'dev' - needs: [docker-hub-deploy, build-and-push-image, build-and-push-image-database] - runs-on: ubuntu-latest - steps: - - name: Checkout repository - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - name: Log in to the Container registry - uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1 - with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: lowercase github.repository_owner - run: | - echo "REPO_OWNER=`echo ${{github.repository_owner}} | tr '[:upper:]' '[:lower:]'`" >>${GITHUB_ENV} - - # Sync Helm chart version with LiteLLM release version (1-1 versioning) - # This allows users to easily map Helm chart versions to LiteLLM versions - # See: https://codefresh.io/docs/docs/ci-cd-guides/helm-best-practices/ - - name: Calculate chart and app versions - id: chart_version - shell: bash - run: | - INPUT_TAG="${{ github.event.inputs.tag }}" - RELEASE_TYPE="${{ github.event.inputs.release_type }}" - - # Chart version = LiteLLM version without 'v' prefix (Helm semver convention) - # v1.81.0 -> 1.81.0, v1.81.0.rc.1 -> 1.81.0.rc.1 - CHART_VERSION="${INPUT_TAG#v}" - - # Add suffix for 'latest' releases (rc already has suffix in tag) - if [ "$RELEASE_TYPE" = "latest" ]; then - CHART_VERSION="${CHART_VERSION}-latest" - fi - - # App version = Docker tag (keeps 'v' prefix to match Docker image tags) - APP_VERSION="${INPUT_TAG}" - - echo "version=${CHART_VERSION}" | tee -a $GITHUB_OUTPUT - echo "app_version=${APP_VERSION}" | tee -a $GITHUB_OUTPUT - - - uses: ./.github/actions/helm-oci-chart-releaser - with: - name: ${{ env.CHART_NAME }} - repository: ${{ env.REPO_OWNER }} - tag: ${{ steps.chart_version.outputs.version }} - app_version: ${{ steps.chart_version.outputs.app_version }} - path: deploy/charts/${{ env.CHART_NAME }} - registry: ${{ env.REGISTRY }} - registry_username: ${{ github.actor }} - registry_password: ${{ secrets.GITHUB_TOKEN }} - update_dependencies: true - - release: - name: "New LiteLLM Release" - needs: [docker-hub-deploy, build-and-push-image, build-and-push-image-database] - permissions: - contents: write - runs-on: "ubuntu-latest" - - steps: - - name: Display version - run: echo "Current version is ${{ github.event.inputs.tag }}" - - name: "Set Release Tag" - run: echo "RELEASE_TAG=${{ github.event.inputs.tag }}" >> $GITHUB_ENV - - name: Display release tag - run: echo "RELEASE_TAG is $RELEASE_TAG" - - name: "Create release" - uses: "actions/github-script@v6" - with: - github-token: "${{ secrets.GITHUB_TOKEN }}" - script: | - const commitHash = "${{ github.event.inputs.commit_hash}}"; - console.log("Commit Hash:", commitHash); // Add this line for debugging - try { - const response = await github.rest.repos.createRelease({ - draft: false, - generate_release_notes: true, - target_commitish: commitHash, - name: process.env.RELEASE_TAG, - owner: context.repo.owner, - prerelease: false, - repo: context.repo.repo, - tag_name: process.env.RELEASE_TAG, - }); - - core.exportVariable('RELEASE_ID', response.data.id); - core.exportVariable('RELEASE_UPLOAD_URL', response.data.upload_url); - } catch (error) { - core.setFailed(error.message); - } - - name: Fetch Release Notes - id: release-notes - uses: actions/github-script@v6 - with: - github-token: "${{ secrets.GITHUB_TOKEN }}" - script: | - try { - const response = await github.rest.repos.getRelease({ - owner: context.repo.owner, - repo: context.repo.repo, - release_id: process.env.RELEASE_ID, - }); - const formattedBody = JSON.stringify(response.data.body).slice(1, -1); - return formattedBody; - } catch (error) { - core.setFailed(error.message); - } - env: - RELEASE_ID: ${{ env.RELEASE_ID }} - - name: Github Releases To Discord - env: - WEBHOOK_URL: ${{ secrets.WEBHOOK_URL }} - REALEASE_TAG: ${{ env.RELEASE_TAG }} - RELEASE_NOTES: ${{ steps.release-notes.outputs.result }} - run: | - curl -H "Content-Type: application/json" -X POST -d '{ - "content": "New LiteLLM release '"${RELEASE_TAG}"'", - "username": "Release Changelog", - "avatar_url": "https://cdn.discordapp.com/avatars/487431320314576937/bd64361e4ba6313d561d54e78c9e7171.png", - "embeds": [ - { - "title": "Changelog for LiteLLM '"${RELEASE_TAG}"'", - "description": "'"${RELEASE_NOTES}"'", - "color": 2105893 - } - ] - }' $WEBHOOK_URL - diff --git a/.github/workflows/ghcr_helm_deploy.yml b/.github/workflows/ghcr_helm_deploy.yml deleted file mode 100644 index 21b2eaafe19..00000000000 --- a/.github/workflows/ghcr_helm_deploy.yml +++ /dev/null @@ -1,67 +0,0 @@ -# Standalone workflow to publish LiteLLM Helm Chart -# Note: The main ghcr_deploy.yml workflow also publishes the Helm chart as part of a full release -name: Build, Publish LiteLLM Helm Chart. New Release -on: - workflow_dispatch: - inputs: - tag: - description: "LiteLLM version tag (e.g., v1.81.0)" - required: true - -# Defines two custom environment variables for the workflow. Used for the Container registry domain, and a name for the Docker image that this workflow builds. -env: - REGISTRY: ghcr.io - IMAGE_NAME: ${{ github.repository }} - REPO_OWNER: ${{github.repository_owner}} - -# There is a single job in this workflow. It's configured to run on the latest available version of Ubuntu. -jobs: - build-and-push-helm-chart: - runs-on: ubuntu-latest - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - - name: Log in to the Container registry - uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1 - with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: lowercase github.repository_owner - run: | - echo "REPO_OWNER=`echo ${{github.repository_owner}} | tr '[:upper:]' '[:lower:]'`" >>${GITHUB_ENV} - - # Sync Helm chart version with LiteLLM release version (1-1 versioning) - - name: Calculate chart and app versions - id: chart_version - shell: bash - run: | - INPUT_TAG="${{ github.event.inputs.tag }}" - - # Chart version = LiteLLM version without 'v' prefix - # v1.81.0 -> 1.81.0 - CHART_VERSION="${INPUT_TAG#v}" - - # App version = Docker tag (keeps 'v' prefix) - APP_VERSION="${INPUT_TAG}" - - echo "version=${CHART_VERSION}" | tee -a $GITHUB_OUTPUT - echo "app_version=${APP_VERSION}" | tee -a $GITHUB_OUTPUT - - - name: Lint helm chart - run: helm lint deploy/charts/litellm-helm - - - uses: ./.github/actions/helm-oci-chart-releaser - with: - name: litellm-helm - repository: ${{ env.REPO_OWNER }} - tag: ${{ steps.chart_version.outputs.version }} - app_version: ${{ steps.chart_version.outputs.app_version }} - path: deploy/charts/litellm-helm - registry: ${{ env.REGISTRY }} - registry_username: ${{ github.actor }} - registry_password: ${{ secrets.GITHUB_TOKEN }} - update_dependencies: true - diff --git a/.github/workflows/helm_unit_test.yml b/.github/workflows/helm_unit_test.yml index c4b83af70a1..06836b1d1cd 100644 --- a/.github/workflows/helm_unit_test.yml +++ b/.github/workflows/helm_unit_test.yml @@ -6,22 +6,36 @@ on: branches: - main +permissions: + contents: read + jobs: unit-test: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + with: + persist-credentials: false - name: Set up Helm 3.11.1 - uses: azure/setup-helm@v1 + uses: azure/setup-helm@1a275c3b69536ee54be43f2070a358922e12c8d4 # v4.3.1 with: - version: '3.11.1' + version: "3.11.1" - name: Install Helm Unit Test Plugin run: | helm plugin install https://github.com/helm-unittest/helm-unittest --version v0.4.4 + - name: Verify Helm Unit Test Plugin integrity + run: | + EXPECTED_SHA="e251ba198448629678ff2168e1a469249d998155" + PLUGIN_DIR="$(helm env HELM_PLUGINS)/helm-unittest" + ACTUAL_SHA="$(git -C "$PLUGIN_DIR" rev-parse HEAD)" + if [ "$ACTUAL_SHA" != "$EXPECTED_SHA" ]; then + echo "::error::Helm unittest plugin checksum mismatch! Expected $EXPECTED_SHA but got $ACTUAL_SHA" + exit 1 + fi + echo "Helm unittest plugin integrity verified: $ACTUAL_SHA" - name: Run unit tests - run: - helm unittest -f 'tests/*.yaml' deploy/charts/litellm-helm \ No newline at end of file + run: helm unittest -f 'tests/*.yaml' deploy/charts/litellm-helm diff --git a/.github/workflows/interpret_load_test.py b/.github/workflows/interpret_load_test.py deleted file mode 100644 index 348ff300fff..00000000000 --- a/.github/workflows/interpret_load_test.py +++ /dev/null @@ -1,139 +0,0 @@ -import csv -import os -from github import Github - - -def interpret_results(csv_file): - with open(csv_file, newline="") as csvfile: - csvreader = csv.DictReader(csvfile) - rows = list(csvreader) - """ - in this csv reader - - Create 1 new column "Status" - - if a row has a median response time < 300 and an average response time < 300, Status = "Passed ✅" - - if a row has a median response time >= 300 or an average response time >= 300, Status = "Failed ❌" - - Order the table in this order Name, Status, Median Response Time, Average Response Time, Requests/s,Failures/s, Min Response Time, Max Response Time, all other columns - """ - - # Add a new column "Status" - for row in rows: - median_response_time = float( - row["Median Response Time"].strip().rstrip("ms") - ) - average_response_time = float( - row["Average Response Time"].strip().rstrip("s") - ) - - request_count = int(row["Request Count"]) - failure_count = int(row["Failure Count"]) - - failure_percent = round((failure_count / request_count) * 100, 2) - - # Determine status based on conditions - if ( - median_response_time < 300 - and average_response_time < 300 - and failure_percent < 5 - ): - row["Status"] = "Passed ✅" - else: - row["Status"] = "Failed ❌" - - # Construct Markdown table header - markdown_table = "| Name | Status | Median Response Time (ms) | Average Response Time (ms) | Requests/s | Failures/s | Request Count | Failure Count | Min Response Time (ms) | Max Response Time (ms) |" - markdown_table += ( - "\n| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |" - ) - - # Construct Markdown table rows - for row in rows: - markdown_table += f"\n| {row['Name']} | {row['Status']} | {row['Median Response Time']} | {row['Average Response Time']} | {row['Requests/s']} | {row['Failures/s']} | {row['Request Count']} | {row['Failure Count']} | {row['Min Response Time']} | {row['Max Response Time']} |" - print("markdown table: ", markdown_table) - return markdown_table - - -def _get_docker_run_command_stable_release(release_version): - return f""" -\n\n -## Docker Run LiteLLM Proxy - -``` -docker run \\ --e STORE_MODEL_IN_DB=True \\ --p 4000:4000 \\ -ghcr.io/berriai/litellm:litellm_stable_release_branch-{release_version} -``` - """ - - -def _get_docker_run_command(release_version): - return f""" -\n\n -## Docker Run LiteLLM Proxy - -``` -docker run \\ --e STORE_MODEL_IN_DB=True \\ --p 4000:4000 \\ -ghcr.io/berriai/litellm:main-{release_version} -``` - """ - - -def get_docker_run_command(release_version): - if "stable" in release_version: - return _get_docker_run_command_stable_release(release_version) - else: - return _get_docker_run_command(release_version) - - -if __name__ == "__main__": - return - csv_file = "load_test_stats.csv" # Change this to the path of your CSV file - markdown_table = interpret_results(csv_file) - - # Update release body with interpreted results - github_token = os.getenv("GITHUB_TOKEN") - g = Github(github_token) - repo = g.get_repo( - "BerriAI/litellm" - ) # Replace with your repository's username and name - latest_release = repo.get_latest_release() - print("got latest release: ", latest_release) - print(latest_release.title) - print(latest_release.tag_name) - - release_version = latest_release.title - - print("latest release body: ", latest_release.body) - print("markdown table: ", markdown_table) - - # check if "Load Test LiteLLM Proxy Results" exists - existing_release_body = latest_release.body - if "Load Test LiteLLM Proxy Results" in latest_release.body: - # find the "Load Test LiteLLM Proxy Results" section and delete it - start_index = latest_release.body.find("Load Test LiteLLM Proxy Results") - existing_release_body = latest_release.body[:start_index] - - docker_run_command = get_docker_run_command(release_version) - print("docker run command: ", docker_run_command) - - new_release_body = ( - existing_release_body - + docker_run_command - + "\n\n" - + "### Don't want to maintain your internal proxy? get in touch 🎉" - + "\nHosted Proxy Alpha: https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions" - + "\n\n" - + "## Load Test LiteLLM Proxy Results" - + "\n\n" - + markdown_table - ) - print("new release body: ", new_release_body) - try: - latest_release.update_release( - name=latest_release.tag_name, - message=new_release_body, - ) - except Exception as e: - print(e) diff --git a/.github/workflows/issue-keyword-labeler.yml b/.github/workflows/issue-keyword-labeler.yml index 936f90f747f..7e2693209b6 100644 --- a/.github/workflows/issue-keyword-labeler.yml +++ b/.github/workflows/issue-keyword-labeler.yml @@ -2,8 +2,8 @@ name: Issue Keyword Labeler on: issues: - types: - - opened + types: + - opened jobs: scan-and-label: @@ -13,7 +13,9 @@ jobs: contents: read steps: - name: Checkout code - uses: actions/checkout@v4 + uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + with: + persist-credentials: false - name: Scan for provider keywords id: scan @@ -24,7 +26,7 @@ jobs: - name: Ensure label exists if: steps.scan.outputs.found == 'true' - uses: actions/github-script@v7 + uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7.1.0 with: github-token: ${{ secrets.GITHUB_TOKEN }} script: | @@ -51,7 +53,7 @@ jobs: - name: Add label to the issue if: steps.scan.outputs.found == 'true' - uses: actions/github-script@v7 + uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7.1.0 with: github-token: ${{ secrets.GITHUB_TOKEN }} script: | @@ -61,4 +63,3 @@ jobs: issue_number: context.issue.number, labels: ['llm translation'] }); - diff --git a/.github/workflows/label-component.yml b/.github/workflows/label-component.yml index fd079fce6c1..e0c2fa94d8c 100644 --- a/.github/workflows/label-component.yml +++ b/.github/workflows/label-component.yml @@ -12,7 +12,7 @@ jobs: issues: write steps: - name: Add component labels - uses: actions/github-script@v7 + uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7.1.0 with: github-token: ${{ secrets.GITHUB_TOKEN }} script: | diff --git a/.github/workflows/llm-translation-testing.yml b/.github/workflows/llm-translation-testing.yml index 7fda37a66dc..922013c4b54 100644 --- a/.github/workflows/llm-translation-testing.yml +++ b/.github/workflows/llm-translation-testing.yml @@ -4,38 +4,41 @@ on: workflow_dispatch: inputs: release_candidate_tag: - description: 'Release candidate tag/version' + description: "Release candidate tag/version" required: true type: string push: tags: - - 'v*-rc*' # Triggers on release candidate tags like v1.0.0-rc1 - + - "v*-rc*" # Triggers on release candidate tags like v1.0.0-rc1 + +permissions: + contents: read + jobs: run-llm-translation-tests: runs-on: ubuntu-latest timeout-minutes: 90 - + steps: - name: Checkout code - uses: actions/checkout@v4 + uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 with: + persist-credentials: false ref: ${{ github.event.inputs.release_candidate_tag || github.ref }} - + - name: Set up Python - uses: actions/setup-python@v5 + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: - python-version: '3.11' - + python-version: "3.11" + - name: Install Poetry - uses: snok/install-poetry@v1 - with: - version: latest - virtualenvs-create: true - virtualenvs-in-project: true - - - name: Cache Poetry dependencies - uses: actions/cache@v3 + run: | + pip install 'poetry==2.3.2' + poetry config virtualenvs.create true + poetry config virtualenvs.in-project true + + - name: Restore Poetry dependencies cache + uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.0.0 with: path: | ~/.cache/pypoetry @@ -43,15 +46,15 @@ jobs: key: ${{ runner.os }}-poetry-${{ hashFiles('**/poetry.lock') }} restore-keys: | ${{ runner.os }}-poetry- - + - name: Install dependencies run: | poetry install --with dev - poetry run pip install pytest-xdist pytest-timeout - + poetry run pip install 'pytest-xdist==3.8.0' 'pytest-timeout==2.4.0' + - name: Create test results directory run: mkdir -p test-results - + - name: Run LLM Translation Tests env: OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} @@ -61,13 +64,14 @@ jobs: AZURE_API_KEY: ${{ secrets.AZURE_API_KEY }} AZURE_API_BASE: ${{ secrets.AZURE_API_BASE }} AZURE_API_VERSION: ${{ secrets.AZURE_API_VERSION }} - # Add other API keys as needed + RC_TAG: ${{ github.event.inputs.release_candidate_tag || github.ref_name }} + COMMIT_SHA: ${{ github.sha }} run: | python .github/workflows/run_llm_translation_tests.py \ - --tag "${{ github.event.inputs.release_candidate_tag || github.ref_name }}" \ - --commit "${{ github.sha }}" \ + --tag "$RC_TAG" \ + --commit "$COMMIT_SHA" \ || true # Continue even if tests fail - + - name: Display test summary if: always() run: | @@ -79,9 +83,9 @@ jobs: else echo "Warning: Test report was not generated" fi - + - name: Upload test artifacts - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 if: always() with: name: LLM-Translation-Artifact-${{ github.event.inputs.release_candidate_tag || github.ref_name }} diff --git a/.github/workflows/load_test.yml b/.github/workflows/load_test.yml deleted file mode 100644 index cdaffa328c9..00000000000 --- a/.github/workflows/load_test.yml +++ /dev/null @@ -1,59 +0,0 @@ -name: Test Locust Load Test - -on: - workflow_run: - workflows: ["Build, Publish LiteLLM Docker Image. New Release"] - types: - - completed - workflow_dispatch: - -jobs: - build: - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v1 - - name: Setup Python - uses: actions/setup-python@v2 - with: - python-version: '3.x' - - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install PyGithub - - name: re-deploy proxy - run: | - echo "Current working directory: $PWD" - ls - python ".github/workflows/redeploy_proxy.py" - env: - LOAD_TEST_REDEPLOY_URL1: ${{ secrets.LOAD_TEST_REDEPLOY_URL1 }} - LOAD_TEST_REDEPLOY_URL2: ${{ secrets.LOAD_TEST_REDEPLOY_URL2 }} - working-directory: ${{ github.workspace }} - - name: Run Load Test - id: locust_run - uses: BerriAI/locust-github-action@master - with: - LOCUSTFILE: ".github/workflows/locustfile.py" - URL: "https://post-release-load-test-proxy.onrender.com/" - USERS: "20" - RATE: "20" - RUNTIME: "300s" - - name: Process Load Test Stats - run: | - echo "Current working directory: $PWD" - ls - python ".github/workflows/interpret_load_test.py" - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - working-directory: ${{ github.workspace }} - - name: Upload CSV as Asset to Latest Release - uses: xresloader/upload-to-github-release@v1 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - file: "load_test_stats.csv;load_test.html" - update_latest_release: true - tag_name: "load-test" - overwrite: true \ No newline at end of file diff --git a/.github/workflows/locustfile.py b/.github/workflows/locustfile.py deleted file mode 100644 index 36dbeee9c48..00000000000 --- a/.github/workflows/locustfile.py +++ /dev/null @@ -1,28 +0,0 @@ -from locust import HttpUser, task, between - - -class MyUser(HttpUser): - wait_time = between(1, 5) - - @task - def chat_completion(self): - headers = { - "Content-Type": "application/json", - "Authorization": "Bearer sk-8N1tLOOyH8TIxwOLahhIVg", - # Include any additional headers you may need for authentication, etc. - } - - # Customize the payload with "model" and "messages" keys - payload = { - "model": "fake-openai-endpoint", - "messages": [ - {"role": "system", "content": "You are a chat bot."}, - {"role": "user", "content": "Hello, how are you?"}, - ], - # Add more data as necessary - } - - # Make a POST request to the "chat/completions" endpoint - response = self.client.post("chat/completions", json=payload, headers=headers) - - # Print or log the response if needed diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml deleted file mode 100644 index 23e4a06da9e..00000000000 --- a/.github/workflows/main.yml +++ /dev/null @@ -1,34 +0,0 @@ -name: Publish Dev Release to PyPI - -on: - workflow_dispatch: - -jobs: - publish-dev-release: - runs-on: ubuntu-latest - - steps: - - name: Checkout code - uses: actions/checkout@v2 - - - name: Set up Python - uses: actions/setup-python@v2 - with: - python-version: 3.8 # Adjust the Python version as needed - - - name: Install dependencies - run: pip install toml twine - - - name: Read version from pyproject.toml - id: read-version - run: | - version=$(python -c 'import toml; print(toml.load("pyproject.toml")["tool"]["commitizen"]["version"])') - printf "LITELLM_VERSION=%s" "$version" >> $GITHUB_ENV - - - name: Check if version exists on PyPI - id: check-version - run: | - set -e - if twine check --repository-url https://pypi.org/simple/ "litellm==$LITELLM_VERSION" >/dev/null 2>&1; then - echo "Version $LITELLM_VERSION already exists on PyPI. Skipping publish." - diff --git a/.github/workflows/publish-migrations.yml b/.github/workflows/publish-migrations.yml deleted file mode 100644 index a5187cb2f55..00000000000 --- a/.github/workflows/publish-migrations.yml +++ /dev/null @@ -1,207 +0,0 @@ -name: Publish Prisma Migrations - -permissions: - contents: write - pull-requests: write - -on: - push: - paths: - - 'schema.prisma' # Check root schema.prisma - branches: - - main - -jobs: - publish-migrations: - if: github.repository == 'BerriAI/litellm' - runs-on: ubuntu-latest - services: - postgres: - image: postgres:14 - env: - POSTGRES_DB: temp_db - POSTGRES_USER: postgres - POSTGRES_PASSWORD: postgres - ports: - - 5432:5432 - options: >- - --health-cmd pg_isready - --health-interval 10s - --health-timeout 5s - --health-retries 5 - - # Add shadow database service - postgres_shadow: - image: postgres:14 - env: - POSTGRES_DB: shadow_db - POSTGRES_USER: postgres - POSTGRES_PASSWORD: postgres - ports: - - 5433:5432 - options: >- - --health-cmd pg_isready - --health-interval 10s - --health-timeout 5s - --health-retries 5 - - steps: - - uses: actions/checkout@v3 - - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: '3.x' - - - name: Install Dependencies - run: | - pip install prisma - pip install python-dotenv - - - name: Generate Initial Migration if None Exists - env: - DATABASE_URL: "postgresql://postgres:postgres@localhost:5432/temp_db" - DIRECT_URL: "postgresql://postgres:postgres@localhost:5432/temp_db" - SHADOW_DATABASE_URL: "postgresql://postgres:postgres@localhost:5433/shadow_db" - run: | - mkdir -p deploy/migrations - echo 'provider = "postgresql"' > deploy/migrations/migration_lock.toml - - if [ -z "$(ls -A deploy/migrations/2* 2>/dev/null)" ]; then - echo "No existing migrations found, creating baseline..." - VERSION=$(date +%Y%m%d%H%M%S) - mkdir -p deploy/migrations/${VERSION}_initial - - echo "Generating initial migration..." - # Save raw output for debugging - prisma migrate diff \ - --from-empty \ - --to-schema-datamodel schema.prisma \ - --shadow-database-url "${SHADOW_DATABASE_URL}" \ - --script > deploy/migrations/${VERSION}_initial/raw_migration.sql - - echo "Raw migration file content:" - cat deploy/migrations/${VERSION}_initial/raw_migration.sql - - echo "Cleaning migration file..." - # Clean the file - sed '/^Installing/d' deploy/migrations/${VERSION}_initial/raw_migration.sql > deploy/migrations/${VERSION}_initial/migration.sql - - # Verify the migration file - if [ ! -s deploy/migrations/${VERSION}_initial/migration.sql ]; then - echo "ERROR: Migration file is empty after cleaning" - echo "Original content was:" - cat deploy/migrations/${VERSION}_initial/raw_migration.sql - exit 1 - fi - - echo "Final migration file content:" - cat deploy/migrations/${VERSION}_initial/migration.sql - - # Verify it starts with SQL - if ! head -n 1 deploy/migrations/${VERSION}_initial/migration.sql | grep -q "^--\|^CREATE\|^ALTER"; then - echo "ERROR: Migration file does not start with SQL command or comment" - echo "First line is:" - head -n 1 deploy/migrations/${VERSION}_initial/migration.sql - echo "Full content is:" - cat deploy/migrations/${VERSION}_initial/migration.sql - exit 1 - fi - - echo "Initial migration generated at $(date -u)" > deploy/migrations/${VERSION}_initial/README.md - fi - - - name: Compare and Generate Migration - if: success() - env: - DATABASE_URL: "postgresql://postgres:postgres@localhost:5432/temp_db" - DIRECT_URL: "postgresql://postgres:postgres@localhost:5432/temp_db" - SHADOW_DATABASE_URL: "postgresql://postgres:postgres@localhost:5433/shadow_db" - run: | - # Create temporary migration workspace - mkdir -p temp_migrations - - # Copy existing migrations (will not fail if directory is empty) - cp -r deploy/migrations/* temp_migrations/ 2>/dev/null || true - - VERSION=$(date +%Y%m%d%H%M%S) - - # Generate diff against existing migrations or empty state - prisma migrate diff \ - --from-migrations temp_migrations \ - --to-schema-datamodel schema.prisma \ - --shadow-database-url "${SHADOW_DATABASE_URL}" \ - --script > temp_migrations/migration_${VERSION}.sql - - # Check if there are actual changes - if [ -s temp_migrations/migration_${VERSION}.sql ]; then - echo "Changes detected, creating new migration" - mkdir -p deploy/migrations/${VERSION}_schema_update - mv temp_migrations/migration_${VERSION}.sql deploy/migrations/${VERSION}_schema_update/migration.sql - echo "Migration generated at $(date -u)" > deploy/migrations/${VERSION}_schema_update/README.md - else - echo "No schema changes detected" - exit 0 - fi - - - name: Verify Migration - if: success() - env: - DATABASE_URL: "postgresql://postgres:postgres@localhost:5432/temp_db" - DIRECT_URL: "postgresql://postgres:postgres@localhost:5432/temp_db" - SHADOW_DATABASE_URL: "postgresql://postgres:postgres@localhost:5433/shadow_db" - run: | - # Create test database - psql "${SHADOW_DATABASE_URL}" -c 'CREATE DATABASE migration_test;' - - # Apply all migrations in order to verify - for migration in deploy/migrations/*/migration.sql; do - echo "Applying migration: $migration" - psql "${SHADOW_DATABASE_URL}" -f $migration - done - - # Add this step before create-pull-request to debug permissions - - name: Check Token Permissions - run: | - echo "Checking token permissions..." - curl -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \ - -H "Accept: application/vnd.github.v3+json" \ - https://api.github.com/repos/BerriAI/litellm/collaborators - - echo "\nChecking if token can create PRs..." - curl -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \ - -H "Accept: application/vnd.github.v3+json" \ - https://api.github.com/repos/BerriAI/litellm - - # Add this debug step before git push - - name: Debug Changed Files - run: | - echo "Files staged for commit:" - git diff --name-status --staged - - echo "\nAll changed files:" - git status - - - name: Create Pull Request - if: success() - uses: peter-evans/create-pull-request@v5 - with: - token: ${{ secrets.GITHUB_TOKEN }} - commit-message: "chore: update prisma migrations" - title: "Update Prisma Migrations" - body: | - Auto-generated migration based on schema.prisma changes. - - Generated files: - - deploy/migrations/${VERSION}_schema_update/migration.sql - - deploy/migrations/${VERSION}_schema_update/README.md - branch: feat/prisma-migration-${{ env.VERSION }} - base: main - delete-branch: true - - - name: Generate and Save Migrations - run: | - # Only add migration files - git add deploy/migrations/ - git status # Debug what's being committed - git commit -m "chore: update prisma migrations" diff --git a/.github/workflows/publish_enterprise.yml b/.github/workflows/publish_enterprise.yml deleted file mode 100644 index 459a233cb71..00000000000 --- a/.github/workflows/publish_enterprise.yml +++ /dev/null @@ -1,94 +0,0 @@ -name: Publish litellm-enterprise to PyPI - -on: - workflow_dispatch: - inputs: - bump: - description: "Version bump type" - required: true - default: "patch" - type: choice - options: - - patch - - minor - - major - -jobs: - publish: - runs-on: ubuntu-latest - if: github.repository == 'BerriAI/litellm' - permissions: - contents: write - pull-requests: write - defaults: - run: - working-directory: enterprise - - steps: - - uses: actions/checkout@v4 - - - uses: actions/setup-python@v5 - with: - python-version: "3.11" - - - name: Install Poetry - run: pip install poetry - - - name: Bump version - id: bump - run: | - OLD=$(poetry version -s) - poetry version ${{ github.event.inputs.bump }} - NEW=$(poetry version -s) - echo "old=$OLD" >> $GITHUB_OUTPUT - echo "new=$NEW" >> $GITHUB_OUTPUT - - - name: Update version refs in root pyproject.toml and requirements.txt - run: | - OLD=${{ steps.bump.outputs.old }} - NEW=${{ steps.bump.outputs.new }} - sed -i "s/litellm-enterprise = {version = \"${OLD}\"/litellm-enterprise = {version = \"${NEW}\"/" ../pyproject.toml - sed -i "s/litellm-enterprise==${OLD}/litellm-enterprise==${NEW}/" ../requirements.txt - - - name: Update poetry.lock - working-directory: . - run: poetry lock - - - name: Build - run: poetry build - - - name: Commit version bump and create PR - id: create-pr - run: | - git config user.name "github-actions[bot]" - git config user.email "github-actions[bot]@users.noreply.github.com" - cd .. - BRANCH="bump/enterprise-${{ steps.bump.outputs.new }}" - git checkout -b "$BRANCH" - git add enterprise/pyproject.toml pyproject.toml requirements.txt poetry.lock - git commit -m "bump: litellm-enterprise ${{ steps.bump.outputs.old }} → ${{ steps.bump.outputs.new }}" - git push origin "$BRANCH" --force - gh pr create \ - --title "bump: litellm-enterprise ${{ steps.bump.outputs.old }} → ${{ steps.bump.outputs.new }}" \ - --body "Version bump for litellm-enterprise. Merge to update main." \ - --head "$BRANCH" \ - --base main \ - || true - PR_URL=$(gh pr list --head "$BRANCH" --json url -q '.[0].url') - echo "pr_url=$PR_URL" >> $GITHUB_OUTPUT - env: - GH_TOKEN: ${{ github.token }} - - - name: Enable auto-merge - run: | - gh pr merge "${{ steps.create-pr.outputs.pr_url }}" --auto --squash - env: - GH_TOKEN: ${{ github.token }} - - - name: Publish to PyPI - env: - TWINE_USERNAME: __token__ - TWINE_PASSWORD: ${{ secrets.PYPI_ENTERPRISE }} - run: | - pip install twine - twine upload dist/litellm_enterprise-${{ steps.bump.outputs.new }}* diff --git a/.github/workflows/publish_proxy_extras.yml b/.github/workflows/publish_proxy_extras.yml deleted file mode 100644 index fa30b153163..00000000000 --- a/.github/workflows/publish_proxy_extras.yml +++ /dev/null @@ -1,74 +0,0 @@ -name: Publish litellm-proxy-extras to PyPI - -on: - workflow_dispatch: - inputs: - bump: - description: "Version bump type" - required: true - default: "patch" - type: choice - options: - - patch - - minor - - major - -jobs: - publish: - runs-on: ubuntu-latest - if: github.repository == 'BerriAI/litellm' - permissions: - contents: write - defaults: - run: - working-directory: litellm-proxy-extras - - steps: - - uses: actions/checkout@v4 - - - uses: actions/setup-python@v5 - with: - python-version: "3.11" - - - name: Install Poetry - run: pip install poetry - - - name: Bump version - id: bump - run: | - OLD=$(poetry version -s) - poetry version ${{ github.event.inputs.bump }} - NEW=$(poetry version -s) - echo "old=$OLD" >> $GITHUB_OUTPUT - echo "new=$NEW" >> $GITHUB_OUTPUT - - - name: Update version refs in root pyproject.toml and requirements.txt - run: | - OLD=${{ steps.bump.outputs.old }} - NEW=${{ steps.bump.outputs.new }} - sed -i "s/litellm-proxy-extras = {version = \"${OLD}\"/litellm-proxy-extras = {version = \"${NEW}\"/" ../pyproject.toml - sed -i "s/litellm-proxy-extras==${OLD}/litellm-proxy-extras==${NEW}/" ../requirements.txt - - - name: Update poetry.lock - working-directory: . - run: poetry lock - - - name: Build - run: poetry build - - - name: Commit version bump - run: | - git config user.name "github-actions[bot]" - git config user.email "github-actions[bot]@users.noreply.github.com" - cd .. - git add litellm-proxy-extras/pyproject.toml pyproject.toml requirements.txt poetry.lock - git commit -m "bump: litellm-proxy-extras ${{ steps.bump.outputs.old }} → ${{ steps.bump.outputs.new }}" - git push - - - name: Publish to PyPI - env: - TWINE_USERNAME: __token__ - TWINE_PASSWORD: ${{ secrets.PYPI_PUBLISH_PASSWORD }} - run: | - pip install twine - twine upload dist/litellm_proxy_extras-${{ steps.bump.outputs.new }}* diff --git a/.github/workflows/publish_to_pypi.yml b/.github/workflows/publish_to_pypi.yml new file mode 100644 index 00000000000..8f675bb3075 --- /dev/null +++ b/.github/workflows/publish_to_pypi.yml @@ -0,0 +1,136 @@ +name: Publish to PyPI + +on: + workflow_dispatch: + +jobs: + preflight-checks: + name: Preflight Checks + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + contents: read + # No environment — read-only checks, no approval needed + outputs: + needs_publish: ${{ steps.check-litellm.outputs.needs_publish }} + version: ${{ steps.check-litellm.outputs.version }} + + steps: + - name: Checkout repo + uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + + - name: Set up Python + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: "3.12" + + - name: Check litellm version on PyPI + id: check-litellm + run: | + VERSION=$(grep -m1 '^version' pyproject.toml | sed 's/version = "\(.*\)"/\1/') + echo "version=$VERSION" >> "$GITHUB_OUTPUT" + echo "Checking if litellm $VERSION exists on PyPI..." + + HTTP_STATUS=$(curl -s -o /dev/null -w "%{http_code}" "https://pypi.org/pypi/litellm/$VERSION/json") + if [ "$HTTP_STATUS" = "200" ]; then + echo "litellm $VERSION already exists on PyPI. Skipping publish." + echo "needs_publish=false" >> "$GITHUB_OUTPUT" + else + echo "litellm $VERSION not found on PyPI. Publish needed." + echo "needs_publish=true" >> "$GITHUB_OUTPUT" + fi + + - name: Sanity check proxy-extras version + run: | + # Read pinned version from requirements.txt + REQ_VERSION=$(grep -oP 'litellm-proxy-extras==\K[0-9.]+' requirements.txt) + if [ -z "$REQ_VERSION" ]; then + echo "::error::Could not find litellm-proxy-extras version in requirements.txt" + exit 1 + fi + echo "requirements.txt pins litellm-proxy-extras==$REQ_VERSION" + + # Read pinned version from pyproject.toml dependency + PYPROJECT_VERSION=$(python3 -c " + import re + with open('pyproject.toml') as f: + content = f.read() + match = re.search(r'litellm-proxy-extras\s*=\s*\{version\s*=\s*\"([^\"]+)\"', content) + if match: + print(match.group(1).lstrip('^~>=')) + else: + import sys + print('::error::Could not find litellm-proxy-extras dependency in pyproject.toml', file=sys.stderr) + sys.exit(1) + ") + echo "pyproject.toml pins litellm-proxy-extras version: $PYPROJECT_VERSION" + + # Check that both pinned versions match + if [ "$REQ_VERSION" != "$PYPROJECT_VERSION" ]; then + echo "::error::Version mismatch: requirements.txt has $REQ_VERSION but pyproject.toml has $PYPROJECT_VERSION" + exit 1 + fi + + # Check that the pinned version exists on PyPI + echo "Checking if litellm-proxy-extras $REQ_VERSION exists on PyPI..." + HTTP_STATUS=$(curl -s -o /dev/null -w "%{http_code}" "https://pypi.org/pypi/litellm-proxy-extras/$REQ_VERSION/json") + if [ "$HTTP_STATUS" != "200" ]; then + echo "::error::litellm-proxy-extras $REQ_VERSION is not published on PyPI yet. Publish it before releasing litellm." + exit 1 + fi + echo "litellm-proxy-extras $REQ_VERSION exists on PyPI. Sanity check passed." + + publish-litellm: + name: Publish litellm to PyPI + needs: preflight-checks + if: needs.preflight-checks.outputs.needs_publish == 'true' + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + id-token: write + contents: read + environment: pypi-publish + + steps: + - name: Checkout repo + uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + + - name: Set up Python + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: "3.12" + + - name: Copy model prices backup + run: cp model_prices_and_context_window.json litellm/model_prices_and_context_window_backup.json + + - name: Install build tools + run: python -m pip install --upgrade pip build==1.4.2 + + - name: Build package + run: | + rm -rf build dist + python -m build + + - name: Verify build artifacts + env: + EXPECTED_VERSION: ${{ needs.preflight-checks.outputs.version }} + run: | + echo "Contents of dist/:" + ls -la dist/ + # Ensure we have both sdist and wheel + ls dist/*.tar.gz + ls dist/*.whl + # Verify built version matches expected + ls dist/ | grep -q "litellm-${EXPECTED_VERSION}" || { + echo "::error::Built artifacts do not match expected version $EXPECTED_VERSION" + ls dist/ + exit 1 + } + + - name: Validate package metadata + run: | + pip install twine==6.2.0 + twine check dist/* + + - name: Publish to PyPI + uses: pypa/gh-action-pypi-publish@ed0c53931b1dc9bd32cbe73a98c7f6766f8a527e # v1.13.0 diff --git a/.github/workflows/read_pyproject_version.yml b/.github/workflows/read_pyproject_version.yml index 8f6310f935b..04b4a38ce19 100644 --- a/.github/workflows/read_pyproject_version.yml +++ b/.github/workflows/read_pyproject_version.yml @@ -3,7 +3,10 @@ name: Read Version from pyproject.toml on: push: branches: - - main # Change this to the default branch of your repository + - main # Change this to the default branch of your repository + +permissions: + contents: read jobs: read-version: @@ -11,20 +14,14 @@ jobs: steps: - name: Checkout code - uses: actions/checkout@v2 - - - name: Set up Python - uses: actions/setup-python@v2 + uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 with: - python-version: 3.8 # Adjust the Python version as needed - - - name: Install dependencies - run: pip install toml + persist-credentials: false - name: Read version from pyproject.toml id: read-version run: | - version=$(python -c 'import toml; print(toml.load("pyproject.toml")["tool"]["commitizen"]["version"])') + version=$(grep -m1 '^version' pyproject.toml | sed 's/version = "\(.*\)"/\1/') printf "LITELLM_VERSION=%s" "$version" >> $GITHUB_ENV - name: Display version diff --git a/.github/workflows/redeploy_proxy.py b/.github/workflows/redeploy_proxy.py deleted file mode 100644 index ed46bef73a2..00000000000 --- a/.github/workflows/redeploy_proxy.py +++ /dev/null @@ -1,20 +0,0 @@ -""" - -redeploy_proxy.py -""" - -import os -import requests -import time - -# send a get request to this endpoint -deploy_hook1 = os.getenv("LOAD_TEST_REDEPLOY_URL1") -response = requests.get(deploy_hook1, timeout=20) - - -deploy_hook2 = os.getenv("LOAD_TEST_REDEPLOY_URL2") -response = requests.get(deploy_hook2, timeout=20) - -print("SENT GET REQUESTS to re-deploy proxy") -print("sleeeping.... for 60s") -time.sleep(60) diff --git a/.github/workflows/regenerate-poetry-lock.yml b/.github/workflows/regenerate-poetry-lock.yml deleted file mode 100644 index c0844f1c705..00000000000 --- a/.github/workflows/regenerate-poetry-lock.yml +++ /dev/null @@ -1,80 +0,0 @@ -name: Regenerate poetry.lock - -# Runs whenever pyproject.toml is merged into main (the most common cause of -# the "pyproject.toml changed significantly since poetry.lock was last generated" -# CI failure). Can also be triggered manually. -on: - push: - branches: - - main - paths: - - pyproject.toml - workflow_dispatch: - -permissions: - contents: write # needed to push the auto/regenerate-poetry-lock-* branch - pull-requests: write # needed to open the PR and enable auto-merge - -jobs: - regenerate-lock: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: "3.11" - - - name: Install Poetry - run: pip install poetry - - - name: Regenerate poetry.lock - run: poetry lock - - - name: Check whether poetry.lock actually changed - id: diff - run: | - if git diff --quiet poetry.lock; then - echo "changed=false" >> "$GITHUB_OUTPUT" - else - echo "changed=true" >> "$GITHUB_OUTPUT" - fi - - - name: Open PR with the refreshed lock file - if: steps.diff.outputs.changed == 'true' - id: open-pr - run: | - BRANCH="auto/regenerate-poetry-lock-$(date +'%Y%m%d%H%M%S')" - git config user.name "github-actions[bot]" - git config user.email "github-actions[bot]@users.noreply.github.com" - git checkout -b "$BRANCH" - git add poetry.lock - git commit -m "chore: regenerate poetry.lock to match pyproject.toml" - git push -f origin "$BRANCH" - - cat > /tmp/pr-body.md << 'BODY' - Automated regeneration of `poetry.lock` after `pyproject.toml` was updated on `main`. - - Fixes the recurring CI failure: - ``` - pyproject.toml changed significantly since poetry.lock was last generated. - Run `poetry lock` to fix the lock file. - ``` - BODY - - PR_URL=$(gh pr create \ - --title "chore: regenerate poetry.lock to match pyproject.toml" \ - --body-file /tmp/pr-body.md \ - --head "$BRANCH" \ - --base main) - echo "pr_url=$PR_URL" >> "$GITHUB_OUTPUT" - env: - GH_TOKEN: ${{ github.token }} - - - name: Enable auto-merge - if: steps.diff.outputs.changed == 'true' - run: | - gh pr merge "${{ steps.open-pr.outputs.pr_url }}" --auto --squash - env: - GH_TOKEN: ${{ github.token }} diff --git a/.github/workflows/reset_stable.yml b/.github/workflows/reset_stable.yml deleted file mode 100644 index f6fed672d47..00000000000 --- a/.github/workflows/reset_stable.yml +++ /dev/null @@ -1,39 +0,0 @@ -name: Reset litellm_stable branch - -on: - release: - types: [published, created] -jobs: - update-stable-branch: - if: ${{ startsWith(github.event.release.tag_name, 'v') && !endsWith(github.event.release.tag_name, '-stable') }} - runs-on: ubuntu-latest - - steps: - - name: Checkout repository - uses: actions/checkout@v3 - - - name: Reset litellm_stable_release_branch branch to the release commit - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - # Configure Git user - git config user.name "github-actions[bot]" - git config user.email "github-actions[bot]@users.noreply.github.com" - - # Fetch all branches and tags - git fetch --all - - # Check if the litellm_stable_release_branch branch exists - if git show-ref --verify --quiet refs/remotes/origin/litellm_stable_release_branch; then - echo "litellm_stable_release_branch branch exists." - git checkout litellm_stable_release_branch - else - echo "litellm_stable_release_branch branch does not exist. Creating it." - git checkout -b litellm_stable_release_branch - fi - - # Reset litellm_stable_release_branch branch to the release commit - git reset --hard $GITHUB_SHA - - # Push the updated litellm_stable_release_branch branch - git push origin litellm_stable_release_branch --force diff --git a/.github/workflows/run_observatory_tests.yml b/.github/workflows/run_observatory_tests.yml index d343098ed32..a25b96766d7 100644 --- a/.github/workflows/run_observatory_tests.yml +++ b/.github/workflows/run_observatory_tests.yml @@ -33,7 +33,9 @@ jobs: timeout-minutes: 30 steps: - name: Checkout repository - uses: actions/checkout@v4 + uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + with: + persist-credentials: false - name: Validate tag input env: @@ -49,11 +51,12 @@ jobs: TAG: ${{ inputs.tag }} AZURE_API_KEY: ${{ secrets.AZURE_API_KEY }} AZURE_API_BASE: ${{ secrets.AZURE_API_BASE }} + WORKSPACE: ${{ github.workspace }} run: | docker run -d \ --name litellm-rc \ -p 4000:4000 \ - -v "${{ github.workspace }}/.github/observatory/litellm_config.yaml:/app/config.yaml" \ + -v "${WORKSPACE}/.github/observatory/litellm_config.yaml:/app/config.yaml" \ -e LITELLM_MASTER_KEY="${LITELLM_MASTER_KEY}" \ -e AZURE_API_KEY="${AZURE_API_KEY}" \ -e AZURE_API_BASE="${AZURE_API_BASE}" \ @@ -77,8 +80,9 @@ jobs: - name: Start cloudflared tunnel run: | - # Install cloudflared + # Install cloudflared (pinned version + checksum) curl -sL https://github.com/cloudflare/cloudflared/releases/download/2025.2.1/cloudflared-linux-amd64 -o /usr/local/bin/cloudflared + echo "afdfadd1ef552e66bffc35246fe30a9bd578356d2d386de95585ccfc432472b8 /usr/local/bin/cloudflared" | sha256sum -c - chmod +x /usr/local/bin/cloudflared # Start a quick tunnel (no account needed) and capture the URL @@ -103,11 +107,11 @@ jobs: - name: Verify tunnel connectivity run: | - echo "Testing tunnel at ${{ env.TUNNEL_URL }}..." + echo "Testing tunnel at ${TUNNEL_URL}..." # Quick tunnels need time for DNS propagation; retry to avoid # transient NXDOMAIN (curl exit code 6) on first attempt. for i in $(seq 1 10); do - if curl -sf "${{ env.TUNNEL_URL }}/health/liveliness" > /dev/null 2>&1; then + if curl -sf "${TUNNEL_URL}/health/liveliness" > /dev/null 2>&1; then echo "Tunnel is working (attempt $i)" exit 0 fi @@ -221,5 +225,5 @@ jobs: - name: Cleanup if: always() run: | - kill "${{ env.CLOUDFLARED_PID }}" 2>/dev/null || true + kill "$CLOUDFLARED_PID" 2>/dev/null || true docker rm -f litellm-rc 2>/dev/null || true diff --git a/.github/workflows/scan_duplicate_issues.yml b/.github/workflows/scan_duplicate_issues.yml index 06e8f453a8c..222ff11f304 100644 --- a/.github/workflows/scan_duplicate_issues.yml +++ b/.github/workflows/scan_duplicate_issues.yml @@ -21,14 +21,15 @@ jobs: contents: read steps: - name: Checkout scripts - uses: actions/checkout@v4 + uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 with: sparse-checkout: .github/scripts + persist-credentials: false - name: Set up Python - uses: actions/setup-python@v5 + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: - python-version: "3.11" + python-version: "3.13" - name: Scan for duplicate issues env: diff --git a/.github/workflows/scorecard.yml b/.github/workflows/scorecard.yml new file mode 100644 index 00000000000..7cd12bb219c --- /dev/null +++ b/.github/workflows/scorecard.yml @@ -0,0 +1,47 @@ +name: Scorecard supply-chain security + +on: + branch_protection_rule: + schedule: + - cron: '27 12 * * 4' + push: + branches: ["main"] + +permissions: read-all + +jobs: + analysis: + name: Scorecard analysis + runs-on: ubuntu-latest + if: github.event.repository.default_branch == github.ref_name + permissions: + security-events: write + id-token: write + # Uncomment for private repos if needed: + # contents: read + # actions: read + + steps: + - name: Checkout code + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + persist-credentials: false + + - name: Run analysis + uses: ossf/scorecard-action@f49aabe0b5af0936a0987cfb85d86b75731b0186 # v2.4.1 + with: + results_file: results.sarif + results_format: sarif + publish_results: true + + - name: Upload artifact + uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1 + with: + name: SARIF file + path: results.sarif + retention-days: 5 + + - name: Upload to code scanning + uses: github/codeql-action/upload-sarif@c10b806170c8ee63ea24152429041b5624f0baf5 # v4.35.1 + with: + sarif_file: results.sarif diff --git a/.github/workflows/simple_pypi_publish.yml b/.github/workflows/simple_pypi_publish.yml deleted file mode 100644 index e1830556819..00000000000 --- a/.github/workflows/simple_pypi_publish.yml +++ /dev/null @@ -1,67 +0,0 @@ -name: Simple PyPI Publish - -on: - workflow_dispatch: - inputs: - version: - description: 'Version to publish (e.g., 1.74.10)' - required: true - type: string - -env: - TWINE_USERNAME: __token__ - -jobs: - publish: - runs-on: ubuntu-latest - if: github.repository == 'BerriAI/litellm' - - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: '3.8' - - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install toml build wheel twine - - - name: Update version in pyproject.toml - run: | - python -c " - import toml - - with open('pyproject.toml', 'r') as f: - data = toml.load(f) - - data['tool']['poetry']['version'] = '${{ github.event.inputs.version }}' - - with open('pyproject.toml', 'w') as f: - toml.dump(data, f) - - print(f'Updated version to ${{ github.event.inputs.version }}') - " - - - name: Copy model prices file - run: | - cp model_prices_and_context_window.json litellm/model_prices_and_context_window_backup.json - - - name: Build package - run: | - rm -rf build dist - python -m build - - - name: Publish to PyPI - env: - TWINE_PASSWORD: ${{ secrets.PYPI_PUBLISH_PASSWORD }} - run: | - twine upload dist/* - - - name: Output success - run: | - echo "✅ Successfully published litellm v${{ github.event.inputs.version }} to PyPI" - echo "📦 Package: https://pypi.org/project/litellm/${{ github.event.inputs.version }}/" \ No newline at end of file diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index 5a9b19fc9ca..c905bb12312 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -2,19 +2,24 @@ name: "Stale Issue Management" on: schedule: - - cron: '0 0 * * *' # Runs daily at midnight UTC + - cron: "0 0 * * *" # Runs daily at midnight UTC workflow_dispatch: +permissions: + issues: write + pull-requests: write + jobs: stale: + if: github.repository == 'BerriAI/litellm' runs-on: ubuntu-latest steps: - - uses: actions/stale@v8 + - uses: actions/stale@1160a2240286f5da8ec72b1c0816ce2481aabf84 # v8 with: repo-token: "${{ secrets.GITHUB_TOKEN }}" stale-issue-message: "This issue has been automatically marked as stale because it has not had recent activity. It will be closed if no further activity occurs." stale-pr-message: "This pull request has been automatically marked as stale because it has not had recent activity. It will be closed if no further activity occurs." - days-before-stale: 90 # Revert to 60 days - days-before-close: 7 # Revert to 7 days + days-before-stale: 90 # Revert to 60 days + days-before-close: 7 # Revert to 7 days stale-issue-label: "stale" - operations-per-run: 1000 \ No newline at end of file + operations-per-run: 1000 diff --git a/.github/workflows/sync-schema.yml b/.github/workflows/sync-schema.yml new file mode 100644 index 00000000000..72a5c56293e --- /dev/null +++ b/.github/workflows/sync-schema.yml @@ -0,0 +1,73 @@ +name: Sync schema.prisma copies + +on: + pull_request: + paths: + - 'schema.prisma' + +# Scoped to ONLY the permissions needed: +# - contents:write to push the sync commit to the PR branch +# - pull-requests:read is implicit (needed to check out the PR) +permissions: + contents: write + +jobs: + sync: + name: Copy root schema to proxy and proxy-extras + runs-on: ubuntu-latest + timeout-minutes: 5 + # Only run on PRs from branches in THIS repo (not forks). + # Fork PRs cannot push back to the head branch with GITHUB_TOKEN, + # and pull_request events from forks have read-only tokens anyway. + # Also reject PRs from branches named after protected branches to + # prevent pushing directly to main/master. + if: >- + github.event.pull_request.head.repo.full_name == github.repository + && github.head_ref != 'main' + && github.head_ref != 'master' + steps: + - name: Checkout PR branch by SHA + uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + with: + # Use the merge commit SHA for safety — github.head_ref is an + # attacker-controlled string (the branch name) and could contain + # unusual characters that cause unexpected git behavior. + ref: ${{ github.event.pull_request.head.sha }} + persist-credentials: true # needed for git push + + - name: Reject symlinked schema files + run: | + for f in schema.prisma litellm/proxy/schema.prisma litellm-proxy-extras/litellm_proxy_extras/schema.prisma; do + if [ -L "$f" ]; then + echo "::error file=$f::$f is a symlink, which is not allowed" + exit 1 + fi + done + + - name: Copy root schema to other locations + run: | + cp schema.prisma litellm/proxy/schema.prisma + cp schema.prisma litellm-proxy-extras/litellm_proxy_extras/schema.prisma + + - name: Check for changes + id: diff + run: | + if git diff --quiet -- litellm/proxy/schema.prisma litellm-proxy-extras/litellm_proxy_extras/schema.prisma; then + echo "changed=false" >> "$GITHUB_OUTPUT" + echo "Schemas already in sync. Nothing to do." + else + echo "changed=true" >> "$GITHUB_OUTPUT" + echo "Schema copies need updating." + fi + + - name: Commit synced schemas + if: steps.diff.outputs.changed == 'true' + run: | + # Push to the PR's head branch (need the branch name for git push). + # We checked out by SHA above for safety, so configure the push target explicitly. + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git checkout -B "$GITHUB_HEAD_REF" + git add -- litellm/proxy/schema.prisma litellm-proxy-extras/litellm_proxy_extras/schema.prisma + git commit -m "chore: sync schema.prisma copies from root" + git push origin "HEAD:$GITHUB_HEAD_REF" diff --git a/.github/workflows/test-linting.yml b/.github/workflows/test-linting.yml index fc0f84a20d4..5bb85716a17 100644 --- a/.github/workflows/test-linting.yml +++ b/.github/workflows/test-linting.yml @@ -2,7 +2,10 @@ name: LiteLLM Linting on: pull_request: - branches: [ main ] + branches: [main] + +permissions: + contents: read jobs: lint: @@ -10,69 +13,73 @@ jobs: timeout-minutes: 5 steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - clean: true + - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + with: + fetch-depth: 0 + clean: true + persist-credentials: false - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: '3.12' + - name: Set up Python + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: "3.12" - - name: Install Poetry - uses: snok/install-poetry@v1 + - name: Install Poetry + run: pip install 'poetry==2.3.2' - - name: Clean Python cache - run: | - find . -type d -name "__pycache__" -exec rm -rf {} + || true - find . -name "*.pyc" -delete || true + - name: Clean Python cache + run: | + find . -type d -name "__pycache__" -exec rm -rf {} + || true + find . -name "*.pyc" -delete || true - - name: Install dependencies - run: | - poetry lock - poetry install --with dev + - name: Check poetry.lock is up to date + run: | + poetry check --lock || (echo "❌ poetry.lock is out of sync with pyproject.toml. Run 'poetry lock' locally and commit the result." && exit 1) - - name: Check Black formatting - run: | - cd litellm - poetry run black --check --exclude '/enterprise/' . - cd .. + - name: Install dependencies + run: | + poetry install --with dev - - name: Debug - Check file state - run: | - echo "Current branch:" - git branch --show-current - echo "Last 3 commits:" - git log --oneline -3 - echo "File content around line 43:" - head -50 litellm/litellm_core_utils/custom_logger_registry.py | tail -10 - - - name: Run Ruff linting - run: | - cd litellm - poetry run ruff check . - cd .. + - name: Check Black formatting + run: | + cd litellm + poetry run black --check --exclude '/enterprise/' . + cd .. - - name: Print OpenAI version - run: | - poetry run python -c "import openai; print(f'OpenAI version: {openai.__version__}')" + - name: Debug - Check file state + run: | + echo "Current branch:" + git branch --show-current + echo "Last 3 commits:" + git log --oneline -3 + echo "File content around line 43:" + head -50 litellm/litellm_core_utils/custom_logger_registry.py | tail -10 - - name: Run MyPy type checking - run: | - cd litellm - poetry run mypy . - cd .. + - name: Run Ruff linting + run: | + cd litellm + poetry run ruff check . + cd .. - - name: Check for circular imports - run: | - cd litellm - poetry run python ../tests/documentation_tests/test_circular_imports.py - cd .. + - name: Print OpenAI version + run: | + poetry run python -c "import openai; print(f'OpenAI version: {openai.__version__}')" - - name: Check import safety - run: | - poetry run python -c "from litellm import *" || (echo '🚨 import failed, this means you introduced unprotected imports! 🚨'; exit 1) + - name: Run MyPy type checking + run: | + cd litellm + poetry run mypy . + cd .. + + - name: Check for circular imports + run: | + cd litellm + poetry run python ../tests/documentation_tests/test_circular_imports.py + cd .. + + - name: Check import safety + run: | + poetry run python -c "from litellm import *" || (echo '🚨 import failed, this means you introduced unprotected imports! 🚨'; exit 1) secret-scan: runs-on: ubuntu-latest @@ -81,27 +88,28 @@ jobs: contents: read steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 + - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + with: + fetch-depth: 0 + persist-credentials: false - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: '3.12' + - name: Set up Python + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: "3.12" - - name: Run secret scan test - run: | - pip install pytest - pytest tests/litellm/test_no_hardcoded_secrets.py -v + - name: Run secret scan test + run: | + pip install 'pytest==9.0.2' + pytest tests/litellm/test_no_hardcoded_secrets.py -v - - name: Run ggshield secret scan - env: - GITGUARDIAN_API_KEY: ${{ secrets.GITGUARDIAN_API_KEY }} - run: | - if [ -n "$GITGUARDIAN_API_KEY" ]; then - pip install ggshield - ggshield secret scan repo . - else - echo "GITGUARDIAN_API_KEY not set, skipping ggshield scan" - fi + - name: Run ggshield secret scan + env: + GITGUARDIAN_API_KEY: ${{ secrets.GITGUARDIAN_API_KEY }} + run: | + if [ -n "$GITGUARDIAN_API_KEY" ]; then + pip install 'ggshield==1.48.0' + ggshield secret scan repo . + else + echo "GITGUARDIAN_API_KEY not set, skipping ggshield scan" + fi diff --git a/.github/workflows/test-litellm-matrix.yml b/.github/workflows/test-litellm-matrix.yml index d0ac28ab41a..860d25636c5 100644 --- a/.github/workflows/test-litellm-matrix.yml +++ b/.github/workflows/test-litellm-matrix.yml @@ -4,6 +4,9 @@ on: pull_request: branches: [main] +permissions: + contents: read + # Cancel in-progress runs for the same PR concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} @@ -12,7 +15,7 @@ concurrency: jobs: test: runs-on: ubuntu-latest - timeout-minutes: 20 # Increased from 15 to 20 + timeout-minutes: 20 # Increased from 15 to 20 strategy: fail-fast: false matrix: @@ -43,7 +46,7 @@ jobs: - name: "integrations" path: "tests/test_litellm/integrations" workers: 2 - reruns: 3 # Integration tests tend to be flakier + reruns: 3 # Integration tests tend to be flakier - name: "core-utils" path: "tests/test_litellm/litellm_core_utils" workers: 2 @@ -117,18 +120,20 @@ jobs: name: test (${{ matrix.test-group.name }}) steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + with: + persist-credentials: false - name: Set up Python - uses: actions/setup-python@v5 + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version: "3.12" - name: Install Poetry - uses: snok/install-poetry@v1 + run: pip install 'poetry==2.3.2' - name: Cache Poetry dependencies - uses: actions/cache@v4 + uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.0.0 with: path: | ~/.cache/pypoetry @@ -144,14 +149,17 @@ jobs: poetry install --with dev,proxy-dev --extras "proxy semantic-router" # pytest-rerunfailures and pytest-xdist are in pyproject.toml dev dependencies poetry run pip install google-genai==1.22.0 \ - google-cloud-aiplatform>=1.38 fastapi-offline==1.7.3 python-multipart==0.0.22 openapi-core + google-cloud-aiplatform==1.115.0 fastapi-offline==1.7.3 python-multipart==0.0.22 openapi-core==0.23.0 - name: Setup litellm-enterprise run: | poetry run pip install --force-reinstall --no-deps -e enterprise/ - name: Generate Prisma client + env: + PRISMA_BINARY_CACHE_DIR: ${{ runner.temp }}/prisma-cache run: | + poetry run pip install nodejs-wheel-binaries==24.13.1 poetry run prisma generate --schema litellm/proxy/schema.prisma - name: Run tests - ${{ matrix.test-group.name }} diff --git a/.github/workflows/test-litellm-ui-build.yml b/.github/workflows/test-litellm-ui-build.yml index b0a8b648a44..6b0b3a413a6 100644 --- a/.github/workflows/test-litellm-ui-build.yml +++ b/.github/workflows/test-litellm-ui-build.yml @@ -16,10 +16,12 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v4 + uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + with: + persist-credentials: false - name: Setup Node.js - uses: actions/setup-node@v4 + uses: actions/setup-node@a0853c24544627f65ddf259abe73b1d18a591444 # v5.0 with: node-version: "20" cache: "npm" diff --git a/.github/workflows/test-litellm.yml b/.github/workflows/test-litellm.yml index 3f8369df926..0c040b3ebe7 100644 --- a/.github/workflows/test-litellm.yml +++ b/.github/workflows/test-litellm.yml @@ -4,45 +4,50 @@ name: LiteLLM Mock Tests (folder - tests/test_litellm) # the same tests in parallel across 10 jobs for faster CI times. # Kept for manual debugging only. on: - workflow_dispatch: # Manual trigger only + workflow_dispatch: # Manual trigger only # pull_request: # branches: [ main ] +permissions: + contents: read + jobs: test: runs-on: ubuntu-latest timeout-minutes: 25 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + with: + persist-credentials: false - - name: Thank You Message - run: | - echo "### 🙏 Thank you for contributing to LiteLLM!" >> $GITHUB_STEP_SUMMARY - echo "Your PR is being tested now. We appreciate your help in making LiteLLM better!" >> $GITHUB_STEP_SUMMARY + - name: Thank You Message + run: | + echo "### 🙏 Thank you for contributing to LiteLLM!" >> $GITHUB_STEP_SUMMARY + echo "Your PR is being tested now. We appreciate your help in making LiteLLM better!" >> $GITHUB_STEP_SUMMARY - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: '3.12' + - name: Set up Python + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: "3.12" - - name: Install Poetry - uses: snok/install-poetry@v1 + - name: Install Poetry + run: pip install 'poetry==2.3.2' - - name: Install dependencies - run: | - poetry lock - poetry install --with dev,proxy-dev --extras "proxy semantic-router" - poetry run pip install "pytest-retry==1.6.3" - poetry run pip install pytest-xdist - poetry run pip install "google-genai==1.22.0" - poetry run pip install "google-cloud-aiplatform>=1.38" - poetry run pip install "fastapi-offline==1.7.3" - poetry run pip install "python-multipart>=0.0.20" - poetry run pip install "openapi-core" - - name: Setup litellm-enterprise as local package - run: | - poetry run pip install --force-reinstall --no-deps -e enterprise/ - - name: Run tests - run: | - poetry run pytest tests/test_litellm --tb=short -vv --maxfail=10 -n 4 --durations=50 + - name: Install dependencies + run: | + poetry lock + poetry install --with dev,proxy-dev --extras "proxy semantic-router" + poetry run pip install "pytest-retry==1.6.3" + poetry run pip install 'pytest-xdist==3.8.0' + poetry run pip install "google-genai==1.22.0" + poetry run pip install "google-cloud-aiplatform==1.115.0" + poetry run pip install "fastapi-offline==1.7.3" + poetry run pip install "python-multipart==0.0.22" + poetry run pip install "openapi-core==0.23.0" + - name: Setup litellm-enterprise as local package + run: | + poetry run pip install --force-reinstall --no-deps -e enterprise/ + - name: Run tests + run: | + poetry run pytest tests/test_litellm --tb=short -vv --maxfail=10 -n 4 --durations=50 diff --git a/.github/workflows/test-mcp.yml b/.github/workflows/test-mcp.yml index 2e32aae7680..1b228ab76bb 100644 --- a/.github/workflows/test-mcp.yml +++ b/.github/workflows/test-mcp.yml @@ -2,7 +2,10 @@ name: LiteLLM MCP Tests (folder - tests/mcp_tests) on: pull_request: - branches: [ main ] + branches: [main] + +permissions: + contents: read jobs: test: @@ -10,38 +13,40 @@ jobs: timeout-minutes: 25 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + with: + persist-credentials: false - - name: Thank You Message - run: | - echo "### 🙏 Thank you for contributing to LiteLLM!" >> $GITHUB_STEP_SUMMARY - echo "Your PR is being tested now. We appreciate your help in making LiteLLM better!" >> $GITHUB_STEP_SUMMARY + - name: Thank You Message + run: | + echo "### 🙏 Thank you for contributing to LiteLLM!" >> $GITHUB_STEP_SUMMARY + echo "Your PR is being tested now. We appreciate your help in making LiteLLM better!" >> $GITHUB_STEP_SUMMARY - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: '3.12' + - name: Set up Python + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: "3.12" - - name: Install Poetry - uses: snok/install-poetry@v1 + - name: Install Poetry + run: pip install 'poetry==2.3.2' - - name: Install dependencies - run: | - poetry lock - poetry install --with dev,proxy-dev --extras "proxy semantic-router" - poetry run pip install "pytest==7.3.1" - poetry run pip install "pytest-retry==1.6.3" - poetry run pip install "pytest-cov==5.0.0" - poetry run pip install "pytest-asyncio==0.21.1" - poetry run pip install "respx==0.22.0" - poetry run pip install "pydantic==2.11.0" - poetry run pip install "mcp==1.25.0" - poetry run pip install pytest-xdist + - name: Install dependencies + run: | + poetry lock + poetry install --with dev,proxy-dev --extras "proxy semantic-router" + poetry run pip install "pytest==7.3.1" + poetry run pip install "pytest-retry==1.6.3" + poetry run pip install "pytest-cov==5.0.0" + poetry run pip install "pytest-asyncio==0.21.1" + poetry run pip install "respx==0.22.0" + poetry run pip install "pydantic==2.11.0" + poetry run pip install "mcp==1.25.0" + poetry run pip install 'pytest-xdist==3.8.0' - - name: Setup litellm-enterprise as local package - run: | - poetry run pip install --force-reinstall --no-deps -e enterprise/ + - name: Setup litellm-enterprise as local package + run: | + poetry run pip install --force-reinstall --no-deps -e enterprise/ - - name: Run MCP tests - run: | - poetry run pytest tests/mcp_tests -x -vv -n 4 --cov=litellm --cov-report=xml --durations=5 + - name: Run MCP tests + run: | + poetry run pytest tests/mcp_tests -x -vv -n 4 --cov=litellm --cov-report=xml --durations=5 diff --git a/.github/workflows/test-model-map.yaml b/.github/workflows/test-model-map.yaml index ae5ac402e23..429f9e1ce0a 100644 --- a/.github/workflows/test-model-map.yaml +++ b/.github/workflows/test-model-map.yaml @@ -2,13 +2,18 @@ name: Validate model_prices_and_context_window.json on: pull_request: - branches: [ main ] + branches: [main] + +permissions: + contents: read jobs: validate-model-prices-json: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + with: + persist-credentials: false - name: Validate model_prices_and_context_window.json run: | diff --git a/.github/workflows/test-proxy-e2e-azure-batches.yml b/.github/workflows/test-proxy-e2e-azure-batches.yml index 4d74f3db0ac..7cbbe0b338f 100644 --- a/.github/workflows/test-proxy-e2e-azure-batches.yml +++ b/.github/workflows/test-proxy-e2e-azure-batches.yml @@ -9,6 +9,9 @@ concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} cancel-in-progress: true +permissions: + contents: read + jobs: proxy_e2e_azure_batches_tests: runs-on: ubuntu-latest @@ -30,18 +33,20 @@ jobs: --health-retries 5 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + with: + persist-credentials: false - name: Set up Python - uses: actions/setup-python@v5 + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version: "3.12" - name: Install Poetry - uses: snok/install-poetry@v1 + run: pip install 'poetry==2.3.2' - name: Cache Poetry dependencies - uses: actions/cache@v4 + uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.0.0 with: path: | ~/.cache/pypoetry @@ -56,14 +61,17 @@ jobs: run: | poetry config virtualenvs.in-project true poetry install --with dev,proxy-dev --extras "proxy" - poetry run pip install psycopg2-binary uvicorn fastapi httpx tenacity + poetry run pip install psycopg2-binary==2.9.11 uvicorn==0.42.0 fastapi==0.135.2 httpx==0.28.1 tenacity==9.1.4 - name: Setup litellm-enterprise run: | poetry run pip install --force-reinstall --no-deps -e enterprise/ - name: Generate Prisma client + env: + PRISMA_BINARY_CACHE_DIR: ${{ runner.temp }}/prisma-cache run: | + poetry run pip install nodejs-wheel-binaries==24.13.1 poetry run prisma generate --schema litellm/proxy/schema.prisma - name: Run Prisma migrations @@ -87,4 +95,3 @@ jobs: --tb=short \ --maxfail=3 \ --durations=10 - diff --git a/.github/workflows/test-unit-caching-redis.yml b/.github/workflows/test-unit-caching-redis.yml new file mode 100644 index 00000000000..ca274324f2f --- /dev/null +++ b/.github/workflows/test-unit-caching-redis.yml @@ -0,0 +1,38 @@ +name: "Unit Tests: Caching (Redis)" + +# Uses cloud Redis credentials — only runs on trusted branches, not PRs. +# This prevents external PRs from accessing Redis credentials. +on: + push: + branches: [main, "litellm_*"] + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + caching-redis: + uses: ./.github/workflows/_test-unit-services-base.yml + with: + # Redis-only tests that do NOT require provider API keys. + # Tests needing API keys (test_caching.py, test_caching_ssl.py, test_prometheus_service.py, + # test_router_caching.py) are in Phase 3 integration workflows. + test-path: >- + tests/local_testing/test_dual_cache.py + tests/local_testing/test_redis_batch_optimizations.py + tests/local_testing/test_router_utils.py + workers: 2 + reruns: 2 + timeout-minutes: 20 + enable-redis: true + enable-postgres: false + secrets: + REDIS_HOST: ${{ secrets.REDIS_HOST }} + REDIS_PORT: ${{ secrets.REDIS_PORT }} + REDIS_PASSWORD: ${{ secrets.REDIS_PASSWORD }} + DATABASE_URL: ${{ secrets.DATABASE_URL }} + POSTGRES_USER: ${{ secrets.POSTGRES_USER }} + POSTGRES_PASSWORD: ${{ secrets.POSTGRES_PASSWORD }} diff --git a/.github/workflows/test-unit-core-utils.yml b/.github/workflows/test-unit-core-utils.yml new file mode 100644 index 00000000000..2f3698fdf60 --- /dev/null +++ b/.github/workflows/test-unit-core-utils.yml @@ -0,0 +1,20 @@ +name: "Unit Tests: Core Utilities" + +on: + pull_request: + branches: [main] + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + core-utils: + uses: ./.github/workflows/_test-unit-base.yml + with: + test-path: "tests/test_litellm/litellm_core_utils" + workers: 2 + reruns: 1 diff --git a/.github/workflows/test-unit-documentation.yml b/.github/workflows/test-unit-documentation.yml new file mode 100644 index 00000000000..d8b30de6844 --- /dev/null +++ b/.github/workflows/test-unit-documentation.yml @@ -0,0 +1,67 @@ +name: "Unit Tests: Documentation Validation" + +on: + pull_request: + branches: [main] + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + documentation: + runs-on: ubuntu-latest + timeout-minutes: 10 + + steps: + - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + with: + persist-credentials: false + + - name: Set up Python + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: "3.12" + + - name: Install Poetry + run: pip install 'poetry==2.3.2' + + - name: Cache Poetry dependencies + uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 + with: + path: | + ~/.cache/pypoetry + ~/.cache/pip + .venv + key: ${{ runner.os }}-poetry-${{ hashFiles('poetry.lock') }} + restore-keys: | + ${{ runner.os }}-poetry- + + - name: Install dependencies + run: | + poetry config virtualenvs.in-project true + poetry install --with dev,proxy-dev --extras "proxy semantic-router" + poetry run pip install google-genai==1.22.0 \ + google-cloud-aiplatform==1.115.0 fastapi-offline==1.7.3 python-multipart==0.0.22 openapi-core==0.23.0 + + - name: Setup litellm-enterprise + run: | + poetry run pip install --force-reinstall --no-deps -e enterprise/ + + - name: Generate Prisma client + env: + PRISMA_BINARY_CACHE_DIR: ${{ runner.temp }}/prisma-cache + run: | + poetry run pip install nodejs-wheel-binaries==24.13.1 + poetry run prisma generate --schema litellm/proxy/schema.prisma + + # Run the same documentation tests that CircleCI ran (as direct Python scripts) + - name: Run documentation validation tests + run: | + poetry run python ./tests/documentation_tests/test_env_keys.py + poetry run python ./tests/documentation_tests/test_router_settings.py + poetry run python ./tests/documentation_tests/test_api_docs.py + poetry run python ./tests/documentation_tests/test_circular_imports.py diff --git a/.github/workflows/test-unit-enterprise-routing.yml b/.github/workflows/test-unit-enterprise-routing.yml new file mode 100644 index 00000000000..13ae3efedba --- /dev/null +++ b/.github/workflows/test-unit-enterprise-routing.yml @@ -0,0 +1,24 @@ +name: "Unit Tests: Enterprise, Google GenAI & Routing" + +on: + pull_request: + branches: [main] + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + enterprise-routing: + uses: ./.github/workflows/_test-unit-base.yml + with: + test-path: >- + tests/test_litellm/enterprise + tests/test_litellm/google_genai + tests/test_litellm/router_utils + tests/test_litellm/router_strategy + workers: 2 + reruns: 2 diff --git a/.github/workflows/test-unit-integrations.yml b/.github/workflows/test-unit-integrations.yml new file mode 100644 index 00000000000..2789f99d81c --- /dev/null +++ b/.github/workflows/test-unit-integrations.yml @@ -0,0 +1,20 @@ +name: "Unit Tests: Integrations (Callbacks & Logging)" + +on: + pull_request: + branches: [main] + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + integrations: + uses: ./.github/workflows/_test-unit-base.yml + with: + test-path: "tests/test_litellm/integrations" + workers: 2 + reruns: 3 diff --git a/.github/workflows/test-unit-llm-providers.yml b/.github/workflows/test-unit-llm-providers.yml new file mode 100644 index 00000000000..6c00272b0c8 --- /dev/null +++ b/.github/workflows/test-unit-llm-providers.yml @@ -0,0 +1,29 @@ +name: "Unit Tests: LLM Provider Transformations" + +on: + pull_request: + branches: [main] + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + vertex-ai: + name: Vertex AI + uses: ./.github/workflows/_test-unit-base.yml + with: + test-path: "tests/test_litellm/llms/vertex_ai" + workers: 1 + reruns: 2 + + other-providers: + name: All Other Providers + uses: ./.github/workflows/_test-unit-base.yml + with: + test-path: "tests/test_litellm/llms --ignore=tests/test_litellm/llms/vertex_ai" + workers: 2 + reruns: 2 diff --git a/.github/workflows/test-unit-misc.yml b/.github/workflows/test-unit-misc.yml new file mode 100644 index 00000000000..9228decd7cc --- /dev/null +++ b/.github/workflows/test-unit-misc.yml @@ -0,0 +1,31 @@ +name: "Unit Tests: MCP, Secrets, Containers & Misc" + +on: + pull_request: + branches: [main] + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + misc: + uses: ./.github/workflows/_test-unit-base.yml + with: + test-path: >- + tests/test_litellm/secret_managers + tests/test_litellm/a2a_protocol + tests/test_litellm/anthropic_interface + tests/test_litellm/completion_extras + tests/test_litellm/containers + tests/test_litellm/experimental_mcp_client + tests/test_litellm/images + tests/test_litellm/interactions + tests/test_litellm/passthrough + tests/test_litellm/vector_stores + tests/test_litellm/test_*.py + workers: 2 + reruns: 2 diff --git a/.github/workflows/test-unit-proxy-auth.yml b/.github/workflows/test-unit-proxy-auth.yml new file mode 100644 index 00000000000..e71821db701 --- /dev/null +++ b/.github/workflows/test-unit-proxy-auth.yml @@ -0,0 +1,20 @@ +name: "Unit Tests: Proxy Auth & Key Management" + +on: + pull_request: + branches: [main] + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + proxy-auth: + uses: ./.github/workflows/_test-unit-base.yml + with: + test-path: "tests/test_litellm/proxy/auth tests/test_litellm/proxy/hooks tests/test_litellm/proxy/policy_engine tests/test_litellm/proxy/client" + workers: 2 + reruns: 2 diff --git a/.github/workflows/test-unit-proxy-db.yml b/.github/workflows/test-unit-proxy-db.yml new file mode 100644 index 00000000000..bdfb6efeef1 --- /dev/null +++ b/.github/workflows/test-unit-proxy-db.yml @@ -0,0 +1,45 @@ +name: "Unit Tests: Proxy DB Operations" + +# Uses DATABASE_URL secret — only runs on trusted branches, not PRs. +on: + push: + branches: [main, "litellm_*"] + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + proxy-db: + strategy: + fail-fast: false + matrix: + include: + # Key generation tests must NOT run in parallel (event loop conflicts with logging worker) + - test-group: key-generation + test-path: "tests/proxy_unit_tests/test_key_generate_prisma.py" + workers: 0 + timeout: 30 + - test-group: auth-checks + test-path: "tests/proxy_unit_tests/test_auth_checks.py tests/proxy_unit_tests/test_user_api_key_auth.py" + workers: 8 + timeout: 20 + - test-group: remaining + test-path: "tests/proxy_unit_tests --ignore=tests/proxy_unit_tests/test_key_generate_prisma.py --ignore=tests/proxy_unit_tests/test_auth_checks.py --ignore=tests/proxy_unit_tests/test_user_api_key_auth.py" + workers: 8 + timeout: 20 + uses: ./.github/workflows/_test-unit-services-base.yml + with: + test-path: ${{ matrix.test-path }} + workers: ${{ matrix.workers }} + reruns: 2 + timeout-minutes: ${{ matrix.timeout }} + enable-redis: false + enable-postgres: true + secrets: + DATABASE_URL: ${{ secrets.DATABASE_URL }} + POSTGRES_USER: ${{ secrets.POSTGRES_USER }} + POSTGRES_PASSWORD: ${{ secrets.POSTGRES_PASSWORD }} diff --git a/.github/workflows/test-unit-proxy-endpoints.yml b/.github/workflows/test-unit-proxy-endpoints.yml new file mode 100644 index 00000000000..caff3b3ae06 --- /dev/null +++ b/.github/workflows/test-unit-proxy-endpoints.yml @@ -0,0 +1,35 @@ +name: "Unit Tests: Proxy API Endpoints" + +on: + pull_request: + branches: [main] + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + proxy-endpoints: + uses: ./.github/workflows/_test-unit-base.yml + with: + test-path: >- + tests/test_litellm/proxy/management_endpoints + tests/test_litellm/proxy/guardrails + tests/test_litellm/proxy/management_helpers + tests/test_litellm/proxy/anthropic_endpoints + tests/test_litellm/proxy/google_endpoints + tests/test_litellm/proxy/openai_files_endpoint + tests/test_litellm/proxy/response_api_endpoints + tests/test_litellm/proxy/image_endpoints + tests/test_litellm/proxy/vector_store_endpoints + tests/test_litellm/proxy/agent_endpoints + tests/test_litellm/proxy/discovery_endpoints + tests/test_litellm/proxy/health_endpoints + tests/test_litellm/proxy/public_endpoints + tests/test_litellm/proxy/prompts + tests/test_litellm/proxy/ui_crud_endpoints + workers: 2 + reruns: 2 diff --git a/.github/workflows/test-unit-proxy-infra.yml b/.github/workflows/test-unit-proxy-infra.yml new file mode 100644 index 00000000000..4dfbbe317ed --- /dev/null +++ b/.github/workflows/test-unit-proxy-infra.yml @@ -0,0 +1,28 @@ +name: "Unit Tests: Proxy Infrastructure" + +on: + pull_request: + branches: [main] + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + proxy-infra: + uses: ./.github/workflows/_test-unit-base.yml + with: + test-path: >- + tests/test_litellm/proxy/db + tests/test_litellm/proxy/middleware + tests/test_litellm/proxy/spend_tracking + tests/test_litellm/proxy/pass_through_endpoints + tests/test_litellm/proxy/_experimental + tests/test_litellm/proxy/experimental + tests/test_litellm/proxy/common_utils + tests/test_litellm/proxy/test_*.py + workers: 2 + reruns: 2 diff --git a/.github/workflows/test-unit-proxy-legacy.yml b/.github/workflows/test-unit-proxy-legacy.yml new file mode 100644 index 00000000000..a9391137263 --- /dev/null +++ b/.github/workflows/test-unit-proxy-legacy.yml @@ -0,0 +1,96 @@ +name: "Unit Tests: Proxy Legacy Tests" + +on: + pull_request: + branches: [main] + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + test: + runs-on: ubuntu-latest + timeout-minutes: 20 + strategy: + fail-fast: false + matrix: + test-group: + - name: "auth-and-jwt" + path: "tests/proxy_unit_tests/test_[a-j]*.py" + - name: "key-generation" + path: "tests/proxy_unit_tests/test_[k-o]*.py" + - name: "proxy-config" + path: "tests/proxy_unit_tests/test_prisma*.py tests/proxy_unit_tests/test_project*.py tests/proxy_unit_tests/test_prompt*.py tests/proxy_unit_tests/test_proxy_[c-r]*.py" + - name: "proxy-server" + path: "tests/proxy_unit_tests/test_proxy_server.py" + - name: "proxy-server-extras" + path: "tests/proxy_unit_tests/test_proxy_server_*.py tests/proxy_unit_tests/test_proxy_setting_guardrails.py" + - name: "proxy-utils" + path: "tests/proxy_unit_tests/test_proxy_utils.py" + - name: "proxy-token-counter" + path: "tests/proxy_unit_tests/test_proxy_token_counter.py" + - name: "proxy-response-and-misc" + path: "tests/proxy_unit_tests/test_[r-t]*.py" + - name: "proxy-user-auth-and-spend" + path: "tests/proxy_unit_tests/test_[u-z]*.py" + + name: ${{ matrix.test-group.name }} + + steps: + - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + with: + persist-credentials: false + + - name: Set up Python + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: "3.12" + + - name: Install Poetry + run: pip install 'poetry==2.3.2' + + - name: Cache Poetry dependencies + uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 + with: + path: | + ~/.cache/pypoetry + ~/.cache/pip + .venv + key: ${{ runner.os }}-poetry-${{ hashFiles('poetry.lock') }} + restore-keys: | + ${{ runner.os }}-poetry- + + - name: Install dependencies + run: | + poetry config virtualenvs.in-project true + poetry install --with dev,proxy-dev --extras "proxy semantic-router" + poetry run pip install google-genai==1.22.0 \ + google-cloud-aiplatform==1.115.0 fastapi-offline==1.7.3 python-multipart==0.0.22 openapi-core==0.23.0 + + - name: Setup litellm-enterprise + run: | + poetry run pip install --force-reinstall --no-deps -e enterprise/ + + - name: Generate Prisma client + env: + PRISMA_BINARY_CACHE_DIR: ${{ runner.temp }}/prisma-cache + run: | + poetry run pip install nodejs-wheel-binaries==24.13.1 + poetry run prisma generate --schema litellm/proxy/schema.prisma + + - name: Run tests - ${{ matrix.test-group.name }} + env: + TEST_PATH: ${{ matrix.test-group.path }} + run: | + poetry run pytest ${TEST_PATH} \ + --tb=short -vv \ + --maxfail=10 \ + -n 2 \ + --reruns 1 \ + --reruns-delay 1 \ + --dist=loadscope \ + --durations=20 diff --git a/.github/workflows/test-unit-responses-caching-types.yml b/.github/workflows/test-unit-responses-caching-types.yml new file mode 100644 index 00000000000..7f3acac2803 --- /dev/null +++ b/.github/workflows/test-unit-responses-caching-types.yml @@ -0,0 +1,20 @@ +name: "Unit Tests: Responses, Caching & Types" + +on: + pull_request: + branches: [main] + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + responses-caching-types: + uses: ./.github/workflows/_test-unit-base.yml + with: + test-path: "tests/test_litellm/responses tests/test_litellm/caching tests/test_litellm/types" + workers: 2 + reruns: 2 diff --git a/.github/workflows/test-unit-security.yml b/.github/workflows/test-unit-security.yml new file mode 100644 index 00000000000..b38c82b1c24 --- /dev/null +++ b/.github/workflows/test-unit-security.yml @@ -0,0 +1,28 @@ +name: "Unit Tests: Security" + +# Uses DATABASE_URL secret — only runs on trusted branches, not PRs. +on: + push: + branches: [main, "litellm_*"] + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + security: + uses: ./.github/workflows/_test-unit-services-base.yml + with: + test-path: "tests/proxy_security_tests/" + workers: 1 + reruns: 2 + timeout-minutes: 20 + enable-redis: false + enable-postgres: true + secrets: + DATABASE_URL: ${{ secrets.DATABASE_URL }} + POSTGRES_USER: ${{ secrets.POSTGRES_USER }} + POSTGRES_PASSWORD: ${{ secrets.POSTGRES_PASSWORD }} diff --git a/.github/workflows/test_server_root_path.yml b/.github/workflows/test_server_root_path.yml index c359e38bff9..47636ce8e92 100644 --- a/.github/workflows/test_server_root_path.yml +++ b/.github/workflows/test_server_root_path.yml @@ -17,13 +17,15 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v4 + uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + with: + persist-credentials: false - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 + uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12 - name: Build Docker image - uses: docker/build-push-action@v5 + uses: docker/build-push-action@0adf9959216b96bec444f325f1e493d4aa344497 #v6.14 with: context: . file: ./docker/Dockerfile.non_root diff --git a/.github/workflows/zizmor.yml b/.github/workflows/zizmor.yml new file mode 100644 index 00000000000..9a1e899fed5 --- /dev/null +++ b/.github/workflows/zizmor.yml @@ -0,0 +1,31 @@ +name: GitHub Actions Security Analysis + +on: + push: + branches: [main] + pull_request: + branches: [main] + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +permissions: {} + +jobs: + zizmor: + name: zizmor + runs-on: ubuntu-latest + timeout-minutes: 5 + permissions: + security-events: write + contents: read + actions: read + steps: + - name: Checkout repository + uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + with: + persist-credentials: false + + - name: Run zizmor + uses: zizmorcore/zizmor-action@71321a20a9ded102f6e9ce5718a2fcec2c4f70d8 # v0.5.2 diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 9396f323e45..2bc361bc48f 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -14,12 +14,12 @@ repos: types: [python] files: (litellm/|litellm_proxy_extras/|enterprise/).*\.py exclude: ^litellm/__init__.py$ - # - id: black - # name: black - # entry: poetry run black - # language: system - # types: [python] - # files: (litellm/|litellm_proxy_extras/|enterprise/).*\.py + - id: black + name: black + entry: poetry run black + language: system + types: [python] + files: (litellm/|litellm_proxy_extras/).*\.py - repo: https://github.com/pycqa/flake8 rev: 7.0.0 # The version of flake8 to use hooks: diff --git a/.semgrep/rules/security/no-claude-directory.yml b/.semgrep/rules/security/no-claude-directory.yml new file mode 100644 index 00000000000..7d120a7c23c --- /dev/null +++ b/.semgrep/rules/security/no-claude-directory.yml @@ -0,0 +1,18 @@ +rules: + - id: no-claude-directory-committed + message: > + .claude/ directory must not be committed to the repository. + It contains local Claude Code settings (permissions, worktree paths) that are + developer-machine-specific and may expose internal paths or credentials. + Add .claude/ to .gitignore instead. + severity: ERROR + languages: [generic] + paths: + include: + - "/.claude/**" + - "/.claude/*" + pattern-regex: '[\s\S]+' + metadata: + category: security + tags: [supply-chain, secrets] + confidence: HIGH diff --git a/CLAUDE.md b/CLAUDE.md index d9061b5e2be..f0478120181 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -140,6 +140,11 @@ LiteLLM is a unified interface for 100+ LLM providers with two main components: - **Check index coverage.** For new or modified queries, check `schema.prisma` for a supporting index. Prefer extending an existing index (e.g. `@@index([a])` → `@@index([a, b])`) over adding a new one, unless it's a `@@unique`. Only add indexes for large/frequent queries. - **Keep schema files in sync.** Apply schema changes to all `schema.prisma` copies (`schema.prisma`, `litellm/proxy/`, `litellm-proxy-extras/`, `litellm-js/spend-logs/` for SpendLogs) with a migration under `litellm-proxy-extras/litellm_proxy_extras/migrations/`. +### Setup Wizard (`litellm/setup_wizard.py`) +- The wizard is implemented as a single `SetupWizard` class with `@staticmethod` methods — keep it that way. No module-level functions except `run_setup_wizard()` (the public entrypoint) and pure helpers (color, ANSI). +- Use `litellm.utils.check_valid_key(model, api_key)` for credential validation — never roll a custom completion call. +- Do not hardcode provider env-key names or model lists that already exist in the codebase. Add a `test_model` field to each provider entry to drive `check_valid_key`; set it to `None` for providers that can't be validated with a single API key (Azure, Bedrock, Ollama). + ### Enterprise Features - Enterprise-specific code in `enterprise/` directory - Optional features enabled via environment variables diff --git a/README.md b/README.md index 67f2f3a2048..a6dc597574e 100644 --- a/README.md +++ b/README.md @@ -266,6 +266,7 @@ Support for more providers. Missing a provider or LLM Platform, raise a [feature + @@ -402,7 +403,7 @@ Support for more providers. Missing a provider or LLM Platform, raise a [feature # Enterprise For companies that need better security, user management and professional support -[Talk to founders](https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions) +[Talk to founders](https://enterprise.litellm.ai/demo) This covers: - ✅ **Features under the [LiteLLM Commercial License](https://docs.litellm.ai/docs/proxy/enterprise):** @@ -452,7 +453,6 @@ All these checks must pass before your PR can be merged. - [Schedule Demo 👋](https://calendly.com/d/4mp-gd3-k5k/berriai-1-1-onboarding-litellm-hosted-version) - [Community Discord 💭](https://discord.gg/wuPM9dRgDw) - [Community Slack 💭](https://www.litellm.ai/support) -- Our numbers 📞 +1 (770) 8783-106 / ‭+1 (412) 618-6238‬ - Our emails ✉️ ishaan@berri.ai / krrish@berri.ai # Why did we build this diff --git a/ci_cd/security_scans.sh b/ci_cd/security_scans.sh index e0f370e0035..061e454465a 100755 --- a/ci_cd/security_scans.sh +++ b/ci_cd/security_scans.sh @@ -10,13 +10,13 @@ echo "Starting security scans for LiteLLM..." # Function to install Trivy and required tools install_trivy() { echo "Installing Trivy and required tools..." + TRIVY_VERSION="0.35.0" sudo apt-get update - sudo apt-get install -y wget apt-transport-https gnupg lsb-release jq curl bsdmainutils - wget -qO - https://aquasecurity.github.io/trivy-repo/deb/public.key | sudo apt-key add - - echo "deb https://aquasecurity.github.io/trivy-repo/deb $(lsb_release -sc) main" | sudo tee -a /etc/apt/sources.list.d/trivy.list - sudo apt-get update - sudo apt-get install trivy - echo "Trivy and required tools installed successfully" + sudo apt-get install -y wget jq curl bsdmainutils + wget -qO trivy.deb "https://github.com/aquasecurity/trivy/releases/download/v${TRIVY_VERSION}/trivy_${TRIVY_VERSION}_Linux-64bit.deb" + sudo dpkg -i trivy.deb + rm trivy.deb + echo "Trivy ${TRIVY_VERSION} installed successfully" } # Function to install Grype @@ -163,6 +163,9 @@ run_grype_scans() { "CVE-2026-25639" # axios - full fix requires 1.x major version bump; pinned to >=0.30.2 to clear other axios CVEs, upgrade to 1.x in follow-up "CVE-2026-2297" # Python 3.13 SourcelessFileLoader audit hook bypass - no fix available in base image "GHSA-qffp-2rhf-9h96" # tar hardlink path traversal - from nodejs_wheel bundled npm, not used in application runtime code + "CVE-2026-2673" # OpenSSL 3.6.1 TLS 1.3 key exchange group negotiation issue - no fix available yet + "CVE-2026-3644" # Python 3.13 vulnerability - no fix available in base image + "CVE-2026-4224" # Python 3.13 Expat parser stack overflow in ElementDeclHandler - no fix available in base image ) # Build JSON array of allowlisted CVE IDs for jq diff --git a/cookbook/benchmark/readme.md b/cookbook/benchmark/readme.md index 57115eb96a9..afa59aa91ee 100644 --- a/cookbook/benchmark/readme.md +++ b/cookbook/benchmark/readme.md @@ -178,4 +178,4 @@ Benchmark Results for 'When will BerriAI IPO?': ``` ## Support -**🤝 Schedule a 1-on-1 Session:** Book a [1-on-1 session](https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions) with Krrish and Ishaan, the founders, to discuss any issues, provide feedback, or explore how we can improve LiteLLM for you. +**🤝 Schedule a 1-on-1 Session:** Book a [1-on-1 session](https://enterprise.litellm.ai/demo) with Krrish and Ishaan, the founders, to discuss any issues, provide feedback, or explore how we can improve LiteLLM for you. diff --git a/cookbook/codellama-server/README.MD b/cookbook/codellama-server/README.MD index b158bb083f2..82a7e62f40a 100644 --- a/cookbook/codellama-server/README.MD +++ b/cookbook/codellama-server/README.MD @@ -143,7 +143,6 @@ All responses from the server are returned in the following format (for all LLM - [Our calendar 👋](https://calendly.com/d/4mp-gd3-k5k/berriai-1-1-onboarding-litellm-hosted-version) - [Community Discord 💭](https://discord.gg/wuPM9dRgDw) -- Our numbers 📞 +1 (770) 8783-106 / +1 (412) 618-6238 - Our emails ✉️ ishaan@berri.ai / krrish@berri.ai ## Roadmap diff --git a/cookbook/litellm_proxy_server/readme.md b/cookbook/litellm_proxy_server/readme.md index d0b0592c433..2c1eab72c24 100644 --- a/cookbook/litellm_proxy_server/readme.md +++ b/cookbook/litellm_proxy_server/readme.md @@ -164,7 +164,6 @@ All responses from the server are returned in the following format (for all LLM - [Our calendar 👋](https://calendly.com/d/4mp-gd3-k5k/berriai-1-1-onboarding-litellm-hosted-version) - [Community Discord 💭](https://discord.gg/wuPM9dRgDw) -- Our numbers 📞 +1 (770) 8783-106 / +1 (412) 618-6238 - Our emails ✉️ ishaan@berri.ai / krrish@berri.ai ## Roadmap diff --git a/docs/my-website/blog/anthropic_opus_4_5_and_advanced_features/index.md b/docs/my-website/blog/anthropic_opus_4_5_and_advanced_features/index.md index 8a54426dfb0..21ba3d60790 100644 --- a/docs/my-website/blog/anthropic_opus_4_5_and_advanced_features/index.md +++ b/docs/my-website/blog/anthropic_opus_4_5_and_advanced_features/index.md @@ -3,18 +3,9 @@ slug: anthropic_advanced_features title: "Day 0 Support: Claude 4.5 Opus (+Advanced Features)" date: 2025-11-25T10:00:00 authors: - - name: Sameer Kankute - title: SWE @ LiteLLM (LLM Translation) - url: https://www.linkedin.com/in/sameer-kankute/ - image_url: https://pbs.twimg.com/profile_images/2001352686994907136/ONgNuSk5_400x400.jpg - - name: Krrish Dholakia - title: "CEO, LiteLLM" - url: https://www.linkedin.com/in/krish-d/ - image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg - - name: Ishaan Jaff - title: "CTO, LiteLLM" - url: https://www.linkedin.com/in/reffajnaahsi/ - image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg + - sameer + - krrish + - ishaan-alt description: "Guide to Claude Opus 4.5 and advanced features in LiteLLM: Tool Search, Programmatic Tool Calling, and Effort Parameter." tags: [anthropic, claude, tool search, programmatic tool calling, effort, advanced features] hide_table_of_contents: false @@ -25,6 +16,8 @@ import TabItem from '@theme/TabItem'; This guide covers Anthropic's latest model (Claude Opus 4.5) and its advanced features now available in LiteLLM: Tool Search, Programmatic Tool Calling, Tool Input Examples, and the Effort Parameter. +{/* truncate */} + --- | Feature | Supported Models | diff --git a/docs/my-website/blog/anthropic_wildcard_model_access_incident/index.md b/docs/my-website/blog/anthropic_wildcard_model_access_incident/index.md index f6172cd6744..8d58e18e580 100644 --- a/docs/my-website/blog/anthropic_wildcard_model_access_incident/index.md +++ b/docs/my-website/blog/anthropic_wildcard_model_access_incident/index.md @@ -3,18 +3,9 @@ slug: anthropic-wildcard-model-access-incident title: "Incident Report: Wildcard Blocking New Models After Cost Map Reload" date: 2026-02-23T10:00:00 authors: - - name: Sameer Kankute - title: SWE @ LiteLLM (LLM Translation) - url: https://www.linkedin.com/in/sameer-kankute/ - image_url: https://pbs.twimg.com/profile_images/2001352686994907136/ONgNuSk5_400x400.jpg - - name: Krrish Dholakia - title: "CEO, LiteLLM" - url: https://www.linkedin.com/in/krish-d/ - image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg - - name: Ishaan Jaff - title: "CTO, LiteLLM" - url: https://www.linkedin.com/in/reffajnaahsi/ - image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg + - sameer + - krrish + - ishaan-alt tags: [incident-report, proxy, auth, model-access] hide_table_of_contents: false --- diff --git a/docs/my-website/blog/authors.yml b/docs/my-website/blog/authors.yml index 2a49a736333..1b1ef4d34c4 100644 --- a/docs/my-website/blog/authors.yml +++ b/docs/my-website/blog/authors.yml @@ -4,6 +4,12 @@ litellm: url: https://github.com/BerriAI/litellm image_url: https://github.com/BerriAI.png +sameer: + name: Sameer Kankute + title: SWE @ LiteLLM (LLM Translation) + url: https://www.linkedin.com/in/sameer-kankute/ + image_url: https://pbs.twimg.com/profile_images/2001352686994907136/ONgNuSk5_400x400.jpg + krrish: name: Krrish Dholakia title: CEO, LiteLLM @@ -22,3 +28,21 @@ ishaan-alt: title: CTO, LiteLLM url: https://www.linkedin.com/in/reffajnaahsi/ image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg + +ryan: + name: Ryan Crabbe + title: Performance Engineer, LiteLLM + url: https://www.linkedin.com/in/ryan-crabbe-0b9687214 + image_url: https://media.licdn.com/dms/image/v2/D5603AQHt1t9Z4BJ6Gw/profile-displayphoto-shrink_400_400/profile-displayphoto-shrink_400_400/0/1724453682340?e=1772064000&v=beta&t=VXdmr13rsNB05wyA2F1TENOB5UuDHUZ0FCHTolNyR5M + +alexsander: + name: Alexsander Hamir + title: Performance Engineer, LiteLLM + url: https://www.linkedin.com/in/alexsander-baptista/ + image_url: https://github.com/AlexsanderHamir.png + +yuneng: + name: Yuneng Jiang + title: SWE @ LiteLLM (Full Stack) + url: https://www.linkedin.com/in/yuneng-david-jiang-455676139/ + image_url: https://avatars.githubusercontent.com/u/171294688?v=4 diff --git a/docs/my-website/blog/ci_cd_v2_improvements/index.md b/docs/my-website/blog/ci_cd_v2_improvements/index.md new file mode 100644 index 00000000000..84f8f7bda6b --- /dev/null +++ b/docs/my-website/blog/ci_cd_v2_improvements/index.md @@ -0,0 +1,55 @@ +--- +slug: ci-cd-v2-improvements +title: "Announcing CI/CD v2 for LiteLLM" +date: 2026-03-30T21:30:00 +authors: + - krrish +description: "CI/CD v2 introduces isolated environments, stronger security gates, and safer release separation for LiteLLM." +tags: [engineering, ci-cd, security] +hide_table_of_contents: false +--- + +import Image from '@theme/IdealImage'; + +The CI/CD v2 is now live for LiteLLM. + + + +
+Building on the roadmap from our [security incident](https://docs.litellm.ai/blog/security-townhall-updates#roadmap), CI/CD v2 introduces isolated environments, stronger security gates, and safer release separation for LiteLLM. + +## What changed + +- Security scans and unit tests run in isolated environments. +- Validation and release are separated into different repositories, making it harder for an attacker to reach release credentials. +- Trusted Publishing for PyPI releases - this means no long-lived credentials are used to publish releases. +- Immutable Docker release tags - this means no tampering of Docker release tags after they are published [Learn more](https://docs.docker.com/docker-hub/repos/manage/hub-images/immutable-tags/). Note: work for GHCR docker releases is planned as well. + +## What's next + +Moving forward, we plan on: +- Adopting OpenSSF (this is a set of security criteria that projects should meet to demonstrate a strong security posture - [Learn more](https://baseline.openssf.org/versions/2026-02-19.html)) + - We've added Scorecard and Allstar to our Github + +- Adding SLSA Build Provenance to our CI/CD pipeline - this means we allow users to independently verify that a release came from us and prevent silent modifications of releases after they are published. + + +We hope that this will mean you can be confident that the releases you are using are safe and from us. + + +## The principle + +The new CI/CD pipeline reflects the principles, outlined below, and is designed to be more secure and reliable: + +- **Limit** what each package can access +- **Reduce** the number of sensitive environment variables +- **Avoid** compromised packages +- **Prevent** release tampering + + +## How to help: + +Help us plan April's stability sprint - https://github.com/BerriAI/litellm/issues/24825 \ No newline at end of file diff --git a/docs/my-website/blog/claude_code_beta_headers/index.md b/docs/my-website/blog/claude_code_beta_headers/index.md index 44567f616aa..ee07da79397 100644 --- a/docs/my-website/blog/claude_code_beta_headers/index.md +++ b/docs/my-website/blog/claude_code_beta_headers/index.md @@ -3,18 +3,9 @@ slug: claude-code-beta-headers-incident title: "Incident Report: Invalid beta headers with Claude Code" date: 2026-02-16T10:00:00 authors: - - name: Sameer Kankute - title: SWE @ LiteLLM (LLM Translation) - url: https://www.linkedin.com/in/sameer-kankute/ - image_url: https://pbs.twimg.com/profile_images/2001352686994907136/ONgNuSk5_400x400.jpg - - name: Ishaan Jaff - title: "CTO, LiteLLM" - url: https://www.linkedin.com/in/reffajnaahsi/ - image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg - - name: Krrish Dholakia - title: "CEO, LiteLLM" - url: https://www.linkedin.com/in/krish-d/ - image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg + - sameer + - ishaan-alt + - krrish tags: [incident-report, anthropic, stability] hide_table_of_contents: false --- @@ -173,5 +164,5 @@ curl -X POST "https://your-proxy-url/reload/anthropic_beta_headers" \ ## Related documentation -- [Managing Anthropic Beta Headers](../proxy/sync_anthropic_beta_headers.md) - Complete configuration guide +- [Managing Anthropic Beta Headers](../../docs/proxy/sync_anthropic_beta_headers) - Complete configuration guide - [`anthropic_beta_headers_config.json`](https://github.com/BerriAI/litellm/blob/main/litellm/anthropic_beta_headers_config.json) - Current configuration file diff --git a/docs/my-website/blog/claude_opus_4_6/index.md b/docs/my-website/blog/claude_opus_4_6/index.md index e44420bd570..eeb5d5ff2f8 100644 --- a/docs/my-website/blog/claude_opus_4_6/index.md +++ b/docs/my-website/blog/claude_opus_4_6/index.md @@ -3,18 +3,9 @@ slug: claude_opus_4_6 title: "Day 0 Support: Claude Opus 4.6" date: 2026-02-05T10:00:00 authors: - - name: Sameer Kankute - title: SWE @ LiteLLM (LLM Translation) - url: https://www.linkedin.com/in/sameer-kankute/ - image_url: https://pbs.twimg.com/profile_images/2001352686994907136/ONgNuSk5_400x400.jpg - - name: Ishaan Jaff - title: "CTO, LiteLLM" - url: https://www.linkedin.com/in/reffajnaahsi/ - image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg - - name: Krrish Dholakia - title: "CEO, LiteLLM" - url: https://www.linkedin.com/in/krish-d/ - image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg + - sameer + - ishaan-alt + - krrish description: "Day 0 support for Claude Opus 4.6 on LiteLLM AI Gateway - use across Anthropic, Azure, Vertex AI, and Bedrock." tags: [anthropic, claude, opus 4.6] hide_table_of_contents: false @@ -25,6 +16,8 @@ import TabItem from '@theme/TabItem'; LiteLLM now supports Claude Opus 4.6 on Day 0. Use it across Anthropic, Azure, Vertex AI, and Bedrock through the LiteLLM AI Gateway. +{/* truncate */} + ## Docker Image ```bash diff --git a/docs/my-website/blog/claude_sonnet_4_6/index.md b/docs/my-website/blog/claude_sonnet_4_6/index.md index df54fa09792..12446c82c60 100644 --- a/docs/my-website/blog/claude_sonnet_4_6/index.md +++ b/docs/my-website/blog/claude_sonnet_4_6/index.md @@ -3,14 +3,8 @@ slug: claude_sonnet_4_6 title: "Day 0 Support: Claude Sonnet 4.6" date: 2026-02-17T10:00:00 authors: - - name: Ishaan Jaff - title: "CTO, LiteLLM" - url: https://www.linkedin.com/in/reffajnaahsi/ - image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg - - name: Krrish Dholakia - title: "CEO, LiteLLM" - url: https://www.linkedin.com/in/krish-d/ - image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg + - ishaan-alt + - krrish description: "Day 0 support for Claude Sonnet 4.6 on LiteLLM AI Gateway - use across Anthropic, Azure, Vertex AI, and Bedrock." tags: [anthropic, claude, sonnet 4.6] hide_table_of_contents: false @@ -21,6 +15,8 @@ import TabItem from '@theme/TabItem'; LiteLLM now supports Claude Sonnet 4.6 on Day 0. Use it across Anthropic, Azure, Vertex AI, and Bedrock through the LiteLLM AI Gateway. +{/* truncate */} + ## Docker Image ```bash diff --git a/docs/my-website/blog/fastapi_middleware_performance/index.mdx b/docs/my-website/blog/fastapi_middleware_performance/index.mdx index b0c5ba13634..e373326c071 100644 --- a/docs/my-website/blog/fastapi_middleware_performance/index.mdx +++ b/docs/my-website/blog/fastapi_middleware_performance/index.mdx @@ -3,18 +3,9 @@ slug: fastapi-middleware-performance title: "Your Middleware Could Be a Bottleneck" date: 2026-02-07T10:00:00 authors: - - name: Krrish Dholakia - title: "CEO, LiteLLM" - url: https://www.linkedin.com/in/krish-d/ - image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg - - name: Ishaan Jaff - title: "CTO, LiteLLM" - url: https://www.linkedin.com/in/reffajnaahsi/ - image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg - - name: Ryan Crabbe - title: "Performance Engineer, LiteLLM" - url: https://www.linkedin.com/in/ryan-crabbe-0b9687214 - image_url: https://media.licdn.com/dms/image/v2/D5603AQHt1t9Z4BJ6Gw/profile-displayphoto-shrink_400_400/profile-displayphoto-shrink_400_400/0/1724453682340?e=1772064000&v=beta&t=VXdmr13rsNB05wyA2F1TENOB5UuDHUZ0FCHTolNyR5M + - krrish + - ishaan-alt + - ryan description: "How we improved LiteLLM proxy latency and throughput by replacing a single middleware base class" tags: [performance, fastapi, middleware] hide_table_of_contents: false diff --git a/docs/my-website/blog/gemin_3.1/index.md b/docs/my-website/blog/gemin_3.1/index.md index b81595e4bd5..0afccb49d98 100644 --- a/docs/my-website/blog/gemin_3.1/index.md +++ b/docs/my-website/blog/gemin_3.1/index.md @@ -3,18 +3,9 @@ slug: gemini_3_1_pro title: "DAY 0 Support: Gemini 3.1 Pro on LiteLLM" date: 2026-02-19T10:00:00 authors: - - name: Sameer Kankute - title: SWE @ LiteLLM (LLM Translation) - url: https://www.linkedin.com/in/sameer-kankute/ - image_url: https://pbs.twimg.com/profile_images/2001352686994907136/ONgNuSk5_400x400.jpg - - name: Krrish Dholakia - title: "CEO, LiteLLM" - url: https://www.linkedin.com/in/krish-d/ - image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg - - name: Ishaan Jaff - title: "CTO, LiteLLM" - url: https://www.linkedin.com/in/reffajnaahsi/ - image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg + - sameer + - krrish + - ishaan-alt description: "Guide to using Gemini 3.1 Pro on LiteLLM Proxy and SDK with day 0 support." tags: [gemini, day 0 support, llms] hide_table_of_contents: false @@ -28,6 +19,8 @@ import TabItem from '@theme/TabItem'; LiteLLM now supports `gemini-3.1-pro-preview` and all the new API changes along with it. +{/* truncate */} + ## Deploy this version @@ -67,7 +60,7 @@ LiteLLM provides **full end-to-end support** for Gemini 3.1 Pro on: - ✅ `/v1/chat/completions` - OpenAI-compatible chat completions endpoint - ✅ `/v1/responses` - OpenAI Responses API endpoint (streaming and non-streaming) - ✅ [`/v1/messages`](../../docs/anthropic_unified) - Anthropic-compatible messages endpoint -- ✅ `/v1/generateContent` – [Google Gemini API](../../docs/generateContent.md) compatible endpoint +- ✅ `/v1/generateContent` – [Google Gemini API](../../docs/generateContent) compatible endpoint All endpoints support: - Streaming and non-streaming responses @@ -147,4 +140,3 @@ curl -X POST http://localhost:4000/v1/chat/completions \ | `high` | `high` | | `disable` | `minimal` | | `none` | `minimal` | - diff --git a/docs/my-website/blog/gemini_3/index.md b/docs/my-website/blog/gemini_3/index.md index 7263acc12c9..a5b94382b6f 100644 --- a/docs/my-website/blog/gemini_3/index.md +++ b/docs/my-website/blog/gemini_3/index.md @@ -3,18 +3,9 @@ slug: gemini_3 title: "DAY 0 Support: Gemini 3 on LiteLLM" date: 2025-11-19T10:00:00 authors: - - name: Sameer Kankute - title: SWE @ LiteLLM (LLM Translation) - url: https://www.linkedin.com/in/sameer-kankute/ - image_url: https://pbs.twimg.com/profile_images/2001352686994907136/ONgNuSk5_400x400.jpg - - name: Krrish Dholakia - title: "CEO, LiteLLM" - url: https://www.linkedin.com/in/krish-d/ - image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg - - name: Ishaan Jaff - title: "CTO, LiteLLM" - url: https://www.linkedin.com/in/reffajnaahsi/ - image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg + - sameer + - krrish + - ishaan-alt description: "Common questions and best practices for using gemini-3-pro-preview with LiteLLM Proxy and SDK." tags: [gemini, day 0 support, llms] hide_table_of_contents: false @@ -29,6 +20,8 @@ This guide covers common questions and best practices for using `gemini-3-pro-pr ::: +{/* truncate */} + ## Quick Start @@ -976,8 +969,7 @@ messages.append(response.choices[0].message) # ✅ Includes thought signatures ## Additional Resources -- [Gemini Provider Documentation](../gemini.md) -- [Thought Signatures Guide](../gemini.md#thought-signatures) -- [Reasoning Content Documentation](../../reasoning_content.md) -- [Function Calling Guide](../../function_calling.md) - +- [Gemini Provider Documentation](../../docs/providers/gemini) +- [Thought Signatures Guide](../../docs/providers/gemini#thought-signatures) +- [Reasoning Content Documentation](../../docs/reasoning_content) +- [Function Calling Guide](../../docs/completion/function_call) diff --git a/docs/my-website/blog/gemini_3_1_flash_lite/index.md b/docs/my-website/blog/gemini_3_1_flash_lite/index.md index 9ef4bacb2ad..0ae79e5fa67 100644 --- a/docs/my-website/blog/gemini_3_1_flash_lite/index.md +++ b/docs/my-website/blog/gemini_3_1_flash_lite/index.md @@ -3,18 +3,9 @@ slug: gemini_3_1_flash_lite_preview title: "DAY 0 Support: Gemini 3.1 Flash Lite Preview on LiteLLM" date: 2026-03-03T08:00:00 authors: - - name: Sameer Kankute - title: SWE @ LiteLLM (LLM Translation) - url: https://www.linkedin.com/in/sameer-kankute/ - image_url: https://pbs.twimg.com/profile_images/2001352686994907136/ONgNuSk5_400x400.jpg - - name: Krrish Dholakia - title: "CEO, LiteLLM" - url: https://www.linkedin.com/in/krish-d/ - image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg - - name: Ishaan Jaff - title: "CTO, LiteLLM" - url: https://www.linkedin.com/in/reffajnaahsi/ - image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg + - sameer + - krrish + - ishaan-alt description: "Guide to using Gemini 3.1 Flash Lite Preview on LiteLLM Proxy and SDK with day 0 support." tags: [gemini, day 0 support, llms, supernova] hide_table_of_contents: false @@ -32,6 +23,8 @@ LiteLLM now supports `gemini-3.1-flash-lite-preview` with full day 0 support! If you only want cost tracking, you need no change in your current Litellm version. But if you want the support for new features introduced along with it like thinking levels, you will need to use v1.80.8-stable.1 or above. ::: +{/* truncate */} + ## Deploy this version @@ -150,7 +143,7 @@ LiteLLM provides **full end-to-end support** for Gemini 3.1 Flash Lite Preview o - ✅ `/v1/chat/completions` - OpenAI-compatible chat completions endpoint - ✅ `/v1/responses` - OpenAI Responses API endpoint (streaming and non-streaming) - ✅ [`/v1/messages`](../../docs/anthropic_unified) - Anthropic-compatible messages endpoint -- ✅ `/v1/generateContent` – [Google Gemini API](../../docs/generateContent.md) compatible endpoint +- ✅ `/v1/generateContent` – [Google Gemini API](../../docs/generateContent) compatible endpoint All endpoints support: - Streaming and non-streaming responses @@ -172,4 +165,4 @@ LiteLLM automatically maps OpenAI's `reasoning_effort` parameter to Gemini's `th | `medium` | `medium` | Balanced reasoning for moderate complexity | | `high` | `high` | Maximum reasoning depth, complex problems | | `disable` | `minimal` | Disable extended reasoning | -| `none` | `minimal` | No extended reasoning | \ No newline at end of file +| `none` | `minimal` | No extended reasoning | diff --git a/docs/my-website/blog/gemini_3_flash/index.md b/docs/my-website/blog/gemini_3_flash/index.md index 830c21e5f66..5e98d2136b4 100644 --- a/docs/my-website/blog/gemini_3_flash/index.md +++ b/docs/my-website/blog/gemini_3_flash/index.md @@ -3,18 +3,9 @@ slug: gemini_3_flash title: "DAY 0 Support: Gemini 3 Flash on LiteLLM" date: 2025-12-17T10:00:00 authors: - - name: Sameer Kankute - title: SWE @ LiteLLM (LLM Translation) - url: https://www.linkedin.com/in/sameer-kankute/ - image_url: https://pbs.twimg.com/profile_images/2001352686994907136/ONgNuSk5_400x400.jpg - - name: Krrish Dholakia - title: "CEO, LiteLLM" - url: https://www.linkedin.com/in/krish-d/ - image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg - - name: Ishaan Jaff - title: "CTO, LiteLLM" - url: https://www.linkedin.com/in/reffajnaahsi/ - image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg + - sameer + - krrish + - ishaan-alt description: "Guide to using Gemini 3 Flash on LiteLLM Proxy and SDK with day 0 support." tags: [gemini, day 0 support, llms] hide_table_of_contents: false @@ -32,6 +23,8 @@ LiteLLM now supports `gemini-3-flash-preview` and all the new API changes along If you only want cost tracking, you need no change in your current Litellm version. But if you want the support for new features introduced along with it like thinking levels, you will need to use v1.80.8-stable.1 or above. ::: +{/* truncate */} + ## Deploy this version @@ -80,7 +73,7 @@ LiteLLM provides **full end-to-end support** for Gemini 3 Flash on: - ✅ `/v1/chat/completions` - OpenAI-compatible chat completions endpoint - ✅ `/v1/responses` - OpenAI Responses API endpoint (streaming and non-streaming) - ✅ [`/v1/messages`](../../docs/anthropic_unified) - Anthropic-compatible messages endpoint -- ✅ `/v1/generateContent` – [Google Gemini API](../../docs/generateContent.md) compatible endpoint +- ✅ `/v1/generateContent` – [Google Gemini API](../../docs/generateContent) compatible endpoint All endpoints support: - Streaming and non-streaming responses - Function calling with thought signatures @@ -252,4 +245,3 @@ If using this model via vertex_ai, keep the location as global as this is the on | `high` | `high` | | `disable` | `minimal` | | `none` | `minimal` | - diff --git a/docs/my-website/blog/gemini_embedding_2_multimodal/index.md b/docs/my-website/blog/gemini_embedding_2_multimodal/index.md index 8c09432e3b6..d66de1b6c79 100644 --- a/docs/my-website/blog/gemini_embedding_2_multimodal/index.md +++ b/docs/my-website/blog/gemini_embedding_2_multimodal/index.md @@ -3,10 +3,7 @@ slug: gemini_embedding_2_multimodal title: "Gemini Embedding 2 Preview: Multimodal Embeddings on LiteLLM" date: 2025-03-11T10:00:00 authors: - - name: Sameer Kankute - title: SWE @ LiteLLM (LLM Translation) - url: https://www.linkedin.com/in/sameer-kankute/ - image_url: https://pbs.twimg.com/profile_images/2001352686994907136/ONgNuSk5_400x400.jpg + - sameer description: "Generate embeddings from text, images, audio, video, and PDFs with gemini-embedding-2-preview on LiteLLM via Gemini API and Vertex AI." tags: [gemini, embeddings, multimodal, vertex ai] hide_table_of_contents: false @@ -19,6 +16,8 @@ import TabItem from '@theme/TabItem'; LiteLLM now supports **multimodal embeddings** with `gemini-embedding-2-preview`—generating a single embedding from a mix of text, images, audio, video, and PDF content. Available via both the **Gemini API** (API key) and **Vertex AI** (GCP credentials). +{/* truncate */} + ## Supported Input Types | Modality | Supported Formats | diff --git a/docs/my-website/blog/gpt_5_3_codex/index.md b/docs/my-website/blog/gpt_5_3_codex/index.md index 850586538f6..1dfab1f7ac7 100644 --- a/docs/my-website/blog/gpt_5_3_codex/index.md +++ b/docs/my-website/blog/gpt_5_3_codex/index.md @@ -3,18 +3,9 @@ slug: gpt_5_3_codex title: "Day 0 Support: GPT-5.3-Codex" date: 2026-02-24T10:00:00 authors: - - name: Sameer Kankute - title: SWE @ LiteLLM (LLM Translation) - url: https://www.linkedin.com/in/sameer-kankute/ - image_url: https://pbs.twimg.com/profile_images/2001352686994907136/ONgNuSk5_400x400.jpg - - name: Krrish Dholakia - title: "CEO, LiteLLM" - url: https://www.linkedin.com/in/krish-d/ - image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg - - name: Ishaan Jaff - title: "CTO, LiteLLM" - url: https://www.linkedin.com/in/reffajnaahsi/ - image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg + - sameer + - krrish + - ishaan-alt description: "Day 0 support for GPT-5.3-Codex on LiteLLM, including phase parameter handling for Responses API." tags: [openai, gpt-5.3-codex, codex, day 0 support] hide_table_of_contents: false @@ -25,6 +16,8 @@ import TabItem from '@theme/TabItem'; LiteLLM now supports GPT-5.3-Codex on Day 0, including support for the new assistant `phase` metadata on Responses API output items. +{/* truncate */} + ## Why `phase` matters for GPT-5.3-Codex `phase` appears on assistant output items and helps distinguish preamble/commentary turns from final closeout responses. diff --git a/docs/my-website/blog/gpt_5_4/index.md b/docs/my-website/blog/gpt_5_4/index.md index de099736f00..4f7e4344157 100644 --- a/docs/my-website/blog/gpt_5_4/index.md +++ b/docs/my-website/blog/gpt_5_4/index.md @@ -3,18 +3,9 @@ slug: gpt_5_4 title: "Day 0 Support: GPT-5.4" date: 2026-03-05T10:00:00 authors: - - name: Sameer Kankute - title: SWE @ LiteLLM (LLM Translation) - url: https://www.linkedin.com/in/sameer-kankute/ - image_url: https://pbs.twimg.com/profile_images/2001352686994907136/ONgNuSk5_400x400.jpg - - name: Krrish Dholakia - title: "CEO, LiteLLM" - url: https://www.linkedin.com/in/krish-d/ - image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg - - name: Ishaan Jaff - title: "CTO, LiteLLM" - url: https://www.linkedin.com/in/reffajnaahsi/ - image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg + - sameer + - krrish + - ishaan-alt description: "GPT-5.4 model support in LiteLLM" tags: [openai, gpt-5.4, completion] hide_table_of_contents: false @@ -25,6 +16,8 @@ import TabItem from '@theme/TabItem'; LiteLLM now supports fully GPT-5.4! +{/* truncate */} + ## Docker Image ```bash diff --git a/docs/my-website/blog/gpt_5_4_mini_nano/index.md b/docs/my-website/blog/gpt_5_4_mini_nano/index.md new file mode 100644 index 00000000000..6d7c2b33f72 --- /dev/null +++ b/docs/my-website/blog/gpt_5_4_mini_nano/index.md @@ -0,0 +1,106 @@ +--- +slug: gpt_5_4_mini_nano +title: "Day 0 Support: GPT-5.4-mini and GPT-5.4-nano" +date: 2026-03-17T10:00:00 +authors: + - name: Sameer Kankute + title: SWE @ LiteLLM (LLM Translation) + url: https://www.linkedin.com/in/sameer-kankute/ + image_url: https://pbs.twimg.com/profile_images/2001352686994907136/ONgNuSk5_400x400.jpg + - name: Krrish Dholakia + title: "CEO, LiteLLM" + url: https://www.linkedin.com/in/krish-d/ + image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg + - name: Ishaan Jaff + title: "CTO, LiteLLM" + url: https://www.linkedin.com/in/reffajnaahsi/ + image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg +description: "GPT-5.4-mini and GPT-5.4-nano model support in LiteLLM" +tags: [openai, gpt-5.4-mini, gpt-5.4-nano, completion] +hide_table_of_contents: false +--- + +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +LiteLLM now supports GPT-5.4-mini and GPT-5.4-nano — cost-effective models for simple completions and high-throughput workloads. + +:::note +If you're on **v1.82.3-stable** or above, you don't need any update to use these models. +::: + +## Usage + + + + +**1. Setup config.yaml** + +```yaml +model_list: + - model_name: gpt-5.4-mini + litellm_params: + model: openai/gpt-5.4-mini + api_key: os.environ/OPENAI_API_KEY + - model_name: gpt-5.4-nano + litellm_params: + model: openai/gpt-5.4-nano + api_key: os.environ/OPENAI_API_KEY +``` + +**2. Start the proxy** + +```bash +litellm --config /path/to/config.yaml +``` + +**3. Test it** + +```bash +# GPT-5.4-mini +curl -X POST "http://localhost:4000/v1/chat/completions" \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer $LITELLM_KEY" \ + -d '{ + "model": "gpt-5.4-mini", + "messages": [{"role": "user", "content": "What is the capital of France?"}] + }' + +# GPT-5.4-nano +curl -X POST "http://localhost:4000/v1/chat/completions" \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer $LITELLM_KEY" \ + -d '{ + "model": "gpt-5.4-nano", + "messages": [{"role": "user", "content": "What is 2 + 2?"}] + }' +``` + + + + +```python +from litellm import completion + +# GPT-5.4-mini +response = completion( + model="openai/gpt-5.4-mini", + messages=[{"role": "user", "content": "What is the capital of France?"}], +) +print(response.choices[0].message.content) + +# GPT-5.4-nano +response = completion( + model="openai/gpt-5.4-nano", + messages=[{"role": "user", "content": "What is 2 + 2?"}], +) +print(response.choices[0].message.content) +``` + + + + +## Notes + +- Both models support function calling, vision, and tool-use — see the [OpenAI provider docs](../../docs/providers/openai) for advanced usage. +- GPT-5.4-nano is the most cost-effective option for simple tasks; GPT-5.4-mini offers a balance of speed and capability. diff --git a/docs/my-website/blog/guardrail_logging_secret_exposure_incident/index.md b/docs/my-website/blog/guardrail_logging_secret_exposure_incident/index.md new file mode 100644 index 00000000000..71f9e3da011 --- /dev/null +++ b/docs/my-website/blog/guardrail_logging_secret_exposure_incident/index.md @@ -0,0 +1,78 @@ +--- +slug: guardrail-logging-secret-exposure-incident +title: "Incident Report: Guardrail logging exposed secret headers in spend logs and traces" +date: 2026-03-18T10:00:00 +authors: + - litellm +tags: [incident-report, security, guardrails] +hide_table_of_contents: false +--- + +**Date:** March 18, 2026 +**Duration:** Unknown +**Severity:** High +**Status:** Resolved + +## Summary + +When a custom guardrail returned the full LiteLLM request/data dictionary, the guardrail response logged by LiteLLM could include `secret_fields.raw_headers`, including plaintext `Authorization` headers containing API keys or other credentials. + +This information could then propagate to logging and observability surfaces that consume guardrail metadata, including: + +- **Spend logs in the LiteLLM UI:** visible to admins with access to spend-log data +- **OpenTelemetry traces:** visible to anyone with access to the relevant telemetry backend + +LLM calls, proxy routing, and provider execution were not blocked by this bug. The impact was exposure of sensitive request headers in observability and logging paths. + +{/* truncate */} + +--- + +## Background + +LiteLLM keeps internal request data (including request headers) for use during the call. That data is not meant to be written to logs or telemetry. + +When custom guardrails run, their outcomes are logged so they can appear in spend logs, OpenTelemetry traces, and other observability backends. If a guardrail returned the full request payload instead of a minimal result, that internal request data could be included in what was logged. Before the fix, the guardrail logging path did not strip that data before sending it to those systems. + +```mermaid +flowchart TD + inboundRequest["1. Incoming proxy request"] --> storeSecrets["2. Store internal request data"] + storeSecrets --> guardrailRuns["3. Custom guardrail runs"] + guardrailRuns --> fullDataReturn["4. Guardrail returns full request payload"] + fullDataReturn --> loggingBuild["5. Build guardrail log payload"] + loggingBuild --> spendLogs["6a. Persist to spend logs / UI"] + loggingBuild --> otelTraces["6b. Attach to OTEL guardrail spans"] +``` + +--- + +## Root Cause + +The root cause was incomplete sanitization in the guardrail logging path. When building the payload that gets sent to spend logs and traces, LiteLLM prepared guardrail responses for logging but did not strip internal request data (such as headers) from them. If a guardrail returned a response that included that data, it was passed through to the logging and observability systems unchanged. + +--- + +## Impact + +This issue required all of the following: + +1. A custom guardrail returned the full LiteLLM request/data dictionary, or another response object containing `secret_fields`. +2. LiteLLM logged that guardrail response through the standard guardrail logging path. +3. An operator, admin, or telemetry consumer had access to the resulting logs or traces. + +When those conditions were met, sensitive values could become visible through: + +- **Spend logs / UI responses:** guardrail metadata could be included in spend-log payloads rendered in the admin UI. +- **OpenTelemetry traces:** `guardrail_response` could be written as a span attribute on guardrail spans. +- **Other downstream observability backends:** any integration consuming the same guardrail metadata could receive the leaked values. + +This was a logging and telemetry exposure bug. It did not let callers bypass auth, access other tenants directly, or change model behavior, but it could expose plaintext credentials to people with access to those observability systems. + +--- + +## Guidance For Users + +- Upgrade to LiteLLM 1.82.3+. +- If you operated custom guardrails that return the full request/data dict, review whether spend logs or telemetry traces were retained during the affected period. +- Rotate any credentials that may have appeared in `Authorization` or other forwarded request headers in those systems. +- Apply least-privilege access controls to spend-log views and telemetry backends that may contain request-derived metadata. diff --git a/docs/my-website/blog/httpx_cache_eviction_incident/index.md b/docs/my-website/blog/httpx_cache_eviction_incident/index.md index 9e6152d0e63..7fc3789a91d 100644 --- a/docs/my-website/blog/httpx_cache_eviction_incident/index.md +++ b/docs/my-website/blog/httpx_cache_eviction_incident/index.md @@ -3,17 +3,9 @@ slug: httpx-cache-eviction-incident title: "Incident Report: Cache Eviction Closes In-Use httpx Clients" date: 2026-02-27T10:00:00 authors: - - name: Ryan Crabbe - title: Performance Engineer, LiteLLM - url: https://www.linkedin.com/in/ryan-crabbe-0b9687214 - - name: Ishaan Jaff - title: "CTO, LiteLLM" - url: https://www.linkedin.com/in/reffajnaahsi/ - image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg - - name: Krrish Dholakia - title: "CEO, LiteLLM" - url: https://www.linkedin.com/in/krish-d/ - image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg + - ryan + - ishaan-alt + - krrish tags: [incident-report, caching, stability] hide_table_of_contents: false --- @@ -31,6 +23,8 @@ A change to improve Redis connection pool cleanup introduced a regression that c **Impact:** Any proxy instance that hit the cache TTL (default 10 minutes) or capacity limit (200 entries) would have its httpx clients closed out from under it, causing requests to LLM providers to fail with connection errors. +{/* truncate */} + --- ## Background diff --git a/docs/my-website/blog/litellm_observatory/index.md b/docs/my-website/blog/litellm_observatory/index.md index 4554f77fb85..36366e5de22 100644 --- a/docs/my-website/blog/litellm_observatory/index.md +++ b/docs/my-website/blog/litellm_observatory/index.md @@ -3,18 +3,9 @@ slug: litellm-observatory title: "Improve release stability with 24 hour load tests" date: 2026-02-06T10:00:00 authors: - - name: Alexsander Hamir - title: "Performance Engineer, LiteLLM" - url: https://www.linkedin.com/in/alexsander-baptista/ - image_url: https://github.com/AlexsanderHamir.png - - name: Krrish Dholakia - title: "CEO, LiteLLM" - url: https://www.linkedin.com/in/krish-d/ - image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg - - name: Ishaan Jaff - title: "CTO, LiteLLM" - url: https://www.linkedin.com/in/reffajnaahsi/ - image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg + - alexsander + - krrish + - ishaan-alt description: "How we built a long-running, release-validation system to catch regressions before they reach users." tags: [testing, observability, reliability, releases] hide_table_of_contents: false @@ -28,6 +19,8 @@ As LiteLLM adoption has grown, so have expectations around reliability, performa This post introduces **LiteLLM Observatory**, a long-running release-validation system we built to catch regressions before they reach users. +{/* truncate */} + --- ## Why We Built the Observatory @@ -133,4 +126,3 @@ Reliability is an ongoing investment. LiteLLM Observatory is one of several systems we’re building to continuously raise the bar on release quality and operational safety. As LiteLLM evolves, so will our validation tooling, informed by real-world usage and lessons learned. We’ll continue to share those improvements openly as we go. - diff --git a/docs/my-website/blog/minimax_m2_5/index.md b/docs/my-website/blog/minimax_m2_5/index.md index 50084fcc1e5..9bccbdf7979 100644 --- a/docs/my-website/blog/minimax_m2_5/index.md +++ b/docs/my-website/blog/minimax_m2_5/index.md @@ -3,18 +3,9 @@ slug: minimax_m2_5 title: "Day 0 Support: MiniMax-M2.5" date: 2026-02-12T10:00:00 authors: - - name: Sameer Kankute - title: SWE @ LiteLLM (LLM Translation) - url: https://www.linkedin.com/in/sameer-kankute/ - image_url: https://pbs.twimg.com/profile_images/2001352686994907136/ONgNuSk5_400x400.jpg - - name: Krrish Dholakia - title: "CEO, LiteLLM" - url: https://www.linkedin.com/in/krish-d/ - image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg - - name: Ishaan Jaff - title: "CTO, LiteLLM" - url: https://www.linkedin.com/in/reffajnaahsi/ - image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg + - sameer + - krrish + - ishaan-alt description: "Day 0 support for MiniMax-M2.5 on LiteLLM" tags: [minimax, M2.5, llm] hide_table_of_contents: false @@ -25,6 +16,8 @@ import TabItem from '@theme/TabItem'; LiteLLM now supports MiniMax-M2.5 on Day 0. Use it across OpenAI-compatible and Anthropic-compatible APIs through the LiteLLM AI Gateway. +{/* truncate */} + ## Supported Models LiteLLM supports the following MiniMax models: diff --git a/docs/my-website/blog/model_cost_map_incident/index.md b/docs/my-website/blog/model_cost_map_incident/index.md index b9ff20e4128..5b4499cc31c 100644 --- a/docs/my-website/blog/model_cost_map_incident/index.md +++ b/docs/my-website/blog/model_cost_map_incident/index.md @@ -3,10 +3,7 @@ slug: model-cost-map-incident title: "Incident Report: Invalid model cost map on main" date: 2026-02-10T10:00:00 authors: - - name: Ishaan Jaffer - title: "CTO, LiteLLM" - url: https://www.linkedin.com/in/ishaanjaffer/ - image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg + - ishaan tags: [incident-report, stability] hide_table_of_contents: false --- diff --git a/docs/my-website/blog/realtime_webrtc_http_endpoints/index.md b/docs/my-website/blog/realtime_webrtc_http_endpoints/index.md index 04c3d3c9097..70fc5b2c48e 100644 --- a/docs/my-website/blog/realtime_webrtc_http_endpoints/index.md +++ b/docs/my-website/blog/realtime_webrtc_http_endpoints/index.md @@ -3,18 +3,9 @@ slug: realtime_webrtc_http_endpoints title: "Realtime WebRTC HTTP Endpoints" date: 2026-03-12T10:00:00 authors: - - name: Sameer Kankute - title: SWE @ LiteLLM (LLM Translation) - url: https://www.linkedin.com/in/sameer-kankute/ - image_url: https://pbs.twimg.com/profile_images/2001352686994907136/ONgNuSk5_400x400.jpg - - name: Krrish Dholakia - title: "CEO, LiteLLM" - url: https://www.linkedin.com/in/krish-d/ - image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg - - name: Ishaan Jaff - title: "CTO, LiteLLM" - url: https://www.linkedin.com/in/reffajnaahsi/ - image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg + - sameer + - krrish + - ishaan-alt description: "Use the LiteLLM proxy to route OpenAI-style WebRTC realtime via HTTP: client_secrets and SDP exchange." tags: [realtime, webrtc, proxy, openai] hide_table_of_contents: false @@ -24,6 +15,8 @@ import WebRTCTester from '@site/src/components/WebRTCTester'; Connect to the Realtime API via WebRTC from browser/mobile clients. LiteLLM handles auth and key management. +{/* truncate */} + ## How it works ![WebRTC flow: Browser, LiteLLM Proxy, and OpenAI/Azure](../../img/webrtc_flow.png) @@ -116,4 +109,3 @@ A: Set the correct `api_version` in `litellm_params` (or via the `AZURE_API_VERS **Q: What if I get no audio?** A: Make sure you grant microphone permission, ensure `pc.ontrack` assigns the audio element with `autoplay` enabled, check your network/firewall for WebRTC traffic, and inspect the browser console for ICE or SDP errors. - diff --git a/docs/my-website/blog/responses_api_encrypted_content_incident/index.md b/docs/my-website/blog/responses_api_encrypted_content_incident/index.md index 19b55898caa..fd5a9e76c42 100644 --- a/docs/my-website/blog/responses_api_encrypted_content_incident/index.md +++ b/docs/my-website/blog/responses_api_encrypted_content_incident/index.md @@ -3,18 +3,9 @@ slug: responses-api-encrypted-content-incident title: "Incident Report: Encrypted Content Failures in Multi-Region Responses API Load Balancing" date: 2026-02-24T10:00:00 authors: - - name: Sameer Kankute - title: SWE @ LiteLLM (LLM Translation) - url: https://www.linkedin.com/in/sameer-kankute/ - image_url: https://pbs.twimg.com/profile_images/2001352686994907136/ONgNuSk5_400x400.jpg - - name: Krrish Dholakia - title: "CEO, LiteLLM" - url: https://www.linkedin.com/in/krish-d/ - image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg - - name: Ishaan Jaff - title: "CTO, LiteLLM" - url: https://www.linkedin.com/in/reffajnaahsi/ - image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg + - sameer + - krrish + - ishaan-alt tags: [incident-report, proxy, responses-api, load-balancing] hide_table_of_contents: false --- diff --git a/docs/my-website/blog/security_townhall_updates/index.md b/docs/my-website/blog/security_townhall_updates/index.md new file mode 100644 index 00000000000..b997de9c185 --- /dev/null +++ b/docs/my-website/blog/security_townhall_updates/index.md @@ -0,0 +1,190 @@ +--- +slug: security-townhall-updates +title: "Security Townhall Updates" +date: 2026-03-27T12:00:00 +authors: + - krrish + - ishaan-alt +description: "What happened, what we've done, and what comes next for LiteLLM's release and security processes." +tags: [security, incident-report] +hide_table_of_contents: false +--- + +import Image from '@theme/IdealImage'; + +Thank you to everyone who joined our town hall. + +We wanted to use that time to walk through what we know, what we've done so far, and how we're improving LiteLLM's release and security processes going forward. This post is a written version of that update. [Slides available here](https://drive.google.com/file/d/17hsSG7nk-OYL7VRCTbTa7McrWREtS9OO/view?usp=sharing) + +{/* truncate */} + +## What happened + +On March 24, 2026 at 10:39 UTC, LiteLLM v1.82.7 was pushed to PyPI. Version v1.82.8 was published soon after. Those packages were live for about 40 minutes before being quarantined by PyPI. By 16:00 UTC, the LiteLLM team had worked with PyPI to delete the affected packages. + +At this point, our understanding is that this was a supply-chain incident affecting those two published versions. + +## How did this happen? + +Our understanding is that the issue came from the [compromised Trivy security scanner](https://www.aquasec.com/blog/trivy-supply-chain-attack-what-you-need-to-know/) dependency in our CI/CD pipeline. + + + +There were three major contributing factors: + +### 1. Shared CI/CD environment + +At the time, everything was running on CircleCI, and all steps shared a common environment. That increased blast radius: if one component was compromised, it could potentially access credentials or context intended for other parts of the pipeline. + +### 2. Static credentials in environment variables + +Release credentials, including credentials for PyPI, GHCR, and Docker publishing, were available as static secrets in the environment. That meant a compromised step could access long-lived release credentials. + +### 3. Unpinned Trivy dependency + +In our security scanning component, we had an unpinned Trivy dependency. Our present understanding is that a compromised Trivy package ran during the scan, had access to environment variables, and enabled attackers to obtain those credentials. + +**In summary:** a compromised package in CI had access to secrets it should not have had, and those secrets were then used in the release path. + +## What we've already done + + +In the last 3 days, we've taken the following steps: + +### 1. Minimize Scope of Impact + +#### Prevented further key abuse + +We deleted or rotated all impacted or adjacent secret keys, including PyPI, GitHub, Docker, and related credentials. Out of an abundance of caution, we've also rotated LiteLLM maintainer accounts. + +#### Prevent branch attacks + +We removed roughly 6,000 open branches and added an auto-deletion policy for branches merged into `main`. This reduces the surface area for branch-based abuse. + +#### Pinned CI/CD dependencies + +We've pinned all Github Actions, and are working on pinning all CircleCI dependencies as well. + +#### Paused releases + +We've paused new releases until we've confirmed codebase security and put stronger release controls in place. + +### 2. Secured LiteLLM + +#### Forensic analysis + +We are working with Google's Mandiant cybersecurity team to confirm the source of the attack and verify the security of the codebase. We also confirmed that no malicious code was pushed to `main`. + +#### Confirm Application Security + +In parallel, we are working with whitehat hackers at [Veria Labs](https://verialabs.com/) to verify application security and review improvements to our CI/CD process. + +We have also confirmed that the last 20 LiteLLM releases contain no indicators of compromise, and that no unauthenticated attacks can be made against LiteLLM Proxy based on our current investigation. [Check Security Blog for release verification.](https://docs.litellm.ai/blog/security-update-march-2026#verified-safe-versions) + +#### Created a security working group + +We created a new security working group inside LiteLLM focused on: + +- Building threat models +- Auditing the build process and dependencies + +If you're interested in joining the security working group, please file an issue [here](https://github.com/BerriAI/litellm-security-wg). + +### 3. Improved CI/CD + +We've already begun making structural changes to how releases are built and published. These align with our goals (covered in the next section) around isolated environments, ephemeral credentials, and release auditing. + +## Roadmap + +We plan on following 4 guiding principles for our new CI/CD pipeline: + +1. **Limit** what each package can access +2. **Reduce** the number of sensitive environment variables +3. **Avoid** compromised packages +4. **Prevent** release tampering + + +### Isolated environments + + + +We are breaking our CI/CD into 4 semantic concepts: + +1. Unit tests +2. Integration tests +3. Security scans +4. Release publishing + +And will be running each of these in isolated environments. + +This will limit the damage that any single compromised component can cause. + +### Ephemeral credentials + +We plan to move to ephemeral credentials for PyPI (Trusted Publisher) and GHCR (Token-based authentication) releases. This will reduce the risk of credentials being leaked or compromised. + +We have already begun doing this: + +- PyPI Trusted Publisher on GitHub Actions [PR](https://github.com/BerriAI/litellm/pull/24654) +- GHCR Token-based authentication on GitHub Actions [PR](https://github.com/BerriAI/litellm/pull/24683) + +### Release auditing + +Our goal is to allow users to independently verify that a release came from us and prevent silent modifications of releases after they are published. + +This will ensure, your releases are safe, even when: +- Stolen PyPI/GHCR credentials are used to publish malicious releases +- Tampered registry artifacts are published +- Tag mutations are made after the release is published + +We believe that [Cosign](https://github.com/sigstore/cosign) is a good fit for this, and have already begun working on it [PR](https://github.com/BerriAI/litellm/pull/24683). + + +### Avoid Compromised Packages + +- Move to pinned, verified SHAs for packages and actions used in CI/CD, avoiding `latest` wherever possible. +- Add a cooldown period before upgrading to a new version of a package - allows more time to investigate and verify the new version. + +We've added zizmor to help us catch issues such as unpinned dependencies and credential leakage. [commit](https://github.com/BerriAI/litellm/commit/a671275f5c5b0e1fb1adacdf3b6ef779aaa5d56c). + + +## Frequently Asked Questions + +**Q: Did you observe any lateral movement into your corporate environment during this incident?** + +A: No. Our investigation to date, conducted in coordination with external security experts, has found no evidence of lateral movement into our internal corporate systems. The incident was isolated to the CI/CD pipeline and the release path for specific versions (v1.82.7 and v1.82.8). As a proactive measure, we have rotated all potentially impacted or adjacent secrets—including PyPI, GitHub, and Docker credentials—and updated maintainer account security to ensure continued isolation. + +**Q: Do you expect delays in future product releases due to these new security measures?** + +A: We are committed to balancing security with speed. While we have temporarily paused releases to implement stronger controls, we are moving quickly to automate our new security protocols. We are currently implementing isolated CI/CD environments, ephemeral credentials (via Trusted Publishers), and release auditing with Cosign. These improvements are designed to be integrated into our automated pipeline, allowing us to maintain a fast release cadence while ensuring every package is verified and secure. + +**Q: Were older packages impacted?** + +Our current findings show no indicators of compromise in the last 20 versions of LiteLLM. This was manually verified by our team and independently reviewed by Veria Labs. + +We have also published the verified versions for users to use. [Check Security Blog for release verification.](https://docs.litellm.ai/blog/security-update-march-2026#verified-safe-versions) + + + +## Questions & Support + +If you believe your systems may be affected, contact us immediately: + +- **Security:** security@berri.ai +- **Support:** support@berri.ai +- **Slack:** Reach out to the LiteLLM team directly [here](https://join.slack.com/t/litellmossslack/shared_invite/zt-3o7nkuyfr-p_kbNJj8taRfXGgQI1~YyA) + +## Hiring + +We are currently hiring for: + +- DevOps Engineer - to keep ci/cd secure and running smoothly +- Security Engineer - to keep the application secure + +If you're interest in joining, please apply [here](https://jobs.ashbyhq.com/litellm) \ No newline at end of file diff --git a/docs/my-website/blog/security_townhall_updates/shared_ci_cd_environment.png b/docs/my-website/blog/security_townhall_updates/shared_ci_cd_environment.png new file mode 100644 index 00000000000..29ec195b7fb Binary files /dev/null and b/docs/my-website/blog/security_townhall_updates/shared_ci_cd_environment.png differ diff --git a/docs/my-website/blog/security_update_march_2026/index.md b/docs/my-website/blog/security_update_march_2026/index.md new file mode 100644 index 00000000000..1c298fe372f --- /dev/null +++ b/docs/my-website/blog/security_update_march_2026/index.md @@ -0,0 +1,786 @@ +--- +slug: security-update-march-2026 +title: "Security Update: Suspected Supply Chain Incident" +date: 2026-03-24T14:00:00 +authors: + - krrish + - ishaan-alt +description: "As of 2:00 PM ET on March 24, 2026" +tags: [security, incident-report] +hide_table_of_contents: false +--- + +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; +import VersionVerificationTable from '@site/src/components/VersionVerificationTable'; + +> **Status:** Active investigation +> **Last updated:** March 27, 2026 + +> **Update (March 30):** A new **clean** version of LiteLLM is now available (v1.83.0). This was released by our new [CI/CD v2](https://docs.litellm.ai/blog/ci-cd-v2-improvements) pipeline which added isolated environments, stronger security gates, and safer release separation for LiteLLM. + +> **Update (March 27):** Review Townhall updates, including explanation of the incident, what we've done, and what comes next. [Learn more](https://docs.litellm.ai/blog/security-townhall-updates) + +> **Update (March 27):** Added [Verified safe versions](#verified-safe-versions) section with SHA-256 checksums for all audited PyPI and Docker releases. + +> **Update (March 26):** Added `checkmarx[.]zone` to [Indicators of compromise](#indicators-of-compromise-iocs) + +> **Update (March 25):** Added community-contributed scripts for scanning GitHub Actions and GitLab CI pipelines for the compromised versions. See [How to check if you are affected](#how-to-check-if-you-are-affected). s/o [@Zach Fury](https://www.linkedin.com/in/fryware/) for these scripts. + + +## TLDR; +- The compromised PyPI packages were **litellm==1.82.7** and **litellm==1.82.8**. Those packages were live on March 24, 2026 from 10:39 UTC for about 40 minutes before being quarantined by PyPI. +- We believe that the compromise originated from the [Trivy dependency](https://www.aquasec.com/blog/trivy-supply-chain-attack-what-you-need-to-know/) used in our CI/CD security scanning workflow. +- Customers running the official LiteLLM Proxy Docker image were not impacted. That deployment path pins dependencies in requirements.txt and does not rely on the compromised PyPI packages. +- ~~We have paused all new LiteLLM releases until we complete a broader supply-chain review and confirm the release path is safe.~~ **Updated:** We have now released a new **safe** version of LiteLLM (v1.83.0) by our new [CI/CD v2](https://docs.litellm.ai/blog/ci-cd-v2-improvements) pipeline which added isolated environments, stronger security gates, and safer release separation for LiteLLM. We have also verified the codebase is safe and no malicious code was pushed to `main`. + + +## Overview + +LiteLLM AI Gateway is investigating a suspected supply chain attack involving unauthorized PyPI package publishes. Current evidence suggests a maintainer's PyPI account may have been compromised and used to distribute malicious code. + +At this time, we believe this incident may be linked to the broader [Trivy security compromise](https://www.aquasec.com/blog/trivy-supply-chain-attack-what-you-need-to-know/), in which stolen credentials were reportedly used to gain unauthorized access to the LiteLLM publishing pipeline. + +This investigation is ongoing. Details below may change as we confirm additional findings. + +## Confirmed affected versions + +The following LiteLLM versions published to PyPI were impacted: + +- **v1.82.7**: contained a malicious payload in the LiteLLM AI Gateway `proxy_server.py` +- **v1.82.8**: contained `litellm_init.pth` and a malicious payload in the LiteLLM AI Gateway `proxy_server.py` + +If you installed or ran either of these versions, review the recommendations below immediately. + +Note: These versions have already been removed from PyPI. + +## What happened + +Initial evidence suggests the attacker bypassed official CI/CD workflows and uploaded malicious packages directly to PyPI. + +These compromised versions appear to have included a credential stealer designed to: + +- Harvest secrets by scanning for: + - environment variables + - SSH keys + - cloud provider credentials (AWS, GCP, Azure) + - Kubernetes tokens + - database passwords +- Encrypt and exfiltrate data via a `POST` request to `models.litellm.cloud`, which is **not** an official BerriAI / LiteLLM domain + +## Who is affected + +You may be affected if **any** of the following are true: + +- You installed or upgraded LiteLLM via `pip` on **March 24, 2026**, between **10:39 UTC and 16:00 UTC** +- You ran `pip install litellm` without pinning a version and received **v1.82.7** or **v1.82.8** +- You built a Docker image during this window that included `pip install litellm` without a pinned version +- A dependency in your project pulled in LiteLLM as a transitive, unpinned dependency + (for example through AI agent frameworks, MCP servers, or LLM orchestration tools) + +You are **not** affected if any of the following are true: + +**LiteLLM AI Gateway/Proxy users:** Customers running the official LiteLLM Proxy Docker image were not impacted. That deployment path pins dependencies in requirements.txt and does not rely on the compromised PyPI packages. + +- You are using **LiteLLM Cloud** +- You are using the official LiteLLM AI Gateway Docker image: `ghcr.io/berriai/litellm` +- You are on **v1.82.6 or earlier** and did not upgrade during the affected window +- You installed LiteLLM from source via the GitHub repository, which was **not** compromised + + +### How to check if you are affected + + + + +```bash +pip show litellm +``` + + + +Go to the proxy base url, and check the version of the installed LiteLLM. + +![Proxy version check](../../img/security_update_march_2026/proxy_version.png) + + + +Scans all repositories in a GitHub organization for workflow jobs that installed the compromised versions. + +**Requirements:** Python 3 and `requests` (`pip install requests`). + +**Setup:** + +```bash +export GITHUB_TOKEN="your-github-pat" +``` + +**Run:** + +```bash +python find_litellm_github.py +``` + +Set the `ORG` variable in the script to your GitHub organization name. + +Both scripts default to scanning jobs from **today**. Adjust the `WINDOW_START` and `WINDOW_END` constants to cover **March 24, 2026** (the incident date) if running on a different day. + +
+View full script (find_litellm_github.py) + +```python +#!/usr/bin/env python3 +""" +Scan all GitHub Actions jobs in a GitHub org that ran between +0800-1244 UTC today and identify any that installed litellm 1.82.7 or 1.82.8. + +Adjust WINDOW_START / WINDOW_END to cover March 24, 2026 if running later. +""" + +import io +import os +import re +import sys +import zipfile +from concurrent.futures import ThreadPoolExecutor, as_completed +from datetime import datetime, timezone + +import requests + +GITHUB_URL = "https://api.github.com" +ORG = "your-org" # <-- set to your GitHub organization +TOKEN = os.environ.get("GITHUB_TOKEN", "") + +TODAY = datetime.now(timezone.utc).date() +WINDOW_START = datetime(TODAY.year, TODAY.month, TODAY.day, 8, 0, 0, tzinfo=timezone.utc) +WINDOW_END = datetime(TODAY.year, TODAY.month, TODAY.day, 12, 44, 0, tzinfo=timezone.utc) + +TARGET_VERSIONS = {"1.82.7", "1.82.8"} +VERSION_PATTERN = re.compile(r"litellm[=\-](\d+\.\d+\.\d+)", re.IGNORECASE) + +SESSION = requests.Session() +SESSION.headers.update({ + "Authorization": f"Bearer {TOKEN}", + "Accept": "application/vnd.github+json", + "X-GitHub-Api-Version": "2022-11-28", +}) + + +def get_paginated(url, params=None): + params = dict(params or {}) + params.setdefault("per_page", 100) + page = 1 + while True: + params["page"] = page + resp = SESSION.get(url, params=params, timeout=30) + if resp.status_code == 404: + return + resp.raise_for_status() + data = resp.json() + if isinstance(data, dict): + items = next((v for v in data.values() if isinstance(v, list)), []) + else: + items = data + if not items: + break + yield from items + if len(items) < params["per_page"]: + break + page += 1 + + +def parse_ts(ts_str): + if not ts_str: + return None + return datetime.fromisoformat(ts_str.replace("Z", "+00:00")) + + +def get_repos(): + repos = [] + for r in get_paginated(f"{GITHUB_URL}/orgs/{ORG}/repos", {"type": "all"}): + repos.append({"id": r["id"], "name": r["name"], "full_name": r["full_name"]}) + return repos + + +def get_runs_in_window(repo_full_name): + created_filter = ( + f"{WINDOW_START.strftime('%Y-%m-%dT%H:%M:%SZ')}" + f"..{WINDOW_END.strftime('%Y-%m-%dT%H:%M:%SZ')}" + ) + url = f"{GITHUB_URL}/repos/{repo_full_name}/actions/runs" + runs = [] + for run in get_paginated(url, {"created": created_filter, "per_page": 100}): + ts = parse_ts(run.get("run_started_at") or run.get("created_at")) + if ts and WINDOW_START <= ts <= WINDOW_END: + runs.append(run) + return runs + + +def get_jobs_for_run(repo_full_name, run_id): + url = f"{GITHUB_URL}/repos/{repo_full_name}/actions/runs/{run_id}/jobs" + jobs = [] + for job in get_paginated(url, {"filter": "all"}): + ts = parse_ts(job.get("started_at")) + if ts and WINDOW_START <= ts <= WINDOW_END: + jobs.append(job) + return jobs + + +def fetch_job_log(repo_full_name, job_id): + url = f"{GITHUB_URL}/repos/{repo_full_name}/actions/jobs/{job_id}/logs" + resp = SESSION.get(url, timeout=60, allow_redirects=True) + if resp.status_code in (403, 404, 410): + return "" + resp.raise_for_status() + + content_type = resp.headers.get("Content-Type", "") + if "zip" in content_type or resp.content[:2] == b"PK": + try: + with zipfile.ZipFile(io.BytesIO(resp.content)) as zf: + parts = [] + for name in sorted(zf.namelist()): + with zf.open(name) as f: + parts.append(f.read().decode("utf-8", errors="replace")) + return "\n".join(parts) + except zipfile.BadZipFile: + pass + return resp.text + + +def check_job(repo_full_name, job): + job_id = job["id"] + job_name = job["name"] + run_id = job["run_id"] + started = job.get("started_at", "") + + log_text = fetch_job_log(repo_full_name, job_id) + if not log_text: + return None + + found_versions = set() + context_lines = [] + for line in log_text.splitlines(): + m = VERSION_PATTERN.search(line) + if m: + ver = m.group(1) + if ver in TARGET_VERSIONS: + found_versions.add(ver) + context_lines.append(line.strip()) + + if not found_versions: + return None + + return { + "repo": repo_full_name, + "run_id": run_id, + "job_id": job_id, + "job_name": job_name, + "started_at": started, + "versions": sorted(found_versions), + "context": context_lines[:10], + "job_url": job.get("html_url", f"https://github.com/{repo_full_name}/actions/runs/{run_id}"), + } + + +def main(): + if not TOKEN: + print("ERROR: Set GITHUB_TOKEN environment variable.", file=sys.stderr) + sys.exit(1) + + print(f"Time window : {WINDOW_START.isoformat()} -> {WINDOW_END.isoformat()}") + print(f"Hunting for : litellm {', '.join(sorted(TARGET_VERSIONS))}") + print() + + print(f"Fetching repositories for org '{ORG}'...") + repos = get_repos() + print(f" Found {len(repos)} repositories") + print() + + jobs_to_check = [] + + print("Scanning workflow runs for time window...") + for repo in repos: + full_name = repo["full_name"] + try: + runs = get_runs_in_window(full_name) + except requests.HTTPError as e: + print(f" WARN: {full_name} - {e}", file=sys.stderr) + continue + if not runs: + continue + print(f" {full_name}: {len(runs)} run(s) in window") + for run in runs: + try: + jobs = get_jobs_for_run(full_name, run["id"]) + except requests.HTTPError as e: + print(f" WARN: run {run['id']} - {e}", file=sys.stderr) + continue + for job in jobs: + jobs_to_check.append((full_name, job)) + + total = len(jobs_to_check) + print(f"\nFetching logs for {total} job(s)...") + print() + + hits = [] + with ThreadPoolExecutor(max_workers=8) as pool: + futures = { + pool.submit(check_job, full_name, job): (full_name, job["id"]) + for full_name, job in jobs_to_check + } + done = 0 + for future in as_completed(futures): + done += 1 + full_name, jid = futures[future] + try: + result = future.result() + except Exception as e: + print(f" ERROR {full_name} job {jid}: {e}", file=sys.stderr) + continue + if result: + hits.append(result) + print( + f" [{done}/{total}] {full_name} job {jid}" + + (f" *** HIT: litellm {result['versions']} ***" if result else ""), + flush=True, + ) + + print() + print("=" * 72) + print(f"RESULTS: {len(hits)} job(s) installed litellm {' or '.join(sorted(TARGET_VERSIONS))}") + print("=" * 72) + + if not hits: + print("No matches found.") + return + + for h in sorted(hits, key=lambda x: x["started_at"]): + print() + print(f" Repo : {h['repo']}") + print(f" Job : {h['job_name']} (#{h['job_id']})") + print(f" Run ID : {h['run_id']}") + print(f" Started : {h['started_at']}") + print(f" Versions : litellm {', '.join(h['versions'])}") + print(f" URL : {h['job_url']}") + print(f" Log lines :") + for line in h["context"]: + print(f" {line}") + + +if __name__ == "__main__": + main() +``` + +
+ + + + +Scans all projects in a GitLab group (including subgroups) for CI/CD jobs that installed the compromised versions. + +**Requirements:** Python 3 and `requests` (`pip install requests`). + +**Setup:** + +```bash +export GITLAB_TOKEN="your-gitlab-pat" +``` + +**Run:** + +```bash +python find_litellm_jobs.py +``` + +Set the `GROUP_NAME` variable in the script to your GitLab group name. + +Both scripts default to scanning jobs from **today**. Adjust the `WINDOW_START` and `WINDOW_END` constants to cover **March 24, 2026** (the incident date) if running on a different day. + +
+View full script (find_litellm_jobs.py) + +```python +#!/usr/bin/env python3 +""" +Scan all GitLab CI/CD jobs in a GitLab group that ran between +0800-1244 UTC today and identify any that installed litellm 1.82.7 or 1.82.8. + +Adjust WINDOW_START / WINDOW_END to cover March 24, 2026 if running later. +""" + +import os +import re +import sys +from concurrent.futures import ThreadPoolExecutor, as_completed +from datetime import datetime, timezone + +import requests + +GITLAB_URL = "https://gitlab.com" +GROUP_NAME = "YourGroup" # <-- set to your GitLab group name +TOKEN = os.environ.get("GITLAB_TOKEN", "") + +TODAY = datetime.now(timezone.utc).date() +WINDOW_START = datetime(TODAY.year, TODAY.month, TODAY.day, 8, 0, 0, tzinfo=timezone.utc) +WINDOW_END = datetime(TODAY.year, TODAY.month, TODAY.day, 12, 44, 0, tzinfo=timezone.utc) + +TARGET_VERSIONS = {"1.82.7", "1.82.8"} +VERSION_PATTERN = re.compile(r"litellm[=\-](\d+\.\d+\.\d+)", re.IGNORECASE) + +HEADERS = {"PRIVATE-TOKEN": TOKEN} +SESSION = requests.Session() +SESSION.headers.update(HEADERS) + + +def get_paginated(url, params=None): + params = dict(params or {}) + params.setdefault("per_page", 100) + page = 1 + while True: + params["page"] = page + resp = SESSION.get(url, params=params, timeout=30) + resp.raise_for_status() + data = resp.json() + if not data: + break + yield from data + if len(data) < params["per_page"]: + break + page += 1 + + +def get_group_id(group_name): + resp = SESSION.get(f"{GITLAB_URL}/api/v4/groups/{group_name}", timeout=30) + resp.raise_for_status() + return resp.json()["id"] + + +def get_all_projects(group_id): + projects = [] + for p in get_paginated( + f"{GITLAB_URL}/api/v4/groups/{group_id}/projects", + {"include_subgroups": "true", "archived": "false"}, + ): + projects.append({"id": p["id"], "name": p["path_with_namespace"]}) + return projects + + +def parse_ts(ts_str): + if not ts_str: + return None + ts_str = ts_str.replace("Z", "+00:00") + return datetime.fromisoformat(ts_str) + + +def jobs_in_window(project_id): + matching = [] + url = f"{GITLAB_URL}/api/v4/projects/{project_id}/jobs" + params = {"per_page": 100, "scope[]": ["success", "failed", "canceled", "running"]} + + page = 1 + while True: + params["page"] = page + resp = SESSION.get(url, params=params, timeout=30) + if resp.status_code == 403: + return matching + resp.raise_for_status() + jobs = resp.json() + if not jobs: + break + + stop_early = False + for job in jobs: + ts = parse_ts(job.get("started_at") or job.get("created_at")) + if ts is None: + continue + if ts > WINDOW_END: + continue + if ts < WINDOW_START: + stop_early = True + continue + matching.append(job) + + if stop_early or len(jobs) < 100: + break + page += 1 + + return matching + + +def fetch_trace(project_id, job_id): + url = f"{GITLAB_URL}/api/v4/projects/{project_id}/jobs/{job_id}/trace" + resp = SESSION.get(url, timeout=60) + if resp.status_code in (403, 404): + return "" + resp.raise_for_status() + return resp.text + + +def check_job(project_name, project_id, job): + job_id = job["id"] + job_name = job["name"] + ref = job.get("ref", "") + started = job.get("started_at", job.get("created_at", "")) + + trace = fetch_trace(project_id, job_id) + if not trace: + return None + + found_versions = set() + for match in VERSION_PATTERN.finditer(trace): + ver = match.group(1) + if ver in TARGET_VERSIONS: + found_versions.add(ver) + + if not found_versions: + return None + + context_lines = [] + for line in trace.splitlines(): + if VERSION_PATTERN.search(line): + ver_match = VERSION_PATTERN.search(line) + if ver_match and ver_match.group(1) in TARGET_VERSIONS: + context_lines.append(line.strip()) + + return { + "project": project_name, + "project_id": project_id, + "job_id": job_id, + "job_name": job_name, + "ref": ref, + "started_at": started, + "versions": sorted(found_versions), + "context": context_lines[:10], + "job_url": f"{GITLAB_URL}/{project_name}/-/jobs/{job_id}", + } + + +def main(): + if not TOKEN: + print("ERROR: Set GITLAB_TOKEN environment variable.", file=sys.stderr) + sys.exit(1) + + print(f"Time window : {WINDOW_START.isoformat()} -> {WINDOW_END.isoformat()}") + print(f"Hunting for : litellm {', '.join(sorted(TARGET_VERSIONS))}") + print() + + print(f"Resolving group '{GROUP_NAME}'...") + group_id = get_group_id(GROUP_NAME) + + print("Fetching projects...") + projects = get_all_projects(group_id) + print(f" Found {len(projects)} projects") + print() + + all_jobs_to_check = [] + + print("Scanning job listings for time window...") + for proj in projects: + try: + jobs = jobs_in_window(proj["id"]) + except requests.HTTPError as e: + print(f" WARN: {proj['name']} - {e}", file=sys.stderr) + continue + if jobs: + print(f" {proj['name']}: {len(jobs)} job(s) in window") + for j in jobs: + all_jobs_to_check.append((proj["name"], proj["id"], j)) + + total = len(all_jobs_to_check) + print(f"\nFetching traces for {total} job(s)...") + print() + + hits = [] + with ThreadPoolExecutor(max_workers=10) as pool: + futures = { + pool.submit(check_job, pname, pid, job): (pname, job["id"]) + for pname, pid, job in all_jobs_to_check + } + done = 0 + for future in as_completed(futures): + done += 1 + pname, jid = futures[future] + try: + result = future.result() + except Exception as e: + print(f" ERROR checking {pname} job {jid}: {e}", file=sys.stderr) + continue + if result: + hits.append(result) + print(f" [{done}/{total}] checked {pname} job {jid}" + + (f" *** HIT: litellm {result['versions']} ***" if result else ""), + flush=True) + + print() + print("=" * 72) + print(f"RESULTS: {len(hits)} job(s) installed litellm {' or '.join(sorted(TARGET_VERSIONS))}") + print("=" * 72) + + if not hits: + print("No matches found.") + return + + for h in sorted(hits, key=lambda x: x["started_at"]): + print() + print(f" Project : {h['project']}") + print(f" Job : {h['job_name']} (#{h['job_id']})") + print(f" Branch/tag: {h['ref']}") + print(f" Started : {h['started_at']}") + print(f" Versions : litellm {', '.join(h['versions'])}") + print(f" URL : {h['job_url']}") + print(f" Log lines :") + for line in h["context"]: + print(f" {line}") + + +if __name__ == "__main__": + main() +``` + +
+ + + + +*CI/CD scripts contributed by the community ([original gist](https://gist.github.com/fryz/93ec8d4898ffe5b5ac5706a208823ef3)). Review before running.* + + +## Indicators of compromise (IoCs) + +Review affected systems for the following indicators: + +- `litellm_init.pth` present in your `site-packages` +- Outbound traffic or requests to `models.litellm[.]cloud` + This domain is **not** affiliated with LiteLLM +- Outbound traffic or requests to `checkmarx[.]zone` + This domain is **not** affiliated with LiteLLM + + +## Immediate actions for affected users + +If you installed or ran **v1.82.7** or **v1.82.8**, take the following actions immediately. + +### 1. Rotate all secrets + +Treat any credentials present on the affected systems as compromised, including: + +- API keys +- Cloud access keys +- Database passwords +- SSH keys +- Kubernetes tokens +- Any secrets stored in environment variables or configuration files + +### 2. Inspect your filesystem + +Check your `site-packages` directory for a file named `litellm_init.pth`: + +```bash +find /usr/lib/python3.13/site-packages/ -name "litellm_init.pth" +``` + +If present: + +- remove it immediately +- investigate the host for further compromise +- preserve relevant artifacts if your security team is performing forensics + +### 3. Audit version history + +Review your: + +- Local environments +- CI/CD pipelines +- Docker builds +- Deployment logs + +Confirm whether **v1.82.7** or **v1.82.8** was installed anywhere. + +Pin LiteLLM to a known safe version such as **v1.82.6 or earlier**, or to a later verified release once announced. + + +## Response and remediation + +The LiteLLM AI Gateway team has already taken the following steps: + +- Removed compromised packages from PyPI +- Rotated maintainer credentials and established new authorized maintainers +- Engaged Google's Mandiant security team to assist with forensic analysis of the build and publishing chain + + +## Verified safe versions + +We have audited every LiteLLM release published between v1.78.0 and v1.82.6 across both PyPI and Docker. Each artifact was verified by: + +1. Downloading the published artifact and computing its SHA-256 digest +2. Scanning for the known [indicators of compromise](#indicators-of-compromise-iocs) (IOCs) +3. Comparing the artifact contents against the corresponding Git commit in the BerriAI/litellm repository + +**All versions listed below are confirmed clean.** + + + + + + + + + + + + + + + +## Questions and support + +If you believe your systems may be affected, contact us immediately: + +- **Security:** `security@berri.ai` +- **Support:** `support@berri.ai` +- **Slack:** Reach out to the LiteLLM team directly + +For real-time updates, follow [LiteLLM (YC W23) on X](https://x.com/LiteLLM). + diff --git a/docs/my-website/blog/server_root_path/index.md b/docs/my-website/blog/server_root_path/index.md index d7925baf6b4..13b7365cc9e 100644 --- a/docs/my-website/blog/server_root_path/index.md +++ b/docs/my-website/blog/server_root_path/index.md @@ -3,17 +3,9 @@ slug: server-root-path-incident title: "Incident Report: SERVER_ROOT_PATH regression broke UI routing" date: 2026-02-21T10:00:00 authors: - - name: Yuneng Jiang - title: SWE @ LiteLLM (Full Stack) - url: https://www.linkedin.com/in/yunengjiang/ - - name: Ishaan Jaff - title: "CTO, LiteLLM" - url: https://www.linkedin.com/in/reffajnaahsi/ - image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg - - name: Krrish Dholakia - title: "CEO, LiteLLM" - url: https://www.linkedin.com/in/krish-d/ - image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg + - yuneng + - ishaan-alt + - krrish tags: [incident-report, ui, stability] hide_table_of_contents: false --- diff --git a/docs/my-website/blog/sub_millisecond_proxy_overhead/index.md b/docs/my-website/blog/sub_millisecond_proxy_overhead/index.md index 1857383363c..7f8ac086a21 100644 --- a/docs/my-website/blog/sub_millisecond_proxy_overhead/index.md +++ b/docs/my-website/blog/sub_millisecond_proxy_overhead/index.md @@ -3,18 +3,9 @@ slug: sub-millisecond-proxy-overhead title: "Achieving Sub-Millisecond Proxy Overhead" date: 2026-02-02T10:00:00 authors: - - name: Alexsander Hamir - title: "Performance Engineer, LiteLLM" - url: https://www.linkedin.com/in/alexsander-baptista/ - image_url: https://github.com/AlexsanderHamir.png - - name: Krrish Dholakia - title: "CEO, LiteLLM" - url: https://www.linkedin.com/in/krish-d/ - image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg - - name: Ishaan Jaff - title: "CTO, LiteLLM" - url: https://www.linkedin.com/in/reffajnaahsi/ - image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg + - alexsander + - krrish + - ishaan-alt description: "Our Q1 performance target and architectural direction for achieving sub-millisecond proxy overhead on modest hardware." tags: [performance, architecture] hide_table_of_contents: false @@ -32,6 +23,8 @@ Proxy overhead refers to the latency introduced by LiteLLM itself, independent o To measure it, we run the same workload directly against the provider and through LiteLLM at identical QPS (for example, 1,000 QPS) and compare the latency delta. To reduce noise, the load generator, LiteLLM, and a mock LLM endpoint all run on the same machine, ensuring the difference reflects proxy overhead rather than network latency. +{/* truncate */} + --- ## Where We're Coming From diff --git a/docs/my-website/blog/vanta_compliance_recertification/index.md b/docs/my-website/blog/vanta_compliance_recertification/index.md new file mode 100644 index 00000000000..d05c113967f --- /dev/null +++ b/docs/my-website/blog/vanta_compliance_recertification/index.md @@ -0,0 +1,18 @@ +--- +slug: vanta-compliance-recertification +title: "LiteLLM + Vanta: SOC 2 Type 2 and ISO 27001 Recertification" +date: 2026-03-30T10:00:00 +authors: + - krrish +description: "LiteLLM is partnering with Vanta on SOC 2 Type 2 and ISO 27001 recertification and engaging independent auditors for verification." +tags: [security, compliance] +hide_table_of_contents: true +--- + +![LiteLLM x Vanta SOC-2 Recertification](/img/blog/vanta_soc2_recertification.png) + +We are partnering with [Vanta](https://www.vanta.com/) to recertify LiteLLM's compliance for SOC 2 Type 2 and ISO 27001. + +As part of this process, we are also identifying independent auditors to validate and verify our compliance posture. + +This is part of our commitment to being the most secure and transparent AI Gateway possible. diff --git a/docs/my-website/blog/video_characters_litellm/index.md b/docs/my-website/blog/video_characters_litellm/index.md index 263a17d7191..a0f87385d81 100644 --- a/docs/my-website/blog/video_characters_litellm/index.md +++ b/docs/my-website/blog/video_characters_litellm/index.md @@ -3,18 +3,9 @@ slug: video_characters_api title: "New Video Characters, Edit and Extension API support" date: 2026-03-16T10:00:00 authors: - - name: Sameer Kankute - title: SWE @ LiteLLM - url: https://www.linkedin.com/in/sameer-kankute/ - image_url: https://pbs.twimg.com/profile_images/2001352686994907136/ONgNuSk5_400x400.jpg - - name: Krrish Dholakia - title: "CEO, LiteLLM" - url: https://www.linkedin.com/in/krish-d/ - image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg - - name: Ishaan Jaff - title: "CTO, LiteLLM" - url: https://www.linkedin.com/in/reffajnaahsi/ - image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg + - sameer + - krrish + - ishaan-alt description: "LiteLLM now supports creating, retrieving, and managing reusable video characters across multiple video generations." tags: [videos, characters, proxy, routing] hide_table_of_contents: false @@ -22,6 +13,8 @@ hide_table_of_contents: false LiteLLM now supoports videos character, edit and extension apis. +{/* truncate */} + ## What's New Four new endpoints for video character operations: @@ -125,4 +118,4 @@ Router knows exactly which deployment to use **Behind the scenes:** - Character ID format: `character_` - Metadata includes: provider, model_id, original_character_id -- Transparent to you - just use the ID, LiteLLM handles routing \ No newline at end of file +- Transparent to you - just use the ID, LiteLLM handles routing diff --git a/docs/my-website/blog/vllm_embeddings_incident/index.md b/docs/my-website/blog/vllm_embeddings_incident/index.md index a1ce8152857..26387e66dd0 100644 --- a/docs/my-website/blog/vllm_embeddings_incident/index.md +++ b/docs/my-website/blog/vllm_embeddings_incident/index.md @@ -3,18 +3,9 @@ slug: vllm-embeddings-incident title: "Incident Report: vLLM Embeddings Broken by encoding_format Parameter" date: 2026-02-18T10:00:00 authors: - - name: Sameer Kankute - title: SWE @ LiteLLM (LLM Translation) - url: https://www.linkedin.com/in/sameer-kankute/ - image_url: https://pbs.twimg.com/profile_images/2001352686994907136/ONgNuSk5_400x400.jpg - - name: Krrish Dholakia - title: "CEO, LiteLLM" - url: https://www.linkedin.com/in/krish-d/ - image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg - - name: Ishaan Jaff - title: "CTO, LiteLLM" - url: https://www.linkedin.com/in/reffajnaahsi/ - image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg + - sameer + - krrish + - ishaan-alt tags: [incident-report, embeddings, vllm] hide_table_of_contents: false --- diff --git a/docs/my-website/docs/anthropic_unified/index.md b/docs/my-website/docs/anthropic_unified/index.md index 9981547ce1f..f8a50e14da5 100644 --- a/docs/my-website/docs/anthropic_unified/index.md +++ b/docs/my-website/docs/anthropic_unified/index.md @@ -506,12 +506,15 @@ Request body will be in the Anthropic messages API format. **litellm follows the A system prompt providing context or specific instructions to the model. - **temperature** (number): Controls randomness in the model's responses. Valid range: `0 < temperature < 1`. -- **thinking** (object): +- **thinking** (object): Configuration for enabling extended thinking. If enabled, it includes: - - **budget_tokens** (integer): + - **budget_tokens** (integer): Minimum of 1024 tokens (and less than `max_tokens`). - - **type** (enum): + - **type** (enum): E.g., `"enabled"`. + - **summary** (string, optional): + Enables the summary style for thinking blocks. Possible values: `"auto"`, `"concise"`, `"detailed"`, `"disabled"`. + When routing to non-Anthropic providers (e.g., `openai/gpt-5.1`), the `summary` value is preserved and forwarded to the downstream API. - **tool_choice** (object): Instructs how the model should utilize any provided tools. - **tools** (array of objects): diff --git a/docs/my-website/docs/completion/prompt_caching.md b/docs/my-website/docs/completion/prompt_caching.md index dca5f5c0cff..402c7b9f4c7 100644 --- a/docs/my-website/docs/completion/prompt_caching.md +++ b/docs/my-website/docs/completion/prompt_caching.md @@ -6,6 +6,8 @@ import TabItem from '@theme/TabItem'; Supported Providers: - OpenAI (`openai/`) - Anthropic API (`anthropic/`) +- Google AI Studio (`gemini/`) +- Vertex AI (`vertex_ai/`, `vertex_ai_beta/`) - Bedrock (`bedrock/`, `bedrock/invoke/`, `bedrock/converse`) ([All models bedrock supports prompt caching on](https://docs.aws.amazon.com/bedrock/latest/userguide/prompt-caching.html)) - Deepseek API (`deepseek/`) @@ -257,7 +259,7 @@ Anthropic charges for cache writes. Specify the content to cache with `"cache_control": {"type": "ephemeral"}`. -If you pass that in for any other llm provider, it will be ignored. +This same format also works for [Gemini / Vertex AI](#google-ai-studio--vertex-ai-gemini-example). For other providers, it will be ignored. @@ -356,6 +358,208 @@ print(response.usage) +### Google AI Studio / Vertex AI (Gemini) Example + +Use the same Anthropic-style `cache_control` format — LiteLLM automatically translates it to Google's [context caching API](https://ai.google.dev/api/caching). + +**How it works under the hood:** +1. Messages with `cache_control` are separated and sent to Google's `cachedContents` API +2. The cached content ID is then passed as `cachedContent` in the Gemini request body +3. Works across all three providers: `gemini/` (Google AI Studio), `vertex_ai/`, and `vertex_ai_beta/` +4. Requires a minimum of **1024 tokens** in the cached content — below that, caching is silently skipped + + + + +```python +from litellm import completion +import os + +os.environ["GEMINI_API_KEY"] = "" + +response = completion( + model="gemini/gemini-2.5-flash", + messages=[ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are an AI assistant tasked with analyzing legal documents.", + }, + { + "type": "text", + "text": "Here is the full text of a complex legal agreement" * 400, + "cache_control": {"type": "ephemeral"}, + }, + ], + }, + { + "role": "user", + "content": "what are the key terms and conditions in this agreement?", + }, + ], +) + +print(response.usage) +``` + + + +1. Setup config.yaml + +```yaml +model_list: + - model_name: gemini-2.5-flash + litellm_params: + model: gemini/gemini-2.5-flash + api_key: os.environ/GEMINI_API_KEY +``` + +2. Start proxy + +```bash +litellm --config /path/to/config.yaml +``` + +3. Test it! + +```python +from openai import OpenAI + +client = OpenAI( + api_key="LITELLM_PROXY_KEY", # sk-1234 + base_url="LITELLM_PROXY_BASE", # http://0.0.0.0:4000 +) + +response = client.chat.completions.create( + model="gemini-2.5-flash", + messages=[ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are an AI assistant tasked with analyzing legal documents.", + }, + { + "type": "text", + "text": "Here is the full text of a complex legal agreement" * 400, + "cache_control": {"type": "ephemeral"}, + }, + ], + }, + { + "role": "user", + "content": "what are the key terms and conditions in this agreement?", + }, + ], +) + +print(response.usage) +``` + + + + +#### Vertex AI + +For Vertex AI, use `vertex_ai/` prefix: + + + + +```python +from litellm import completion + +response = completion( + model="vertex_ai/gemini-2.5-flash", + vertex_project="my-gcp-project", + vertex_location="us-central1", + messages=[ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are an AI assistant tasked with analyzing legal documents.", + }, + { + "type": "text", + "text": "Here is the full text of a complex legal agreement" * 400, + "cache_control": {"type": "ephemeral"}, + }, + ], + }, + { + "role": "user", + "content": "what are the key terms and conditions in this agreement?", + }, + ], +) + +print(response.usage) +``` + + + +1. Setup config.yaml + +```yaml +model_list: + - model_name: gemini-2.5-flash + litellm_params: + model: vertex_ai/gemini-2.5-flash + vertex_project: my-gcp-project + vertex_location: us-central1 +``` + +2. Start proxy + +```bash +litellm --config /path/to/config.yaml +``` + +3. Test it! + +```python +from openai import OpenAI + +client = OpenAI( + api_key="LITELLM_PROXY_KEY", # sk-1234 + base_url="LITELLM_PROXY_BASE", # http://0.0.0.0:4000 +) + +response = client.chat.completions.create( + model="gemini-2.5-flash", + messages=[ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are an AI assistant tasked with analyzing legal documents.", + }, + { + "type": "text", + "text": "Here is the full text of a complex legal agreement" * 400, + "cache_control": {"type": "ephemeral"}, + }, + ], + }, + { + "role": "user", + "content": "what are the key terms and conditions in this agreement?", + }, + ], +) + +print(response.usage) +``` + + + + ### Deepeek Example Works the same as OpenAI. diff --git a/docs/my-website/docs/data_security.md b/docs/my-website/docs/data_security.md index 2c4b1247e2b..d93d17aa0de 100644 --- a/docs/my-website/docs/data_security.md +++ b/docs/my-website/docs/data_security.md @@ -128,8 +128,6 @@ We'll review all reports promptly. Note that we don't currently offer a bug boun Legal Entity Name: Berrie AI Incorporated -Company Phone Number: 7708783106 - Point of contact email address for security incidents: krrish@berri.ai Point of contact email address for general security-related questions: krrish@berri.ai diff --git a/docs/my-website/docs/debugging/local_debugging.md b/docs/my-website/docs/debugging/local_debugging.md index 8a56d6c34a0..53daa4e366b 100644 --- a/docs/my-website/docs/debugging/local_debugging.md +++ b/docs/my-website/docs/debugging/local_debugging.md @@ -67,6 +67,6 @@ response = completion("command-nightly", messages, logger_fn=my_custom_logging_f ## Still Seeing Issues? -Text us @ +17708783106 or Join the [Discord](https://discord.com/invite/wuPM9dRgDw). +Join the [Discord](https://discord.com/invite/wuPM9dRgDw). We promise to help you in `lite`ning speed ❤️ diff --git a/docs/my-website/docs/enterprise.md b/docs/my-website/docs/enterprise.md index 6dccf7ff4e7..a3fc9e38b6e 100644 --- a/docs/my-website/docs/enterprise.md +++ b/docs/my-website/docs/enterprise.md @@ -4,7 +4,7 @@ import Image from '@theme/IdealImage'; :::info - ✨ SSO is free for up to 5 users. After that, an enterprise license is required. [Get Started with Enterprise here](https://www.litellm.ai/enterprise) -- Who is Enterprise for? Companies giving access to 100+ users **OR** 10+ AI use-cases. If you're not sure, [get in touch with us](https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions) to discuss your needs. +- Who is Enterprise for? Companies giving access to 100+ users **OR** 10+ AI use-cases. If you're not sure, [get in touch with us](https://enterprise.litellm.ai/demo) to discuss your needs. ::: For companies that need SSO, user management and professional support for LiteLLM Proxy @@ -36,7 +36,7 @@ Manage Yourself - you can deploy our Docker Image or build a custom image from o ### What’s the cost of the Self-Managed Enterprise edition? -Self-Managed Enterprise deployments require our team to understand your exact needs. [Get in touch with us to learn more](https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions) +Self-Managed Enterprise deployments require our team to understand your exact needs. [Get in touch with us to learn more](https://enterprise.litellm.ai/demo) ### How does deployment with Enterprise License work? @@ -106,7 +106,7 @@ Professional Support can assist with LLM/Provider integrations, deployment, upgr Pricing is based on usage. We can figure out a price that works for your team, on the call. -[**Contact Us to learn more**](https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions) +[**Contact Us to learn more**](https://enterprise.litellm.ai/demo) diff --git a/docs/my-website/docs/fine_tuning.md b/docs/my-website/docs/fine_tuning.md index d0bd98a76f9..52e96f28688 100644 --- a/docs/my-website/docs/fine_tuning.md +++ b/docs/my-website/docs/fine_tuning.md @@ -6,7 +6,7 @@ import TabItem from '@theme/TabItem'; :::info -This is an Enterprise only endpoint [Get Started with Enterprise here](https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions) +This is an Enterprise only endpoint [Get Started with Enterprise here](https://enterprise.litellm.ai/demo) ::: diff --git a/docs/my-website/docs/guides/index.md b/docs/my-website/docs/guides/index.md new file mode 100644 index 00000000000..1641600dab2 --- /dev/null +++ b/docs/my-website/docs/guides/index.md @@ -0,0 +1,78 @@ +--- +title: Guides +sidebar_label: Overview +--- + +import NavigationCards from '@site/src/components/NavigationCards'; + +**Guides** are focused references organized by the job you are trying to do with LiteLLM: make requests, use tools, handle media, manage context, or operate the gateway safely. + +> New to LiteLLM or not sure whether you need the SDK or Gateway path first? Start at [Learn →](/docs/learn) + +--- + +## Build With LiteLLM + + + +--- + +## Operate & Extend + + diff --git a/docs/my-website/docs/index.md b/docs/my-website/docs/index.md index ba605e316d3..ca63c9e39ff 100644 --- a/docs/my-website/docs/index.md +++ b/docs/my-website/docs/index.md @@ -1,689 +1,462 @@ +--- +id: index +title: Getting Started +sidebar_label: Quickstart +--- + import Tabs from '@theme/Tabs'; import TabItem from '@theme/TabItem'; +import NavigationCards from '@site/src/components/NavigationCards'; +import Image from '@theme/IdealImage'; -# LiteLLM - Getting Started + -https://github.com/BerriAI/litellm +**LiteLLM** is an open-source library that gives you a single, unified interface to call 100+ LLMs — OpenAI, Anthropic, Vertex AI, Bedrock, and more — using the OpenAI format. -## **Call 100+ LLMs using the OpenAI Input/Output Format** +- Call any provider using the same `completion()` interface — no re-learning the API for each one +- Consistent output format regardless of which provider or model you use +- Built-in retry / fallback logic across multiple deployments via the [Router](./routing.md) +- Self-hosted [LLM Gateway (Proxy)](./simple_proxy) with virtual keys, cost tracking, and an admin UI -- Translate inputs to provider's endpoints (`/chat/completions`, `/responses`, `/embeddings`, `/images`, `/audio`, `/batches`, and more) -- [Consistent output](https://docs.litellm.ai/docs/supported_endpoints) - same response format regardless of which provider you use -- Retry/fallback logic across multiple deployments (e.g. Azure/OpenAI) - [Router](https://docs.litellm.ai/docs/routing) -- Track spend & set budgets per project [LiteLLM Proxy Server](https://docs.litellm.ai/docs/simple_proxy) +[![PyPI](https://img.shields.io/pypi/v/litellm.svg)](https://pypi.org/project/litellm/) +[![GitHub Stars](https://img.shields.io/github/stars/BerriAI/litellm?style=social)](https://github.com/BerriAI/litellm) -## How to use LiteLLM +--- -You can use LiteLLM through either the Proxy Server or Python SDK. Both gives you a unified interface to access multiple LLMs (100+ LLMs). Choose the option that best fits your needs: - -
Stripeimage Google ADK Greptile OpenHands
- - - - - - - - - - - - - - - - - - - - - - - - -
LiteLLM Proxy ServerLiteLLM Python SDK
Use CaseCentral service (LLM Gateway) to access multiple LLMsUse LiteLLM directly in your Python code
Who Uses It?Gen AI Enablement / ML Platform TeamsDevelopers building LLM projects
Key Features• Centralized API gateway with authentication & authorization
• Multi-tenant cost tracking and spend management per project/user
• Per-project customization (logging, guardrails, caching)
• Virtual keys for secure access control
• Admin dashboard UI for monitoring and management		• Direct Python library integration in your codebase
• Router with retry/fallback logic across multiple deployments (e.g. Azure/OpenAI) - Router
• Application-level load balancing and cost tracking
• Exception handling with OpenAI-compatible errors
• Observability callbacks (Lunary, MLflow, Langfuse, etc.)
- - -## **LiteLLM Python SDK** - -### Basic usage - - - Open In Colab - +## Installation ```shell pip install litellm ``` - - - -```python -from litellm import completion -import os - -## set ENV variables -os.environ["OPENAI_API_KEY"] = "your-api-key" - -response = completion( - model="openai/gpt-4o", - messages=[{ "content": "Hello, how are you?","role": "user"}] -) -``` - - - - -```python -from litellm import completion -import os - -## set ENV variables -os.environ["ANTHROPIC_API_KEY"] = "your-api-key" - -response = completion( - model="anthropic/claude-3-sonnet-20240229", - messages=[{ "content": "Hello, how are you?","role": "user"}] -) -``` - - - - -```python -from litellm import completion -import os - -## set ENV variables -os.environ["XAI_API_KEY"] = "your-api-key" - -response = completion( - model="xai/grok-2-latest", - messages=[{ "content": "Hello, how are you?","role": "user"}] -) -``` - - - -```python -from litellm import completion -import os - -# auth: run 'gcloud auth application-default' -os.environ["VERTEXAI_PROJECT"] = "hardy-device-386718" -os.environ["VERTEXAI_LOCATION"] = "us-central1" - -response = completion( - model="vertex_ai/gemini-1.5-pro", - messages=[{ "content": "Hello, how are you?","role": "user"}] -) -``` - - - - - -```python -from litellm import completion -import os - -## set ENV variables -os.environ["NVIDIA_NIM_API_KEY"] = "nvidia_api_key" -os.environ["NVIDIA_NIM_API_BASE"] = "nvidia_nim_endpoint_url" - -response = completion( - model="nvidia_nim/", - messages=[{ "content": "Hello, how are you?","role": "user"}] -) -``` - - - - - -```python -from litellm import completion -import os - -os.environ["HUGGINGFACE_API_KEY"] = "huggingface_api_key" - -# e.g. Call 'WizardLM/WizardCoder-Python-34B-V1.0' hosted on HF Inference endpoints -response = completion( - model="huggingface/WizardLM/WizardCoder-Python-34B-V1.0", - messages=[{ "content": "Hello, how are you?","role": "user"}], - api_base="https://my-endpoint.huggingface.cloud" -) - -print(response) -``` - - - - - -```python -from litellm import completion -import os - -## set ENV variables -os.environ["AZURE_API_KEY"] = "" -os.environ["AZURE_API_BASE"] = "" -os.environ["AZURE_API_VERSION"] = "" - -# azure call -response = completion( - "azure/", - messages = [{ "content": "Hello, how are you?","role": "user"}] -) -``` - - - - - -```python -from litellm import completion - -response = completion( - model="ollama/llama2", - messages = [{ "content": "Hello, how are you?","role": "user"}], - api_base="http://localhost:11434" -) -``` - - - - -```python -from litellm import completion -import os - -## set ENV variables -os.environ["OPENROUTER_API_KEY"] = "openrouter_api_key" - -response = completion( - model="openrouter/google/palm-2-chat-bison", - messages = [{ "content": "Hello, how are you?","role": "user"}], -) -``` - - - - -```python -from litellm import completion -import os - -## set ENV variables. Visit https://novita.ai/settings/key-management to get your API key -os.environ["NOVITA_API_KEY"] = "novita-api-key" - -response = completion( - model="novita/deepseek/deepseek-r1", - messages=[{ "content": "Hello, how are you?","role": "user"}] -) -``` - - - - - -```python -from litellm import completion -import os - -## set ENV variables. Visit https://vercel.com/docs/ai-gateway#using-the-ai-gateway-with-an-api-key for insturctions on obtaining a key -os.environ["VERCEL_AI_GATEWAY_API_KEY"] = "your-vercel-api-key" - -response = completion( - model="vercel_ai_gateway/openai/gpt-4o", - messages=[{ "content": "Hello, how are you?","role": "user"}] -) -``` - - - - - -### Response Format (OpenAI Chat Completions Format) - -```json -{ - "id": "chatcmpl-565d891b-a42e-4c39-8d14-82a1f5208885", - "created": 1734366691, - "model": "gpt-4o-2024-08-06", - "object": "chat.completion", - "system_fingerprint": null, - "choices": [ - { - "finish_reason": "stop", - "index": 0, - "message": { - "content": "Hello! As an AI language model, I don't have feelings, but I'm operating properly and ready to assist you with any questions or tasks you may have. How can I help you today?", - "role": "assistant", - "tool_calls": null, - "function_call": null - } - } - ], - "usage": { - "completion_tokens": 43, - "prompt_tokens": 13, - "total_tokens": 56, - "completion_tokens_details": null, - "prompt_tokens_details": { - "audio_tokens": null, - "cached_tokens": 0 - }, - "cache_creation_input_tokens": 0, - "cache_read_input_tokens": 0 - } -} -``` - -### Streaming -Set `stream=True` in the `completion` args. - - - - -```python -from litellm import completion -import os - -## set ENV variables -os.environ["OPENAI_API_KEY"] = "your-api-key" - -response = completion( - model="openai/gpt-4o", - messages=[{ "content": "Hello, how are you?","role": "user"}], - stream=True, -) -``` - - - - -```python -from litellm import completion -import os - -## set ENV variables -os.environ["ANTHROPIC_API_KEY"] = "your-api-key" - -response = completion( - model="anthropic/claude-3-sonnet-20240229", - messages=[{ "content": "Hello, how are you?","role": "user"}], - stream=True, -) -``` - - - - -```python -from litellm import completion -import os - -## set ENV variables -os.environ["XAI_API_KEY"] = "your-api-key" - -response = completion( - model="xai/grok-2-latest", - messages=[{ "content": "Hello, how are you?","role": "user"}], - stream=True, -) -``` - - - -```python -from litellm import completion -import os - -# auth: run 'gcloud auth application-default' -os.environ["VERTEX_PROJECT"] = "hardy-device-386718" -os.environ["VERTEX_LOCATION"] = "us-central1" - -response = completion( - model="vertex_ai/gemini-1.5-pro", - messages=[{ "content": "Hello, how are you?","role": "user"}], - stream=True, -) -``` - - - - - -```python -from litellm import completion -import os - -## set ENV variables -os.environ["NVIDIA_NIM_API_KEY"] = "nvidia_api_key" -os.environ["NVIDIA_NIM_API_BASE"] = "nvidia_nim_endpoint_url" - -response = completion( - model="nvidia_nim/", - messages=[{ "content": "Hello, how are you?","role": "user"}] - stream=True, -) -``` - - - - -```python -from litellm import completion -import os - -os.environ["HUGGINGFACE_API_KEY"] = "huggingface_api_key" - -# e.g. Call 'WizardLM/WizardCoder-Python-34B-V1.0' hosted on HF Inference endpoints -response = completion( - model="huggingface/WizardLM/WizardCoder-Python-34B-V1.0", - messages=[{ "content": "Hello, how are you?","role": "user"}], - api_base="https://my-endpoint.huggingface.cloud", - stream=True, -) - -print(response) -``` - - - - - -```python -from litellm import completion -import os - -## set ENV variables -os.environ["AZURE_API_KEY"] = "" -os.environ["AZURE_API_BASE"] = "" -os.environ["AZURE_API_VERSION"] = "" - -# azure call -response = completion( - "azure/", - messages = [{ "content": "Hello, how are you?","role": "user"}], - stream=True, -) -``` - - - - - -```python -from litellm import completion - -response = completion( - model="ollama/llama2", - messages = [{ "content": "Hello, how are you?","role": "user"}], - api_base="http://localhost:11434", - stream=True, -) -``` - - - - -```python -from litellm import completion -import os - -## set ENV variables -os.environ["OPENROUTER_API_KEY"] = "openrouter_api_key" - -response = completion( - model="openrouter/google/palm-2-chat-bison", - messages = [{ "content": "Hello, how are you?","role": "user"}], - stream=True, -) -``` - - - - -```python -from litellm import completion -import os - -## set ENV variables. Visit https://novita.ai/settings/key-management to get your API key -os.environ["NOVITA_API_KEY"] = "novita_api_key" - -response = completion( - model="novita/deepseek/deepseek-r1", - messages = [{ "content": "Hello, how are you?","role": "user"}], - stream=True, -) -``` - - - - - -```python -from litellm import completion -import os - -## set ENV variables. Visit https://vercel.com/docs/ai-gateway#using-the-ai-gateway-with-an-api-key for insturctions on obtaining a key -os.environ["VERCEL_AI_GATEWAY_API_KEY"] = "your-vercel-api-key" - -response = completion( - model="vercel_ai_gateway/openai/gpt-4o", - messages = [{ "content": "Hello, how are you?","role": "user"}], - stream=True, -) -``` - - - - - -### Streaming Response Format (OpenAI Format) - -```json -{ - "id": "chatcmpl-2be06597-eb60-4c70-9ec5-8cd2ab1b4697", - "created": 1734366925, - "model": "claude-3-sonnet-20240229", - "object": "chat.completion.chunk", - "system_fingerprint": null, - "choices": [ - { - "finish_reason": null, - "index": 0, - "delta": { - "content": "Hello", - "role": "assistant", - "function_call": null, - "tool_calls": null, - "audio": null - }, - "logprobs": null - } - ] -} -``` - -### Exception handling - -LiteLLM maps exceptions across all supported providers to the OpenAI exceptions. All our exceptions inherit from OpenAI's exception types, so any error-handling you have for that, should work out of the box with LiteLLM. - -```python -import litellm -from litellm import completion -import os - -os.environ["ANTHROPIC_API_KEY"] = "bad-key" -try: - completion(model="anthropic/claude-instant-1", messages=[{"role": "user", "content": "Hey, how's it going?"}]) -except litellm.AuthenticationError as e: - # Thrown when the API key is invalid - print(f"Authentication failed: {e}") -except litellm.RateLimitError as e: - # Thrown when you've exceeded your rate limit - print(f"Rate limited: {e}") -except litellm.APIError as e: - # Thrown for general API errors - print(f"API error: {e}") -``` -### See How LiteLLM Transforms Your Requests - -Want to understand how LiteLLM parses and normalizes your LLM API requests? Use the `/utils/transform_request` endpoint to see exactly how your request is transformed internally. - -You can try it out now directly on our Demo App! -Go to the [LiteLLM API docs for transform_request](https://litellm-api.up.railway.app/#/llm%20utils/transform_request_utils_transform_request_post) - -LiteLLM will show you the normalized, provider-agnostic version of your request. This is useful for debugging, learning, and understanding how LiteLLM handles different providers and options. - - -### Logging Observability - Log LLM Input/Output ([Docs](https://docs.litellm.ai/docs/observability/callbacks)) -LiteLLM exposes pre defined callbacks to send data to Lunary, MLflow, Langfuse, Helicone, Promptlayer, Traceloop, Slack - -```python -from litellm import completion - -## set env variables for logging tools (API key set up is not required when using MLflow) -os.environ["LUNARY_PUBLIC_KEY"] = "your-lunary-public-key" # get your public key at https://app.lunary.ai/settings -os.environ["HELICONE_API_KEY"] = "your-helicone-key" -os.environ["LANGFUSE_PUBLIC_KEY"] = "" -os.environ["LANGFUSE_SECRET_KEY"] = "" - -os.environ["OPENAI_API_KEY"] - -# set callbacks -litellm.success_callback = ["lunary", "mlflow", "langfuse", "helicone"] # log input/output to lunary, mlflow, langfuse, helicone - -#openai call -response = completion(model="gpt-3.5-turbo", messages=[{"role": "user", "content": "Hi 👋 - i'm openai"}]) -``` - -### Track Costs, Usage, Latency for streaming -Use a callback function for this - more info on custom callbacks: https://docs.litellm.ai/docs/observability/custom_callback - -```python -import litellm - -# track_cost_callback -def track_cost_callback( - kwargs, # kwargs to completion - completion_response, # response from completion - start_time, end_time # start/end time -): - try: - response_cost = kwargs.get("response_cost", 0) - print("streaming response_cost", response_cost) - except: - pass -# set callback -litellm.success_callback = [track_cost_callback] # set custom callback function - -# litellm.completion() call -response = completion( - model="gpt-3.5-turbo", - messages=[ - { - "role": "user", - "content": "Hi 👋 - i'm openai" - } - ], - stream=True -) -``` - -## **LiteLLM Proxy Server (LLM Gateway)** - -Track spend across multiple projects/people - -![ui_3](https://github.com/BerriAI/litellm/assets/29436595/47c97d5e-b9be-4839-b28c-43d7f4f10033) - -The proxy provides: - -1. [Hooks for auth](https://docs.litellm.ai/docs/proxy/virtual_keys#custom-auth) -2. [Hooks for logging](https://docs.litellm.ai/docs/proxy/logging#step-1---create-your-custom-litellm-callback-class) -3. [Cost tracking](https://docs.litellm.ai/docs/proxy/virtual_keys#tracking-spend) -4. [Rate Limiting](https://docs.litellm.ai/docs/proxy/users#set-rate-limits) - -### 📖 Proxy Endpoints - [Swagger Docs](https://litellm-api.up.railway.app/) - -Go here for a complete tutorial with keys + rate limits - [**here**](./proxy/docker_quick_start.md) - -### Quick Start Proxy - CLI +To run the full Proxy Server (LLM Gateway): ```shell pip install 'litellm[proxy]' ``` -#### Step 1: Start litellm proxy +--- + +## Quick Start + +Make your first LLM call using the provider of your choice: + - +```python +from litellm import completion +import os -```shell -$ litellm --model huggingface/bigcode/starcoder +os.environ["OPENAI_API_KEY"] = "your-api-key" -#INFO: Proxy running on http://0.0.0.0:4000 +response = completion( + model="openai/gpt-4o", + messages=[{"role": "user", "content": "Hello, how are you?"}] +) +print(response.choices[0].message.content) ``` + - +```python +from litellm import completion +import os +os.environ["ANTHROPIC_API_KEY"] = "your-api-key" -Step 1. CREATE config.yaml +response = completion( + model="anthropic/claude-3-5-sonnet-20241022", + messages=[{"role": "user", "content": "Hello, how are you?"}] +) +print(response.choices[0].message.content) +``` -Example `litellm_config.yaml` + + -```yaml +```python +from litellm import completion +import os + +# auth: run 'gcloud auth application-default login' +os.environ["VERTEXAI_PROJECT"] = "your-project-id" +os.environ["VERTEXAI_LOCATION"] = "us-central1" + +response = completion( + model="vertex_ai/gemini-1.5-pro", + messages=[{"role": "user", "content": "Hello, how are you?"}] +) +print(response.choices[0].message.content) +``` + + + + +```python +from litellm import completion +import os + +os.environ["AWS_ACCESS_KEY_ID"] = "your-key" +os.environ["AWS_SECRET_ACCESS_KEY"] = "your-secret" +os.environ["AWS_REGION_NAME"] = "us-east-1" + +response = completion( + model="bedrock/anthropic.claude-3-5-sonnet-20241022-v2:0", + messages=[{"role": "user", "content": "Hello, how are you?"}] +) +print(response.choices[0].message.content) +``` + + + + +```python +from litellm import completion + +response = completion( + model="ollama/llama3", + messages=[{"role": "user", "content": "Hello, how are you?"}], + api_base="http://localhost:11434" +) +print(response.choices[0].message.content) +``` + + + + +```python +from litellm import completion +import os + +os.environ["AZURE_API_KEY"] = "your-key" +os.environ["AZURE_API_BASE"] = "https://your-resource.openai.azure.com" +os.environ["AZURE_API_VERSION"] = "2024-02-01" + +response = completion( + model="azure/your-deployment-name", + messages=[{"role": "user", "content": "Hello, how are you?"}] +) +print(response.choices[0].message.content) +``` + + + + +Every response follows the OpenAI Chat Completions format, regardless of provider. ✅ + +### Response Format + +Non-streaming responses return a `ModelResponse` object: + +```json +{ + "id": "chatcmpl-abc123", + "object": "chat.completion", + "created": 1677858242, + "model": "gpt-4o", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": "Hello! I'm doing well, thanks for asking." + }, + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 13, + "completion_tokens": 12, + "total_tokens": 25 + } +} +``` + +Streaming responses (`stream=True`) yield `ModelResponseStream` chunks: + +```json +{ + "id": "chatcmpl-abc123", + "object": "chat.completion.chunk", + "created": 1677858242, + "model": "gpt-4o", + "choices": [ + { + "index": 0, + "delta": { + "role": "assistant", + "content": "Hello" + }, + "finish_reason": null + } + ] +} +``` + +📖 [Full output format reference →](./completion/output) + +:::tip Open in Colab + +Open In Colab + +::: + +--- + +## New to LiteLLM? + +**Want to get started fast?** Head to [Tutorials](/docs/tutorials) for step-by-step walkthroughs — AI coding tools, agent SDKs, proxy setup, and more. + +**Need to understand a specific feature?** Check [Guides](/docs/guides) for streaming, function calling, prompt caching, and other how-tos. + +--- + +## Choose Your Path + + + +--- + +## LiteLLM Python SDK + +### Streaming + +Add `stream=True` to receive chunks as they are generated: + +```python +from litellm import completion +import os + +os.environ["OPENAI_API_KEY"] = "your-api-key" + +for chunk in completion( + model="openai/gpt-4o", + messages=[{"role": "user", "content": "Write a short poem"}], + stream=True, +): + print(chunk.choices[0].delta.content or "", end="") +``` + +### Exception Handling + +LiteLLM maps every provider's errors to the OpenAI exception types — your existing error handling works out of the box: + +```python +import litellm + +try: + litellm.completion( + model="anthropic/claude-instant-1", + messages=[{"role": "user", "content": "Hey!"}] + ) +except litellm.AuthenticationError as e: + print(f"Bad API key: {e}") +except litellm.RateLimitError as e: + print(f"Rate limited: {e}") +except litellm.APIError as e: + print(f"API error: {e}") +``` + +### Logging & Observability + +Send input/output to Langfuse, MLflow, Helicone, Lunary, and more with a single line: + +```python +import litellm + +litellm.success_callback = ["langfuse", "mlflow", "helicone"] + +response = litellm.completion( + model="gpt-4o", + messages=[{"role": "user", "content": "Hi!"}] +) +``` + +📖 [See all observability integrations →](/docs/observability/agentops_integration) + +### Track Costs & Usage + +Use a callback to capture cost per response: + +```python +import litellm + +def track_cost(kwargs, completion_response, start_time, end_time): + print("Cost:", kwargs.get("response_cost", 0)) + +litellm.success_callback = [track_cost] + +litellm.completion( + model="gpt-4o", + messages=[{"role": "user", "content": "Hello!"}], + stream=True +) +``` + +📖 [Custom callback docs →](./observability/custom_callback) + +--- + +## LiteLLM Proxy Server (LLM Gateway) + +The proxy is a self-hosted OpenAI-compatible gateway. Any client that works with OpenAI works with the proxy — no code changes needed. + +![LiteLLM Proxy Dashboard](https://github.com/BerriAI/litellm/assets/29436595/47c97d5e-b9be-4839-b28c-43d7f4f10033) + +#### Step 1 — Start the proxy + + + + +```shell +litellm --model huggingface/bigcode/starcoder +# Proxy running on http://0.0.0.0:4000 +``` + + + + +```yaml title="litellm_config.yaml" model_list: - model_name: gpt-3.5-turbo litellm_params: - model: azure/ - api_base: os.environ/AZURE_API_BASE # runs os.getenv("AZURE_API_BASE") - api_key: os.environ/AZURE_API_KEY # runs os.getenv("AZURE_API_KEY") + model: azure/your-deployment + api_base: os.environ/AZURE_API_BASE + api_key: os.environ/AZURE_API_KEY api_version: "2023-07-01-preview" ``` -Step 2. RUN Docker Image - ```shell docker run \ - -v $(pwd)/litellm_config.yaml:/app/config.yaml \ - -e AZURE_API_KEY=d6*********** \ - -e AZURE_API_BASE=https://openai-***********/ \ - -p 4000:4000 \ - docker.litellm.ai/berriai/litellm:main-latest \ - --config /app/config.yaml --detailed_debug + -v $(pwd)/litellm_config.yaml:/app/config.yaml \ + -e AZURE_API_KEY=your-key \ + -e AZURE_API_BASE=https://your-resource.openai.azure.com/ \ + -p 4000:4000 \ + docker.litellm.ai/berriai/litellm:main-latest \ + --config /app/config.yaml --detailed_debug ``` - -#### Step 2: Make ChatCompletions Request to Proxy +#### Step 2 — Call it with the OpenAI client ```python -import openai # openai v1.0.0+ -client = openai.OpenAI(api_key="anything",base_url="http://0.0.0.0:4000") # set proxy to base_url -# request sent to model set on litellm proxy, `litellm --model` -response = client.chat.completions.create(model="gpt-3.5-turbo", messages = [ - { - "role": "user", - "content": "this is a test request, write a short poem" - } -]) +import openai -print(response) +client = openai.OpenAI(api_key="anything", base_url="http://0.0.0.0:4000") + +response = client.chat.completions.create( + model="gpt-3.5-turbo", + messages=[{"role": "user", "content": "Write a short poem"}] +) +print(response.choices[0].message.content) ``` -## More details +👉 [Full proxy quickstart with Docker →](./proxy/docker_quick_start) -- [exception mapping](./exception_mapping.md) -- [retries + model fallbacks for completion()](./completion/reliable_completions.md) -- [proxy virtual keys & spend management](./proxy/virtual_keys.md) -- [E2E Tutorial for LiteLLM Proxy Server](./proxy/docker_quick_start.md) +:::tip Debugging tool +Use [**`/utils/transform_request`**](./utils/transform_request) to inspect exactly what LiteLLM sends to any provider — useful for debugging prompt formatting, header issues, and provider-specific parameters. +::: + +🔗 [Interactive API explorer (Swagger) →](https://litellm-api.up.railway.app/) + +--- + +## Agent & MCP Gateway + +LiteLLM is a unified gateway for **LLMs, agents, and MCP** — you don't need a separate agent or MCP gateway. One endpoint for 100+ models, A2A agents, and MCP tools. + + + +--- + +## What to Explore Next + + diff --git a/docs/my-website/docs/integrations/index.md b/docs/my-website/docs/integrations/index.md index 95c922cce89..0ad934d5b41 100644 --- a/docs/my-website/docs/integrations/index.md +++ b/docs/my-website/docs/integrations/index.md @@ -1,18 +1,336 @@ -# Integrations +--- +title: Integrations +sidebar_label: Overview +--- + +import NavigationCards from '@site/src/components/NavigationCards'; This section covers integrations with various tools and services that can be used with LiteLLM (either Proxy or SDK). -## AI Agent Frameworks -- **[Letta](./letta.md)** - Build stateful LLM agents with persistent memory using LiteLLM Proxy +--- -## Development Tools -- **[OpenWebUI](../tutorials/openweb_ui.md)** - Self-hosted ChatGPT-style interface +## Observability -## Observability & Monitoring -- **[Langfuse](../observability/langfuse_integration.md)** - LLM observability and analytics -- **[Prometheus](../proxy/prometheus.md)** - Metrics collection and monitoring -- **[PagerDuty](../proxy/pagerduty.md)** - Incident response and alerting -- **[Datadog](../observability/datadog.md)** +Track, debug, and analyze LLM calls with observability platforms. + -Click into each section to learn more about the integrations. \ No newline at end of file +[View all observability integrations →](/docs/integrations/observability_integrations) + +--- + +## Alerting & Monitoring + +Set up alerts, metrics collection, and infrastructure monitoring. + + + +--- + +## Guardrail Providers + +Add safety and content filtering to LLM calls. + + + +[View all guardrail providers →](/docs/guardrail_providers) + +--- + +## Policies + +Define and enforce usage policies across your LLM deployment. + + + +--- + +## AI Tools + +Connect LiteLLM to AI-powered coding and productivity tools. + + + +--- + +## Agent SDKs + +Use LiteLLM with agent frameworks and SDKs. + + + +--- + +## Prompt Management + +Manage, version, and deploy prompts. + + + +--- + +## Manage with AI Agents + +Use AI agents to manage your LiteLLM deployment — create users, teams, keys, models, and more via natural language. + + diff --git a/docs/my-website/docs/integrations/letta.md b/docs/my-website/docs/integrations/letta.md index 2afb82542f2..9711999df5e 100644 --- a/docs/my-website/docs/integrations/letta.md +++ b/docs/my-website/docs/integrations/letta.md @@ -920,9 +920,9 @@ for model in models: ## Resources -- [Letta Documentation](https://docs.letta.ai/) -- [LiteLLM Proxy Documentation](../proxy/quick_start.md) -- [LiteLLM SDK Documentation](../completion/input.md) -- [Function Calling Guide](../completion/function_call.md) -- [Observability Setup](../observability/langfuse_integration.md) -- [Router Configuration](../routing.md) \ No newline at end of file +- [Letta Documentation](https://docs.letta.com/) +- [LiteLLM Proxy Documentation](/docs/simple_proxy) +- [LiteLLM SDK Documentation](/docs/#litellm-python-sdk) +- [Function Calling Guide](/docs/completion/function_call) +- [Observability Setup](/docs/integrations/observability_integrations) +- [Router Configuration](/docs/routing) \ No newline at end of file diff --git a/docs/my-website/docs/integrations/observability_index.md b/docs/my-website/docs/integrations/observability_index.md new file mode 100644 index 00000000000..8ab83950cdc --- /dev/null +++ b/docs/my-website/docs/integrations/observability_index.md @@ -0,0 +1,28 @@ +--- +title: Observability +sidebar_label: Overview +slug: observability_integrations +--- + +Track, debug, and analyze LLM calls with observability platforms. + +import NavigationCards from '@site/src/components/NavigationCards'; + +## Observability Integrations + + + +[View all observability integrations →](/docs/observability/callbacks) diff --git a/docs/my-website/docs/integrations/websearch_interception.md b/docs/my-website/docs/integrations/websearch_interception.md index 0c5d8927013..bc5e8ec0b39 100644 --- a/docs/my-website/docs/integrations/websearch_interception.md +++ b/docs/my-website/docs/integrations/websearch_interception.md @@ -375,7 +375,7 @@ search_tools: - [Search Providers](../search/index.md) - Detailed search provider setup - [Claude Code WebSearch](../tutorials/claude_code_websearch.md) - Using with Claude Code - [Tool Calling](../completion/function_call.md) - General tool calling documentation -- [Callbacks](./custom_callback.md) - Custom callback documentation +- [Callbacks](../observability/custom_callback.md) - Custom callback documentation ## Technical Details diff --git a/docs/my-website/docs/learn/gateway_quickstart.md b/docs/my-website/docs/learn/gateway_quickstart.md new file mode 100644 index 00000000000..acec259758c --- /dev/null +++ b/docs/my-website/docs/learn/gateway_quickstart.md @@ -0,0 +1,174 @@ +--- +title: Gateway Quickstart +sidebar_label: Gateway Quickstart +description: Start LiteLLM Gateway, add models and keys, then connect applications and SDKs to one shared endpoint. +--- + +import NavigationCards from '@site/src/components/NavigationCards'; + +Use this path if you need one shared OpenAI-compatible endpoint for a team or platform. + +If you need a Docker or database-first setup, use the [Docker + Database tutorial](/docs/proxy/docker_quick_start). Otherwise, use the steps below to get to a working request fast. + +## 1. Install The Gateway + +```bash +pip install 'litellm[proxy]' +``` + +## 2. Set One Provider Key + +```bash +export OPENAI_API_KEY="your-api-key" +``` + +## 3. Create `config.yaml` + +```yaml +model_list: + - model_name: gpt-4o-mini + litellm_params: + model: openai/gpt-4o-mini + api_key: os.environ/OPENAI_API_KEY + +general_settings: + master_key: sk-1234 +``` + +## 4. Start The Gateway + +```bash +litellm --config config.yaml +``` + +You should see the proxy start on `http://0.0.0.0:4000`. + +## 5. Send Your First Request + +```bash +curl -X POST 'http://0.0.0.0:4000/chat/completions' \ + -H 'Content-Type: application/json' \ + -H 'Authorization: Bearer sk-1234' \ + -d '{ + "model": "gpt-4o-mini", + "messages": [ + {"role": "user", "content": "Hello from LiteLLM Gateway"} + ] + }' +``` + +## 6. Check The Response + +If the request succeeds, the proxy returns `200 OK` with an OpenAI-style response. + +The assistant text will be in: + +```json +choices[0].message.content +``` + +If your gateway is routing to OpenAI, a real response can look like this: + +```json +{ + "id": "chatcmpl-abc123", + "created": 1677858242, + "model": "gpt-4o-mini-2024-07-18", + "object": "chat.completion", + "system_fingerprint": "fp_406d6473f8", + "choices": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "role": "assistant", + "content": "Hello! How can I assist you today?", + "tool_calls": null, + "function_call": null, + "annotations": [] + } + } + ], + "usage": { + "completion_tokens": 9, + "prompt_tokens": 13, + "total_tokens": 22, + "completion_tokens_details": { + "accepted_prediction_tokens": 0, + "audio_tokens": 0, + "reasoning_tokens": 0, + "rejected_prediction_tokens": 0 + }, + "prompt_tokens_details": { + "audio_tokens": 0, + "cached_tokens": 0 + } + }, + "service_tier": "default" +} +``` + +`id`, `created`, the resolved model version, token counts, and message text will vary by request. Other providers may return a smaller or slightly different set of fields, but `choices[0].message.content` is the main field to read. + +## 7. Add Keys And The UI + +If you need virtual keys, spend tracking, or the admin UI, add a database next. + +- Add `database_url` under `general_settings` +- Use [Virtual keys](/docs/proxy/virtual_keys) for key creation and budgets +- Use [Admin UI](/docs/proxy/ui) to manage models and keys +- Use the [Docker + Database tutorial](/docs/proxy/docker_quick_start) if you want a fuller setup + +## 8. Pick Your Next Step + + + +## When To Use The SDK Path Instead + +If you only need to call models from one application and do not need centralized auth or shared infrastructure, start with the [SDK Quickstart](/docs/learn/sdk_quickstart) instead. diff --git a/docs/my-website/docs/learn/index.md b/docs/my-website/docs/learn/index.md new file mode 100644 index 00000000000..018aec5af00 --- /dev/null +++ b/docs/my-website/docs/learn/index.md @@ -0,0 +1,117 @@ +--- +title: Learn LiteLLM +sidebar_label: Learn +slug: /learn +--- + +import NavigationCards from '@site/src/components/NavigationCards'; + +LiteLLM gives you one OpenAI-compatible interface for 100+ LLM providers. Start with the path that matches your setup. + +--- + +## Start Here + +Pick one path first. + + + +--- + +## Common Tasks + +Jump to a specific task. + + + +--- + +## Docs Map + +Use these when you already know the type of doc you want. + + + +Not sure where to start? Use [SDK Quickstart](/docs/learn/sdk_quickstart) for app code or [Gateway Quickstart](/docs/learn/gateway_quickstart) for shared infrastructure. diff --git a/docs/my-website/docs/learn/sdk_quickstart.md b/docs/my-website/docs/learn/sdk_quickstart.md new file mode 100644 index 00000000000..0fb8c3f02a5 --- /dev/null +++ b/docs/my-website/docs/learn/sdk_quickstart.md @@ -0,0 +1,174 @@ +--- +title: SDK Quickstart +sidebar_label: SDK Quickstart +description: Make your first LiteLLM SDK call, then jump to the right docs for the next feature you need. +--- + +import NavigationCards from '@site/src/components/NavigationCards'; + +Use this path if you are integrating LiteLLM directly into application code. + +## 1. Install LiteLLM + +```bash +pip install litellm==1.82.6 +``` + +## 2. Set Provider Credentials + +Start with one provider and set its environment variables. + +- OpenAI: `OPENAI_API_KEY` +- Anthropic: `ANTHROPIC_API_KEY` +- Azure OpenAI: `AZURE_API_KEY`, `AZURE_API_BASE`, `AZURE_API_VERSION` +- Bedrock: standard AWS credentials +- Vertex AI: `VERTEXAI_PROJECT`, `VERTEXAI_LOCATION` + +If you have not picked a provider yet, browse [all supported providers](/docs/providers). + +## 3. Make Your First Call + +```python +from litellm import completion +import os + +os.environ["OPENAI_API_KEY"] = "your-api-key" + +response = completion( + model="openai/gpt-4o", + messages=[{"role": "user", "content": "Hello, how are you?"}], +) + +print(response.choices[0].message.content) +``` + +## 4. Check The Response + +The line below: + +```python +print(response.choices[0].message.content) +``` + +prints the assistant text, for example: + +```text +Hello! I'm doing well, thanks for asking. +``` + +If you print the full object with: + +```python +print(response) +``` + +you will see a Python `ModelResponse(...)` object. For an OpenAI-backed model, it can look like this: + +```python +ModelResponse( + id='chatcmpl-abc123', + created=1773782130, + model='gpt-4o-2024-08-06', + object='chat.completion', + system_fingerprint='fp_4ff89bf575', + choices=[ + Choices( + finish_reason='stop', + index=0, + message=Message( + content="Hello! I'm just a program, but I'm here to help you. How can I assist you today?", + role='assistant', + tool_calls=None, + function_call=None, + provider_specific_fields={'refusal': None}, + annotations=[] + ), + provider_specific_fields={} + ) + ], + usage=Usage( + completion_tokens=21, + prompt_tokens=13, + total_tokens=34, + completion_tokens_details=CompletionTokensDetailsWrapper(...), + prompt_tokens_details=PromptTokensDetailsWrapper(...) + ), + service_tier='default' +) +``` + +The same response follows an OpenAI-style shape. Conceptually, it looks like this: + +```json +{ + "id": "chatcmpl-abc123", + "object": "chat.completion", + "created": 1677858242, + "model": "gpt-4o", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": "Hello! I'm doing well, thanks for asking." + }, + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 13, + "completion_tokens": 12, + "total_tokens": 25 + } +} +``` + +`id`, `created`, token counts, and message text will vary by request. + +If you call an OpenAI-backed model, you may also see extra fields such as `system_fingerprint`, `service_tier`, `tool_calls`, `function_call`, `annotations`, `provider_specific_fields`, and detailed token usage. For the full output reference, see [completion output](/docs/completion/output). + +Need more provider examples? See the main [Getting Started](/docs/#quick-start) page. + +## 5. Pick Your Next Step + + + +## When To Use Gateway Instead + +Use LiteLLM Gateway if you need centralized auth, virtual keys, spend tracking, shared logging, or one OpenAI-compatible endpoint for multiple apps. + +[Go to Gateway Quickstart →](/docs/learn/gateway_quickstart) diff --git a/docs/my-website/docs/load_test_advanced.md b/docs/my-website/docs/load_test_advanced.md index d35b5f74784..d7bc35e74e0 100644 --- a/docs/my-website/docs/load_test_advanced.md +++ b/docs/my-website/docs/load_test_advanced.md @@ -11,7 +11,7 @@ Tutorial on how to get to 1K+ RPS with LiteLLM Proxy on locust - [Github releases](https://github.com/BerriAI/litellm/releases) - [litellm docker containers](https://github.com/BerriAI/litellm/pkgs/container/litellm) - [litellm database docker container](https://github.com/BerriAI/litellm/pkgs/container/litellm-database) -- [ ] Ensure you're following **ALL** [best practices for production](./proxy/production_setup.md) +- [ ] Ensure you're following **ALL** [best practices for production](./proxy/prod.md) - [ ] Locust - Ensure you're Locust instance can create 1K+ requests per second - 👉 You can use our **[maintained locust instance here](https://locust-load-tester-production.up.railway.app/)** - If you're self hosting locust @@ -222,4 +222,4 @@ class MyUser(HttpUser): def on_start(self): self.api_key = os.getenv('API_KEY', 'sk-1234') self.client.headers.update({'Authorization': f'Bearer {self.api_key}'}) -``` \ No newline at end of file +``` diff --git a/docs/my-website/docs/migration.md b/docs/my-website/docs/migration.md index e1af07d4684..fda1155905d 100644 --- a/docs/my-website/docs/migration.md +++ b/docs/my-website/docs/migration.md @@ -31,5 +31,4 @@ When we have breaking changes (i.e. going from 1.x.x to 2.x.x), we will document **How can we communicate changes better?** Tell us - [Discord](https://discord.com/invite/wuPM9dRgDw) -- Email (krrish@berri.ai/ishaan@berri.ai) -- Text us (+17708783106) +- Email (support@berri.ai) diff --git a/docs/my-website/docs/observability/arize_integration.md b/docs/my-website/docs/observability/arize_integration.md index b3ccf98ea3b..4486fb2b718 100644 --- a/docs/my-website/docs/observability/arize_integration.md +++ b/docs/my-website/docs/observability/arize_integration.md @@ -194,5 +194,4 @@ print(response) - [Schedule Demo 👋](https://calendly.com/d/4mp-gd3-k5k/berriai-1-1-onboarding-litellm-hosted-version) - [Community Discord 💭](https://discord.gg/wuPM9dRgDw) -- Our numbers 📞 +1 (770) 8783-106 / +1 (412) 618-6238 - Our emails ✉️ ishaan@berri.ai / krrish@berri.ai diff --git a/docs/my-website/docs/observability/gcs_bucket_integration.md b/docs/my-website/docs/observability/gcs_bucket_integration.md index 69b956950e5..5f8d42508ae 100644 --- a/docs/my-website/docs/observability/gcs_bucket_integration.md +++ b/docs/my-website/docs/observability/gcs_bucket_integration.md @@ -6,7 +6,7 @@ Log LLM Logs to [Google Cloud Storage Buckets](https://cloud.google.com/storage? :::info -✨ This is an Enterprise only feature [Get Started with Enterprise here](https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions) +✨ This is an Enterprise only feature [Get Started with Enterprise here](https://enterprise.litellm.ai/demo) ::: @@ -79,5 +79,4 @@ curl --location 'http://0.0.0.0:4000/chat/completions' \ - [Schedule Demo 👋](https://calendly.com/d/4mp-gd3-k5k/berriai-1-1-onboarding-litellm-hosted-version) - [Community Discord 💭](https://discord.gg/wuPM9dRgDw) -- Our numbers 📞 +1 (770) 8783-106 / ‭+1 (412) 618-6238‬ - Our emails ✉️ ishaan@berri.ai / krrish@berri.ai diff --git a/docs/my-website/docs/observability/langfuse_integration.md b/docs/my-website/docs/observability/langfuse_integration.md index d3c5a44d481..32849ebdb92 100644 --- a/docs/my-website/docs/observability/langfuse_integration.md +++ b/docs/my-website/docs/observability/langfuse_integration.md @@ -342,5 +342,4 @@ Be aware that if you are continuing an existing trace, and you set `update_trace - [Schedule Demo 👋](https://calendly.com/d/4mp-gd3-k5k/berriai-1-1-onboarding-litellm-hosted-version) - [Community Discord 💭](https://discord.gg/wuPM9dRgDw) -- Our numbers 📞 +1 (770) 8783-106 / ‭+1 (412) 618-6238‬ - Our emails ✉️ ishaan@berri.ai / krrish@berri.ai diff --git a/docs/my-website/docs/observability/langfuse_otel_integration.md b/docs/my-website/docs/observability/langfuse_otel_integration.md index b4c9a2bd1ad..79ad2f6f75d 100644 --- a/docs/my-website/docs/observability/langfuse_otel_integration.md +++ b/docs/my-website/docs/observability/langfuse_otel_integration.md @@ -83,6 +83,9 @@ os.environ["LANGFUSE_OTEL_HOST"] = "https://cloud.langfuse.com" # EU region # Or use self-hosted instance # os.environ["LANGFUSE_OTEL_HOST"] = "https://my-langfuse.company.com" +# Optional: Ignore otel context propagation to prevent parent-child relationships with spans from other providers +# os.environ["OTEL_IGNORE_CONTEXT_PROPAGATION"] = "true" + litellm.callbacks = ["langfuse_otel"] ``` @@ -124,6 +127,9 @@ export LANGFUSE_PUBLIC_KEY="pk-lf-..." export LANGFUSE_SECRET_KEY="sk-lf-..." export LANGFUSE_OTEL_HOST="https://us.cloud.langfuse.com" # Default US region # export LANGFUSE_OTEL_HOST="https://otel.my-langfuse.company.com" # custom OTEL endpoint + +# Optional: Ignore otel context propagation to prevent parent-child relationships with spans from other providers +# export OTEL_IGNORE_CONTEXT_PROPAGATION="true" ``` 2. Setup config.yaml diff --git a/docs/my-website/docs/observability/langsmith_integration.md b/docs/my-website/docs/observability/langsmith_integration.md index cada4122b20..bf867319de8 100644 --- a/docs/my-website/docs/observability/langsmith_integration.md +++ b/docs/my-website/docs/observability/langsmith_integration.md @@ -225,5 +225,4 @@ environment_variables: - [Schedule Demo 👋](https://calendly.com/d/4mp-gd3-k5k/berriai-1-1-onboarding-litellm-hosted-version) - [Community Discord 💭](https://discord.gg/wuPM9dRgDw) -- Our numbers 📞 +1 (770) 8783-106 / ‭+1 (412) 618-6238‬ - Our emails ✉️ ishaan@berri.ai / krrish@berri.ai diff --git a/docs/my-website/docs/observability/logfire_integration.md b/docs/my-website/docs/observability/logfire_integration.md index a1bd43a4bc4..00652c0f1a1 100644 --- a/docs/my-website/docs/observability/logfire_integration.md +++ b/docs/my-website/docs/observability/logfire_integration.md @@ -63,5 +63,4 @@ response = litellm.completion( - [Schedule Demo 👋](https://calendly.com/d/4mp-gd3-k5k/berriai-1-1-onboarding-litellm-hosted-version) - [Community Discord 💭](https://discord.gg/wuPM9dRgDw) -- Our numbers 📞 +1 (770) 8783-106 / ‭+1 (412) 618-6238‬ - Our emails ✉️ ishaan@berri.ai / krrish@berri.ai diff --git a/docs/my-website/docs/observability/lunary_integration.md b/docs/my-website/docs/observability/lunary_integration.md index 8d28321c807..4f1dca9d4af 100644 --- a/docs/my-website/docs/observability/lunary_integration.md +++ b/docs/my-website/docs/observability/lunary_integration.md @@ -176,5 +176,4 @@ You can find more details about the different ways of making requests to the Lit - [Schedule Demo 👋](https://calendly.com/d/4mp-gd3-k5k/berriai-1-1-onboarding-litellm-hosted-version) - [Community Discord 💭](https://discord.gg/wuPM9dRgDw) -- Our numbers 📞 +1 (770) 8783-106 / ‭+1 (412) 618-6238‬ - Our emails ✉️ ishaan@berri.ai / krrish@berri.ai diff --git a/docs/my-website/docs/observability/opik_integration.md b/docs/my-website/docs/observability/opik_integration.md index d28c46f0b4b..5b5cbe0f185 100644 --- a/docs/my-website/docs/observability/opik_integration.md +++ b/docs/my-website/docs/observability/opik_integration.md @@ -261,5 +261,4 @@ All requests made with this key will automatically be tracked in the "TestProjec - [Schedule Demo 👋](https://calendly.com/d/4mp-gd3-k5k/berriai-1-1-onboarding-litellm-hosted-version) - [Community Discord 💭](https://discord.gg/wuPM9dRgDw) -- Our numbers 📞 +1 (770) 8783-106 / ‭+1 (412) 618-6238‬ - Our emails ✉️ ishaan@berri.ai / krrish@berri.ai diff --git a/docs/my-website/docs/observability/phoenix_integration.md b/docs/my-website/docs/observability/phoenix_integration.md index 191f1f8044a..67ba815557e 100644 --- a/docs/my-website/docs/observability/phoenix_integration.md +++ b/docs/my-website/docs/observability/phoenix_integration.md @@ -127,5 +127,4 @@ Depending on which Phoenix Cloud version or deployment you are using, you should - [Schedule Demo 👋](https://calendly.com/d/4mp-gd3-k5k/berriai-1-1-onboarding-litellm-hosted-version) - [Community Discord 💭](https://discord.gg/wuPM9dRgDw) -- Our numbers 📞 +1 (770) 8783-106 / ‭+1 (412) 618-6238‬ - Our emails ✉️ ishaan@berri.ai / krrish@berri.ai diff --git a/docs/my-website/docs/observability/promptlayer_integration.md b/docs/my-website/docs/observability/promptlayer_integration.md index 7f62a316972..9462e755f74 100644 --- a/docs/my-website/docs/observability/promptlayer_integration.md +++ b/docs/my-website/docs/observability/promptlayer_integration.md @@ -84,5 +84,4 @@ Credits to [Nick Bradford](https://github.com/nsbradford), from [Vim-GPT](https: - [Schedule Demo 👋](https://calendly.com/d/4mp-gd3-k5k/berriai-1-1-onboarding-litellm-hosted-version) - [Community Discord 💭](https://discord.gg/wuPM9dRgDw) -- Our numbers 📞 +1 (770) 8783-106 / ‭+1 (412) 618-6238‬ - Our emails ✉️ ishaan@berri.ai / krrish@berri.ai \ No newline at end of file diff --git a/docs/my-website/docs/observability/slack_integration.md b/docs/my-website/docs/observability/slack_integration.md index 0ca7f616683..468d8b5945c 100644 --- a/docs/my-website/docs/observability/slack_integration.md +++ b/docs/my-website/docs/observability/slack_integration.md @@ -101,5 +101,4 @@ response = litellm.completion( - [Schedule Demo 👋](https://calendly.com/d/4mp-gd3-k5k/berriai-1-1-onboarding-litellm-hosted-version) - [Community Discord 💭](https://discord.gg/wuPM9dRgDw) -- Our numbers 📞 +1 (770) 8783-106 / ‭+1 (412) 618-6238‬ - Our emails ✉️ ishaan@berri.ai / krrish@berri.ai diff --git a/docs/my-website/docs/observability/sumologic_integration.md b/docs/my-website/docs/observability/sumologic_integration.md index c30ee94dad4..87e20ca57ed 100644 --- a/docs/my-website/docs/observability/sumologic_integration.md +++ b/docs/my-website/docs/observability/sumologic_integration.md @@ -328,5 +328,4 @@ If you get authentication errors, regenerate the HTTP Source URL in Sumo Logic: - [Schedule Demo 👋](https://calendly.com/d/4mp-gd3-k5k/berriai-1-1-onboarding-litellm-hosted-version) - [Community Discord 💭](https://discord.gg/wuPM9dRgDw) -- Our numbers 📞 +1 (770) 8783-106 / ‭+1 (412) 618-6238‬ - Our emails ✉️ ishaan@berri.ai / krrish@berri.ai diff --git a/docs/my-website/docs/observability/supabase_integration.md b/docs/my-website/docs/observability/supabase_integration.md index fd3f1c3d5a0..c29871d752f 100644 --- a/docs/my-website/docs/observability/supabase_integration.md +++ b/docs/my-website/docs/observability/supabase_integration.md @@ -105,5 +105,4 @@ litellm.modify_integration("supabase",{"table_name": "litellm_logs"}) - [Schedule Demo 👋](https://calendly.com/d/4mp-gd3-k5k/berriai-1-1-onboarding-litellm-hosted-version) - [Community Discord 💭](https://discord.gg/wuPM9dRgDw) -- Our numbers 📞 +1 (770) 8783-106 / ‭+1 (412) 618-6238‬ - Our emails ✉️ ishaan@berri.ai / krrish@berri.ai diff --git a/docs/my-website/docs/observability/wandb_integration.md b/docs/my-website/docs/observability/wandb_integration.md index 37057f43db5..3c1a3363957 100644 --- a/docs/my-website/docs/observability/wandb_integration.md +++ b/docs/my-website/docs/observability/wandb_integration.md @@ -57,5 +57,4 @@ response = litellm.completion( - [Schedule Demo 👋](https://calendly.com/d/4mp-gd3-k5k/berriai-1-1-onboarding-litellm-hosted-version) - [Community Discord 💭](https://discord.gg/wuPM9dRgDw) -- Our numbers 📞 +1 (770) 8783-106 / ‭+1 (412) 618-6238‬ - Our emails ✉️ ishaan@berri.ai / krrish@berri.ai \ No newline at end of file diff --git a/docs/my-website/docs/prompt_management.md b/docs/my-website/docs/prompt_management.md new file mode 100644 index 00000000000..c4e606674b1 --- /dev/null +++ b/docs/my-website/docs/prompt_management.md @@ -0,0 +1,48 @@ +--- +title: Prompt Management with Responses API +--- + +# Prompt Management with Responses API + +Use LiteLLM Prompt Management with `/v1/responses` by passing `prompt_id` and optional `prompt_variables`. + +## Basic Usage + +```bash +curl -X POST "http://localhost:4000/v1/responses" \ + -H "Authorization: Bearer sk-1234" \ + -H "Content-Type: application/json" \ + -d '{ + "model": "gpt-4o", + "prompt_id": "my-responses-prompt", + "prompt_variables": {"topic": "large language models"}, + "input": [] + }' +``` + +## Multi-turn Follow-up in `input` + +To send follow-up turns in one request, pass message history in `input`. + +```bash +curl -X POST "http://localhost:4000/v1/responses" \ + -H "Authorization: Bearer sk-1234" \ + -H "Content-Type: application/json" \ + -d '{ + "model": "gpt-4o", + "prompt_id": "my-responses-prompt", + "prompt_variables": {"topic": "large language models"}, + "input": [ + {"role": "user", "content": "Topic is LLMs. Start short."}, + {"role": "assistant", "content": "Sure, go ahead."}, + {"role": "user", "content": "Now give me 3 bullets and include pricing caveat."} + ] + }' +``` + +## Notes + +- Prompt template messages are merged with your `input` messages. +- Prompt variable substitution applies to prompt message content. +- Tool call payload fields are not substituted by prompt variables. +- For follow-ups with `previous_response_id`, include `prompt_id` again if you want prompt management applied on that turn. diff --git a/docs/my-website/docs/providers/azure/azure.md b/docs/my-website/docs/providers/azure/azure.md index 12ddc1bd98e..682f263c108 100644 --- a/docs/my-website/docs/providers/azure/azure.md +++ b/docs/my-website/docs/providers/azure/azure.md @@ -1032,7 +1032,7 @@ print("list_batches_response=", list_batches_response) -### [Health Check Azure Batch models](./proxy/health.md#batch-models-azure-only) +### [Health Check Azure Batch models](../../proxy/health.md#batch-models-azure-only) ### [BETA] Loadbalance Multiple Azure Deployments diff --git a/docs/my-website/docs/providers/azure/azure_anthropic.md b/docs/my-website/docs/providers/azure/azure_anthropic.md index 4c722b30397..e7cd8fffbf0 100644 --- a/docs/my-website/docs/providers/azure/azure_anthropic.md +++ b/docs/my-website/docs/providers/azure/azure_anthropic.md @@ -372,7 +372,6 @@ response = completion( ## Related Documentation -- [Anthropic Provider Documentation](./anthropic.md) - For standard Anthropic API usage +- [Anthropic Provider Documentation](../anthropic.md) - For standard Anthropic API usage - [Azure OpenAI Documentation](./azure.md) - For Azure OpenAI models -- [Azure Authentication Guide](../secret_managers/azure_key_vault.md) - For Azure AD token setup - +- [Azure Authentication Guide](../../secret_managers/azure_key_vault.md) - For Azure AD token setup diff --git a/docs/my-website/docs/providers/gemini.md b/docs/my-website/docs/providers/gemini.md index 0aaf3d5ae81..87ab5ad40f4 100644 --- a/docs/my-website/docs/providers/gemini.md +++ b/docs/my-website/docs/providers/gemini.md @@ -11,6 +11,7 @@ import TabItem from '@theme/TabItem'; | Provider Doc | [Google AI Studio ↗](https://aistudio.google.com/) | | API Endpoint for Provider | https://generativelanguage.googleapis.com | | Supported OpenAI Endpoints | `/chat/completions`, [`/embeddings`](../embedding/supported_embedding#gemini-ai-embedding-models), `/completions`, [`/videos`](./gemini/videos.md), [`/images/edits`](../image_edits.md) | +| Lyria (music) | [Cost map & notes](./gemini/music.md) | | Pass-through Endpoint | [Supported](../pass_through/google_ai_studio.md) |
@@ -54,6 +55,7 @@ response = completion( - stream - tools - tool_choice +- include_server_side_tool_invocations - functions - response_format - n @@ -856,7 +858,112 @@ curl -X POST 'http://0.0.0.0:4000/chat/completions' \ -### URL Context +### Context Circulation (Server-Side Tool Combination) + +Context circulation allows Gemini 3+ models to combine **built-in tools** (like Google Search) with **your custom functions** in the same request. Without it, Gemini returns an error if you try to use both. + +When enabled, Gemini can execute Google Search server-side, use those results to decide whether to call your custom functions, and return the full chain of reasoning. + +**How it works:** +1. You pass `include_server_side_tool_invocations=True` along with both Google Search and your function tools +2. Gemini executes server-side tools internally and returns `toolCall`/`toolResponse` parts alongside any `functionCall` parts +3. LiteLLM extracts the server-side invocations into `provider_specific_fields["server_side_tool_invocations"]` +4. On subsequent turns, include the full assistant message in your conversation history — LiteLLM re-injects the server-side parts automatically + + + + +```python +from litellm import completion + +response = completion( + model="gemini/gemini-3-flash-preview", + messages=[{"role": "user", "content": "What's the weather in Buenos Aires? If it's raining, schedule a meeting."}], + tools=[ + {"type": "web_search_preview"}, # Google Search (server-side) + { + "type": "function", + "function": { + "name": "schedule_meeting", + "description": "Schedule a meeting", + "parameters": { + "type": "object", + "properties": {"reason": {"type": "string"}}, + "required": ["reason"], + }, + }, + }, + ], + include_server_side_tool_invocations=True, +) + +msg = response.choices[0].message + +# Server-side tool results are in provider_specific_fields +psf = msg.provider_specific_fields or {} +for invocation in psf.get("server_side_tool_invocations", []): + print(invocation["tool_type"]) # e.g. "GOOGLE_SEARCH_WEB" + print(invocation["id"]) + print(invocation["args"]) # e.g. {"queries": ["weather Buenos Aires"]} + print(invocation["response"]) # Search results from Google + +# For multi-turn: just append the full message to history +messages.append(msg) +messages.append({"role": "user", "content": "Thanks!"}) +# LiteLLM automatically re-injects the server-side parts + thought signatures +response2 = completion( + model="gemini/gemini-3-flash-preview", + messages=messages, + tools=tools, + include_server_side_tool_invocations=True, +) +``` + + + + +1. Setup config.yaml +```yaml +model_list: + - model_name: gemini-3-flash + litellm_params: + model: gemini/gemini-3-flash-preview + api_key: os.environ/GEMINI_API_KEY +``` + +2. Start Proxy +```bash +$ litellm --config /path/to/config.yaml +``` + +3. Make Request +```bash +curl -X POST 'http://0.0.0.0:4000/chat/completions' \ +-H 'Content-Type: application/json' \ +-H 'Authorization: Bearer sk-1234' \ +-d '{ + "model": "gemini-3-flash", + "messages": [{"role": "user", "content": "What is the weather in Buenos Aires?"}], + "tools": [ + {"type": "web_search_preview"}, + {"type": "function", "function": {"name": "schedule_meeting", "description": "Schedule a meeting", "parameters": {"type": "object", "properties": {"reason": {"type": "string"}}}}} + ], + "include_server_side_tool_invocations": true +}' +``` + + + + +:::info + +- Context circulation requires **Gemini 3+** models +- Server-side tool invocations (`toolCall`/`toolResponse`) are **not** included in `tool_calls` — they are in `provider_specific_fields["server_side_tool_invocations"]` because they were already executed by Google, not by your code +- `thought_signatures` are automatically preserved alongside server-side invocations for multi-turn coherence + +::: + +### URL Context diff --git a/docs/my-website/docs/providers/gemini/music.md b/docs/my-website/docs/providers/gemini/music.md new file mode 100644 index 00000000000..f3968f2db39 --- /dev/null +++ b/docs/my-website/docs/providers/gemini/music.md @@ -0,0 +1,28 @@ +# Gemini — Lyria (music generation) + +Google Lyria 3 preview models are listed in LiteLLM’s [model cost map](https://github.com/BerriAI/litellm/blob/main/model_prices_and_context_window.json) under the `gemini/` provider for metadata and spend tracking. + +| Property | Details | +|----------|---------| +| Provider route | `gemini/` | +| Models | `gemini/lyria-3-clip-preview`, `gemini/lyria-3-pro-preview` | +| Provider docs | [Gemini API pricing / models ↗](https://ai.google.dev/gemini-api/docs/pricing) | + +## Models + +| Model | Notes | +|-------|--------| +| `gemini/lyria-3-clip-preview` | ~30s clip; paid tier listed as per generated song in Google’s pricing | +| `gemini/lyria-3-pro-preview` | Full song; paid tier listed as per generated song in Google’s pricing | + +Input context limit in the cost map: **131,072** tokens. For modalities, limits, and features, see [Google’s Gemini API docs ↗](https://ai.google.dev/gemini-api/docs/models). + +## LiteLLM behavior + +- **Cost map**: Per-song paid pricing is stored as `output_cost_per_image` on those entries (flat per generation unit). Token-based completion cost may not reflect music billing until a dedicated path exists. +- **API calls**: Use the Gemini API as documented by Google. LiteLLM does not ship a separate `music_generation` helper like Veo’s `video_generation`. + +## Auth + +Same as other Gemini API models: `GEMINI_API_KEY` or `GOOGLE_API_KEY`. + diff --git a/docs/my-website/docs/providers/openai.md b/docs/my-website/docs/providers/openai.md index 80931ad8217..1f4a1687e8b 100644 --- a/docs/my-website/docs/providers/openai.md +++ b/docs/my-website/docs/providers/openai.md @@ -581,6 +581,90 @@ curl -X POST 'http://0.0.0.0:4000/chat/completions' \ See [OpenAI Reasoning documentation](https://platform.openai.com/docs/guides/reasoning) for more details on organization verification requirements. +### Multi-turn Conversations with `reasoning_items` + +For multi-turn conversations you need `reasoning_items`: structured blocks that include the `encrypted_content` token OpenAI uses to restore reasoning state on the next request. Pass `include=["reasoning.encrypted_content"]` on every call where you want that token returned. + + + + +```python showLineNumbers title="Non-streaming: round-trip reasoning_items" +import litellm + +messages = [{"role": "user", "content": "Solve this step by step: 2 + 2"}] + +# Turn 1 — get reasoning_items (encrypted_content); +response = litellm.completion( + model="openai/responses/gpt-5-mini", + messages=messages, + reasoning_effort="low", + include=["reasoning.encrypted_content"], +) + +assistant_msg = response.choices[0].message + +# Turn 2 — pass reasoning_items back; LiteLLM converts to the correct Responses API format +messages.append({ + "role": "assistant", + "content": assistant_msg.content, + "reasoning_items": assistant_msg.reasoning_items, +}) +messages.append({"role": "user", "content": "Now summarize your reasoning."}) + +response2 = litellm.completion( + model="openai/responses/gpt-5-mini", + messages=messages, + reasoning_effort="low", + include=["reasoning.encrypted_content"], +) +``` + + + + +`reasoning_items` (with `encrypted_content`) arrive on the final chunk when the full response completes: + +```python showLineNumbers title="Streaming: collect and round-trip reasoning_items" +import litellm + +messages = [{"role": "user", "content": "Solve this step by step: 2 + 2"}] + +collected_content = [] +collected_reasoning_items = [] + +stream = litellm.completion( + model="openai/responses/gpt-5-mini", + messages=messages, + stream=True, + reasoning_effort="low", + include=["reasoning.encrypted_content"], +) + +for chunk in stream: + delta = chunk.choices[0].delta + if delta.content: + collected_content.append(delta.content) + if getattr(delta, "reasoning_items", None): + collected_reasoning_items.extend(delta.reasoning_items) + +messages.append({ + "role": "assistant", + "content": "".join(collected_content), + "reasoning_items": collected_reasoning_items or None, +}) +messages.append({"role": "user", "content": "Continue the conversation."}) + +response2 = litellm.completion( + model="openai/responses/gpt-5-mini", + messages=messages, + reasoning_effort="low", + include=["reasoning.encrypted_content"], +) +``` + + + + ### Verbosity Control for GPT-5 Models The `verbosity` parameter controls the length and detail of responses from GPT-5 family models. It accepts three values: `"low"`, `"medium"`, or `"high"`. @@ -1153,4 +1237,4 @@ response = completion( LiteLLM supports OpenAI's video generation models including Sora. -For detailed documentation on video generation, see [OpenAI Video Generation →](./openai/video_generation.md) +For detailed documentation on video generation, see [OpenAI Video Generation →](./openai/videos.md) diff --git a/docs/my-website/docs/proxy/call_hooks.md b/docs/my-website/docs/proxy/call_hooks.md index 17354725fd5..5935a29c50b 100644 --- a/docs/my-website/docs/proxy/call_hooks.md +++ b/docs/my-website/docs/proxy/call_hooks.md @@ -7,7 +7,7 @@ import Image from '@theme/IdealImage'; - Enforce 'user' param for all openai endpoint calls :::tip -**Understanding Callback Hooks?** Check out our [Callback Management Guide](../observability/callback_management.md) to understand the differences between proxy-specific hooks like `async_pre_call_hook` and general logging hooks like `async_log_success_event`. +**Understanding Callback Hooks?** Check out our [Callback Guide](../observability/callbacks.md) to understand the differences between proxy-specific hooks like `async_pre_call_hook` and general logging hooks like `async_log_success_event`. ::: ## Which Hook Should I Use? diff --git a/docs/my-website/docs/proxy/config_settings.md b/docs/my-website/docs/proxy/config_settings.md index a0e404e3a18..cc9090c2de6 100644 --- a/docs/my-website/docs/proxy/config_settings.md +++ b/docs/my-website/docs/proxy/config_settings.md @@ -206,7 +206,7 @@ router_settings: | Name | Type | Description | |------|------|-------------| -| completion_model | string | The default model to use for completions when `model` is not specified in the request | +| completion_model | string | The model to use for all completions, overriding any `model` specified in the request | | disable_spend_logs | boolean | If true, turns off writing each transaction to the database | | disable_spend_updates | boolean | If true, turns off all spend updates to the DB. Including key/user/team spend updates. | | disable_master_key_return | boolean | If true, turns off returning master key on UI. (checked on '/user/info' endpoint) | @@ -279,6 +279,33 @@ router_settings: | forward_client_headers_to_llm_api | boolean | If true, forwards the client headers (any `x-` headers and `anthropic-beta` headers) to the backend LLM call | | maximum_spend_logs_retention_period | str | Used to set the max retention time for spend logs in the db, after which they will be auto-purged | | maximum_spend_logs_retention_interval | str | Used to set the interval in which the spend log cleanup task should run in. | +| alert_type_config | dict | Configuration mapping alert types to their handler settings | +| always_include_stream_usage | boolean | If true, includes usage metrics in every streaming response chunk | +| auto_redirect_ui_login_to_sso | boolean | If true, automatically redirects UI login page to SSO provider | +| control_plane_url | string | URL of the control plane for cross-instance state sharing | +| custom_auth_run_common_checks | boolean | If true, runs standard auth validation checks alongside custom auth handlers | +| custom_ui_sso_sign_in_handler | string | Custom handler for SSO sign-in logic in the UI | +| database_connection_pool_timeout | integer | Database connection pool timeout in seconds | +| disable_error_logs | boolean | If true, suppresses error tracking and storage in the database | +| enable_health_check_routing | boolean | If true, enables health check-driven request routing to avoid unhealthy deployments | +| enable_mcp_registry | boolean | If true, enables access to the centralized MCP server registry | +| enforce_rbac | boolean | If true, enables role-based access control (RBAC) for all proxy operations | +| forward_llm_provider_auth_headers | boolean | If true, forwards provider-specific auth headers to LLM API calls | +| health_check_concurrency | integer | Maximum number of concurrent health check operations | +| health_check_staleness_threshold | integer | Maximum age in seconds for health check results before marking deployments as stale | +| maximum_spend_logs_cleanup_cron | string | Cron expression for scheduling automatic spend log cleanup tasks | +| mcp_client_side_auth_header_name | string | HTTP header name for client-side MCP server credentials | +| mcp_internal_ip_ranges | list | CIDR ranges considered internal for non-public MCP server access control | +| mcp_required_fields | list | List of required field names for MCP server submissions | +| mcp_trusted_proxy_ranges | list | CIDR ranges of proxies trusted to forward X-Forwarded-For headers for MCP | +| require_end_user_mcp_access_defined | boolean | If true, requires end users to have explicit MCP access permissions defined | +| role_permissions | list | List of role-based permission configurations | +| search_tools | list | List of search tool configurations for enabling web search capabilities | +| token_rate_limit_type | string | Rate limit counting method: "total", "output", or "input" tokens | +| use_redis_transaction_buffer | boolean | If true, buffers database transactions in Redis before writing | +| use_shared_health_check | boolean | If true, uses Redis-backed shared health check state across multiple proxy instances | +| user_header_mappings | dict | Map custom request headers to user IDs using lookup rules | +| user_header_name | string | HTTP header name to extract user identity from requests | ### router_settings - Reference @@ -361,11 +388,14 @@ router_settings: | redis_url | str | URL for Redis server. **Known performance issue with Redis URL.** | | cache_responses | boolean | Flag to enable caching LLM Responses, if cache set under `router_settings`. If true, caches responses. Defaults to False. | | router_general_settings | RouterGeneralSettings | [SDK-Only] Router general settings - contains optimizations like 'async_only_mode'. [Docs](../routing.md#router-general-settings) | -| optional_pre_call_checks | List[str] | List of pre-call checks to add to the router. Supported: `router_budget_limiting`, `prompt_caching`, `responses_api_deployment_check`, `encrypted_content_affinity`, `deployment_affinity`, `session_affinity`, `forward_client_headers_by_model_group` | +| optional_pre_call_checks | List[str] | List of pre-call checks to add to the router. Supported: `router_budget_limiting`, `prompt_caching`, `responses_api_deployment_check`, `encrypted_content_affinity` (requires LiteLLM >= 1.82.3), `deployment_affinity`, `session_affinity`, `forward_client_headers_by_model_group` | | deployment_affinity_ttl_seconds | int | TTL (seconds) for user-key → deployment affinity mapping when `deployment_affinity` is enabled (configured at Router init / proxy startup). Defaults to `3600` (1 hour). | +| model_group_affinity_config | Dict[str, List[str]] | Per-model-group affinity flags. Keys are model group names; values are lists of checks to enable (`deployment_affinity`, `responses_api_deployment_check`, `session_affinity`). Groups not listed fall back to the global `optional_pre_call_checks`. [Docs](../response_api.md#per-model-group-affinity-configuration) | | ignore_invalid_deployments | boolean | If true, ignores invalid deployments. Default for proxy is True - to prevent invalid models from blocking other models from being loaded. | -| search_tools | List[SearchToolTypedDict] | List of search tool configurations for Search API integration. Each tool specifies a search_tool_name and litellm_params with search_provider, api_key, api_base, etc. [Further Docs](../search.md) | +| search_tools | List[SearchToolTypedDict] | List of search tool configurations for Search API integration. Each tool specifies a search_tool_name and litellm_params with search_provider, api_key, api_base, etc. [Further Docs](../search/index.md) | | guardrail_list | List[GuardrailTypedDict] | List of guardrail configurations for guardrail load balancing. Enables load balancing across multiple guardrail deployments with the same guardrail_name. [Further Docs](./guardrails/guardrail_load_balancing.md) | +| enable_health_check_routing | boolean | If true, enables health check-driven deployment filtering to avoid routing requests to unhealthy deployments | +| health_check_staleness_threshold | integer | Maximum age in seconds for cached health check results before marking deployments as stale | ### environment variables - Reference @@ -401,8 +431,10 @@ router_settings: | AUTH_STRATEGY | Strategy used for authentication (e.g., OAuth, API key) | AUTO_REDIRECT_UI_LOGIN_TO_SSO | Flag to enable automatic redirect of UI login page to SSO when SSO is configured. Default is **false** | AUDIO_SPEECH_CHUNK_SIZE | Chunk size for audio speech processing. Default is 1024 -| ANTHROPIC_API_KEY | API key for Anthropic service +| ANTHROPIC_API_KEY | API key for Anthropic service. Uses `x-api-key` header for authentication. +| ANTHROPIC_AUTH_TOKEN | Alternative auth token for Anthropic service. Uses `Authorization: Bearer` header instead of `x-api-key`. Used as fallback when `ANTHROPIC_API_KEY` is not set. | ANTHROPIC_API_BASE | Base URL for Anthropic API. Default is https://api.anthropic.com +| ANTHROPIC_BASE_URL | Alternative to `ANTHROPIC_API_BASE` for setting the Anthropic API base URL. Used as fallback when `ANTHROPIC_API_BASE` is not set. | ANTHROPIC_TOKEN_COUNTING_BETA_VERSION | Beta version header for Anthropic token counting API. Default is `token-counting-2024-11-01` | AWS_ACCESS_KEY_ID | Access Key ID for AWS services | AWS_BATCH_ROLE_ARN | ARN of the AWS IAM role for batch operations @@ -801,6 +833,7 @@ router_settings: | LITELLM_OTEL_INTEGRATION_ENABLE_EVENTS | Optionally enable semantic logs for OTEL | LITELLM_OTEL_INTEGRATION_ENABLE_METRICS | Optionally enable emantic metrics for OTEL | LITELLM_ENABLE_PYROSCOPE | If true, enables Pyroscope CPU profiling. Profiles are sent to PYROSCOPE_SERVER_ADDRESS. Off by default. See [Pyroscope profiling](/proxy/pyroscope_profiling). +| LITELLM_ENABLE_TEAM_STALE_ALIAS_BYPASS | When `true`, if a team's legacy `model_aliases` entry maps a public model name to an internal `model_name__` deployment, pre-call handling can skip that rewrite when team-scoped sibling deployments exist for the public name—so load balancing / `order` apply across siblings. Default is `false` for backwards compatibility. See [Team-scoped models and legacy aliases](./load_balancing#team-scoped-models-and-legacy-model_aliases). When stale aliases are detected and this flag is off, the proxy may log a one-time warning. | PYROSCOPE_APP_NAME | Application name reported to Pyroscope. Required when LITELLM_ENABLE_PYROSCOPE is true. No default. | PYROSCOPE_SERVER_ADDRESS | Pyroscope server URL to send profiles to. Required when LITELLM_ENABLE_PYROSCOPE is true. No default. | PYROSCOPE_SAMPLE_RATE | Optional. Sample rate for Pyroscope profiling (integer). No default; when unset, the pyroscope-io library default is used. @@ -811,7 +844,7 @@ router_settings: | LITELLM_MODE | Operating mode for LiteLLM (e.g., production, development) | LITELLM_NON_ROOT | Flag to run LiteLLM in non-root mode for enhanced security in Docker containers | LITELLM_RATE_LIMIT_WINDOW_SIZE | Rate limit window size for LiteLLM. Default is 60 -| LITELLM_REASONING_AUTO_SUMMARY | If set to "true", automatically enables detailed reasoning summaries for reasoning models (e.g., o1, o3-mini, deepseek-reasoner). When enabled, adds `summary: "detailed"` to reasoning effort configurations. Default is "false" +| LITELLM_REASONING_AUTO_SUMMARY | If set to "true", automatically enables detailed reasoning summaries (`summary: "detailed"`) for reasoning models across all translation paths (Anthropic adapter, Responses API, etc.). Default is "false" | LITELLM_SALT_KEY | Salt key for encryption in LiteLLM | LITELLM_SSL_CIPHERS | SSL/TLS cipher configuration for faster handshakes. Controls cipher suite preferences for OpenSSL connections. | LITELLM_SECRET_AWS_KMS_LITELLM_LICENSE | AWS KMS encrypted license for LiteLLM @@ -902,6 +935,7 @@ router_settings: | OTEL_SERVICE_NAME | Service name identifier for OpenTelemetry | OTEL_TRACER_NAME | Tracer name for OpenTelemetry tracing | OTEL_LOGS_EXPORTER | Exporter type for OpenTelemetry logs (e.g., console) +| OTEL_IGNORE_CONTEXT_PROPAGATION | When true, ignore parent span context propagation in OpenTelemetry callbacks | PAGERDUTY_API_KEY | API key for PagerDuty Alerting | PANW_PRISMA_AIRS_API_KEY | API key for PANW Prisma AIRS service | PANW_PRISMA_AIRS_API_BASE | Base URL for PANW Prisma AIRS service @@ -949,6 +983,8 @@ router_settings: | QDRANT_URL | Connection URL for Qdrant database | QDRANT_VECTOR_SIZE | Vector size for Qdrant operations. Default is 1536 | REDIS_CONNECTION_POOL_TIMEOUT | Timeout in seconds for Redis connection pool. Default is 5 +| REDIS_CIRCUIT_BREAKER_FAILURE_THRESHOLD | Number of consecutive failures before the Redis circuit breaker opens. Default is 5 +| REDIS_CIRCUIT_BREAKER_RECOVERY_TIMEOUT | Time in seconds before the Redis circuit breaker attempts recovery after opening. Default is 60 | REDIS_CLUSTER_NODES | JSON-formatted list of Redis cluster startup nodes for Redis Cluster mode. Example: `[{"host": "node1", "port": 6379}]` | REDIS_HOST | Hostname for Redis server | REDIS_PASSWORD | Password for Redis service diff --git a/docs/my-website/docs/proxy/configs.md b/docs/my-website/docs/proxy/configs.md index 56a8b9566db..84a6fac1210 100644 --- a/docs/my-website/docs/proxy/configs.md +++ b/docs/my-website/docs/proxy/configs.md @@ -602,6 +602,22 @@ Since you shouldn't use 12.5, round down to **10** to leave a safety buffer. Thi - Total maximum connections: 8 workers × 10 connections = 80 connections - This stays safely under your database's 100 connection limit +## LiteLLM License Key (Enterprise) + +To enable [LiteLLM Enterprise features](https://docs.litellm.ai/docs/proxy/enterprise), set your license key as an environment variable: + +```bash +export LITELLM_LICENSE="eyJ..." +``` + +The license key is a JWT token provided when you purchase a LiteLLM Enterprise license. Once set, LiteLLM will automatically detect and activate enterprise features. + +You can also add it to your `.env` file: + +```env +LITELLM_LICENSE="eyJ..." +``` + ## Extras diff --git a/docs/my-website/docs/proxy/cost_tracking.md b/docs/my-website/docs/proxy/cost_tracking.md index f28eec287d4..f9e22cfecd3 100644 --- a/docs/my-website/docs/proxy/cost_tracking.md +++ b/docs/my-website/docs/proxy/cost_tracking.md @@ -163,7 +163,7 @@ Use this when you want non-proxy admins to access `/spend` endpoints :::info -Schedule a [meeting with us to get your Enterprise License](https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions) +Schedule a [meeting with us to get your Enterprise License](https://enterprise.litellm.ai/demo) ::: diff --git a/docs/my-website/docs/proxy/docker_quick_start.md b/docs/my-website/docs/proxy/docker_quick_start.md index efdc73de43e..58a56604751 100644 --- a/docs/my-website/docs/proxy/docker_quick_start.md +++ b/docs/my-website/docs/proxy/docker_quick_start.md @@ -1,25 +1,90 @@ - import Tabs from '@theme/Tabs'; import TabItem from '@theme/TabItem'; +import Image from '@theme/IdealImage'; # Getting Started Tutorial End-to-End tutorial for LiteLLM Proxy to: -- Add an Azure OpenAI model -- Make a successful /chat/completion call -- Generate a virtual key -- Set RPM limit on virtual key +- Add an Azure OpenAI model +- Make a successful /chat/completion call +- Generate a virtual key +- Set RPM limit on virtual key +## Quick Install (Recommended for local / beginners) + +New to LiteLLM? This is the easiest way to get started locally. One command installs LiteLLM and walks you through setup interactively — no config files to write by hand. + +### 1. Install + +```bash +curl -fsSL https://raw.githubusercontent.com/BerriAI/litellm/main/scripts/install.sh | sh +``` + +This detects your OS, installs `litellm[proxy]`, and drops you straight into the setup wizard. + +### 2. Follow the wizard + +``` +$ litellm --setup + + Welcome to LiteLLM + + Choose your LLM providers + ○ 1. OpenAI GPT-4o, GPT-4o-mini, o1 + ○ 2. Anthropic Claude Opus, Sonnet, Haiku + ○ 3. Azure OpenAI GPT-4o via Azure + ○ 4. Google Gemini Gemini 2.0 Flash, 1.5 Pro + ○ 5. AWS Bedrock Claude, Llama via AWS + ○ 6. Ollama Local models + + ❯ Provider(s): 1,2 + + ❯ OpenAI API key: sk-... + ❯ Anthropic API key: sk-ant-... + + ❯ Port [4000]: + ❯ Master key [auto-generate]: + + ✔ Config saved → ./litellm_config.yaml + + ❯ Start the proxy now? (Y/n): +``` + +The wizard walks you through: +1. Pick your LLM providers (OpenAI, Anthropic, Azure, Bedrock, Gemini, Ollama) +2. Enter API keys for each provider +3. Set a port and master key (or accept the defaults) +4. Config is saved to `./litellm_config.yaml` and the proxy starts immediately + +### 3. Make a call + +Your proxy is running on `http://0.0.0.0:4000`. Test it: + +```bash +curl -X POST 'http://0.0.0.0:4000/chat/completions' \ +-H 'Content-Type: application/json' \ +-H 'Authorization: Bearer ' \ +-d '{ + "model": "gpt-4o", + "messages": [{"role": "user", "content": "Hello!"}] +}' +``` + +:::tip Already have pip installed? +You can skip the curl install and run `litellm --setup` directly after `pip install 'litellm[proxy]'`. +::: + +--- ## Pre-Requisites -- Install LiteLLM Docker Image **OR** LiteLLM CLI (pip package) +Choose your install method. **Docker Compose** users complete their full setup inside the tab and are done. **Docker** and **pip** users continue with the steps below the tabs. -``` +```bash docker pull docker.litellm.ai/berriai/litellm:main-latest ``` @@ -37,7 +102,25 @@ $ pip install 'litellm[proxy]' -Use this docker compose to spin up the proxy with a postgres database running locally. +Docker Compose bundles LiteLLM with a Postgres database. Follow the steps below — the proxy will be fully running by the end. + +### Step 1 — Pull the LiteLLM database image + +LiteLLM provides a dedicated `litellm-database` image for proxy deployments that connect to Postgres. + +```bash +docker pull ghcr.io/berriai/litellm-database:main-latest +``` + +See all available tags on the [GitHub Container Registry](https://github.com/BerriAI/litellm/pkgs/container/litellm-database). + +--- + +### Step 2 — Set up a database + +Complete all three config files **before** running `docker compose up`. The proxy server will not start correctly if any of these are missing. + +#### 2.1 — Get `docker-compose.yml` and create `.env` ```bash # Get the docker compose file @@ -46,26 +129,154 @@ curl -O https://raw.githubusercontent.com/BerriAI/litellm/main/docker-compose.ym # Add the master key - you can change this after setup echo 'LITELLM_MASTER_KEY="sk-1234"' > .env -# Add the litellm salt key - you cannot change this after adding a model -# It is used to encrypt / decrypt your LLM API Key credentials -# We recommend - https://1password.com/password-generator/ -# password generator to get a random hash for litellm salt key +# Add the litellm salt key — cannot be changed after adding a model +# Used to encrypt/decrypt your LLM API key credentials +# Generate a strong random value: https://1password.com/password-generator/ echo 'LITELLM_SALT_KEY="sk-1234"' >> .env -# Start +# Add your model credentials +echo 'AZURE_API_BASE="https://openai-***********/"' >> .env +echo 'AZURE_API_KEY="your-azure-api-key"' >> .env +``` + +#### 2.2 — Create `config.yaml` + +The default `docker-compose.yml` starts a Postgres container at `db:5432`. Your `config.yaml` must include `database_url` pointing to it: + +```yaml +model_list: + - model_name: gpt-4o + litellm_params: + model: azure/my_azure_deployment + api_base: os.environ/AZURE_API_BASE + api_key: os.environ/AZURE_API_KEY + api_version: "2025-01-01-preview" + +general_settings: + master_key: sk-1234 # 🔑 your proxy admin key (must start with sk-) + database_url: "postgresql://llmproxy:dbpassword9090@db:5432/litellm" +``` + +:::tip +`database_url` enables virtual keys, spend tracking, and the UI. Replace it with your [Supabase](https://supabase.com/) or [Neon](https://neon.tech/) connection string if you prefer a managed database. +::: + +#### 2.3 — Create `prometheus.yml` + +This file **must exist as a file** before `docker compose up`. If it is missing, Docker auto-creates it as an empty directory and the Prometheus container fails to start. + +```yaml +global: + scrape_interval: 15s + evaluation_interval: 15s + +scrape_configs: + - job_name: "litellm" + static_configs: + - targets: ["litellm:4000"] +``` + +Also verify that the `config.yaml` volume mount and `--config` flag are **not commented out** in `docker-compose.yml`: + +```yaml +services: + litellm: + volumes: + - ./config.yaml:/app/config.yaml # ✅ must be uncommented + command: + - "--config=/app/config.yaml" # ✅ must be uncommented +``` + +:::warning +All three files (`.env`, `config.yaml`, `prometheus.yml`) must be present before running `docker compose up`. See [Troubleshooting](#troubleshooting) if you run into issues. +::: + +--- + +### Step 3 — Start the proxy server and test it + +After `config.yaml`, `prometheus.yml`, and `.env` are complete, start the proxy: + +```bash docker compose up ``` +Once running, test it with a curl request: + +```bash +curl -X POST 'http://0.0.0.0:4000/chat/completions' \ + -H 'Content-Type: application/json' \ + -H 'Authorization: Bearer sk-1234' \ + -d '{ + "model": "gpt-4o", + "messages": [{"role": "user", "content": "Hello!"}] + }' +``` + +**Expected response:** + +```json +{ + "id": "chatcmpl-abcd", + "created": 1773817678, + "model": "gpt-4o", + "object": "chat.completion", + "system_fingerprint": "fp_6b1ef07cda", + "choices": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "Hello! How can I assist you today?", + "role": "assistant", + "annotations": [] + } + } + ], + "usage": { + "completion_tokens": 9, + "prompt_tokens": 9, + "total_tokens": 18, + "completion_tokens_details": { + "accepted_prediction_tokens": 0, + "audio_tokens": 0, + "reasoning_tokens": 0, + "rejected_prediction_tokens": 0 + }, + "prompt_tokens_details": { + "audio_tokens": 0, + "cached_tokens": 0 + } + }, + "service_tier": "default" +} +``` + +--- + +### Optional — Navigate to the LiteLLM UI and generate a virtual key + +Open [http://localhost:4000/ui](http://localhost:4000/ui) in your browser and log in with your master key (`sk-1234`). + +Navigate to **Virtual Keys** and click **+ Create New Key**: + +LiteLLM UI — Create Virtual Key + +Virtual keys let you track spend, set rate limits, and control model access per user or team. + + -## 1. Add a model +:::note Docker Compose users +Your setup is complete — the steps below are for **Docker** and **pip** users only. +::: -Control LiteLLM Proxy with a config.yaml file. +--- -Setup your config.yaml with your azure model. +## Step 1 — Add a model -Note: When using the proxy with a database, you can also **just add models via UI** (UI is available on `/ui` route). +Control LiteLLM Proxy with a `config.yaml` file. Create one with your Azure model: ```yaml model_list: @@ -89,8 +300,6 @@ You can read more about how model resolution works in the [Model Configuration]( - **`api_base`** (`str`) - The API base for your azure deployment. - **`api_version`** (`str`) - The API Version to use when calling Azure's OpenAI API. Get the latest Inference API version [here](https://learn.microsoft.com/en-us/azure/ai-services/openai/api-version-deprecation?source=recommendations#latest-preview-api-releases). ---- - --- @@ -138,19 +347,19 @@ $ litellm --config /app/config.yaml --detailed_debug +Confirm your config was loaded correctly — you should see this in the logs: -Confirm your config.yaml got mounted correctly - -```bash +``` Loaded config YAML (api_key and environment_variables are not shown): { -"model_list": [ -{ -"model_name ... + "model_list": [ + { + "model_name": ... ``` ### 2.2 Make Call +LiteLLM Proxy is 100% OpenAI-compatible. Test your model via `/chat/completions`: ```bash curl -X POST 'http://0.0.0.0:4000/chat/completions' \ @@ -244,15 +453,17 @@ curl -X POST 'http://0.0.0.0:4000/chat/completions' \ - [Other/Non-Chat Completion Endpoints](../embedding/supported_embedding.md) - [Pass-through for VertexAI, Bedrock, etc.](../pass_through/vertex_ai.md) -## 3. Generate a virtual key +## Optional: Generate a virtual key -Track Spend, and control model access via virtual keys for the proxy +Track spend and control model access via virtual keys for the proxy. -### 3.1 Set up a Database +### Prerequisite — Set up a database -**Requirements** -- Need a postgres database (e.g. [Supabase](https://supabase.com/), [Neon](https://neon.tech/), etc) +:::note Docker Compose users +Your Postgres container is already running — skip ahead to [Create Key w/ RPM Limit](#create-key-w-rpm-limit) below. +::: +**Docker / pip users** — you need a Postgres database (e.g. [Supabase](https://supabase.com/), [Neon](https://neon.tech/), or self-hosted). Add `general_settings` to your `config.yaml`: ```yaml model_list: @@ -268,7 +479,9 @@ general_settings: database_url: "postgresql://:@:/" # 👈 KEY CHANGE ``` -Save config.yaml as `litellm_config.yaml` (used in 3.2). +Save config.yaml as `litellm_config.yaml` before continuing. + +You must finish this setup before starting the proxy server. --- @@ -294,7 +507,7 @@ See All General Settings [here](http://localhost:3000/docs/proxy/configs#all-set `database_url: "postgresql://..."` - Set `DATABASE_URL=postgresql://:@:/` in your env -### 3.2 Start Proxy +### Start Proxy ```bash docker run \ @@ -302,12 +515,11 @@ docker run \ -e AZURE_API_KEY=d6*********** \ -e AZURE_API_BASE=https://openai-***********/ \ -p 4000:4000 \ - docker.litellm.ai/berriai/litellm:main-latest \ + ghcr.io/berriai/litellm-database:main-latest \ --config /app/config.yaml --detailed_debug ``` - -### 3.3 Create Key w/ RPM Limit +### Create Key w/ RPM Limit Create a key with `rpm_limit: 1`. This will only allow 1 request per minute for calls to proxy with this key. @@ -330,9 +542,9 @@ curl -L -X POST 'http://0.0.0.0:4000/key/generate' \ } ``` -### 3.4 Test it! +### Test it! -**Use your virtual key from step 3.3** +**Use the virtual key you just created.** 1st call - Expect to work! @@ -546,6 +758,24 @@ model_list: ## Troubleshooting +### `prometheus.yml` mount error — "not a directory" + +If you see: + +```bash +Error: cannot create subdirectories in ".../prometheus.yml": not a directory +``` + +Docker created `prometheus.yml` as an **empty directory** instead of a file. This happens when the file is missing at `docker compose up` time. + +Fix it: +Then create the file (see [Step 2.3 — Create `prometheus.yml`](#23--create-prometheusyml)) and run `docker compose up` again. +```bash +rm -rf prometheus.yml +``` + +Then create the file (see [Step 2.4](#step-24--create-prometheusyml)) and run `docker compose up` again. + ### Non-root docker image? If you need to run the docker image as a non-root user, use [this](https://github.com/BerriAI/litellm/pkgs/container/litellm-non_root). @@ -645,6 +875,3 @@ LiteLLM Proxy uses the [LiteLLM Python SDK](https://docs.litellm.ai/docs/routing - Our emails ✉️ ishaan@berri.ai / krrish@berri.ai [![Chat on WhatsApp](https://img.shields.io/static/v1?label=Chat%20on&message=WhatsApp&color=success&logo=WhatsApp&style=flat-square)](https://wa.link/huol9n) [![Chat on Discord](https://img.shields.io/static/v1?label=Chat%20on&message=Discord&color=blue&logo=Discord&style=flat-square)](https://discord.gg/wuPM9dRgDw) - - - diff --git a/docs/my-website/docs/proxy/email.md b/docs/my-website/docs/proxy/email.md index 86a79cbcfc8..ba737c6782c 100644 --- a/docs/my-website/docs/proxy/email.md +++ b/docs/my-website/docs/proxy/email.md @@ -203,7 +203,7 @@ After regenerating the key, the user will receive an email notification with: :::info -Customizing Email Branding is an Enterprise Feature [Get in touch with us for a Free Trial](https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions) +Customizing Email Branding is an Enterprise Feature [Get in touch with us for a Free Trial](https://enterprise.litellm.ai/demo) ::: diff --git a/docs/my-website/docs/proxy/endpoint_activity.md b/docs/my-website/docs/proxy/endpoint_activity.md index a66c0f7a5e5..d06727ce4b4 100644 --- a/docs/my-website/docs/proxy/endpoint_activity.md +++ b/docs/my-website/docs/proxy/endpoint_activity.md @@ -114,4 +114,4 @@ Understand spend distribution across endpoints: - [Customer Usage](./customer_usage.md) - Track spend and usage for individual customers - [Cost Tracking](./cost_tracking.md) - Comprehensive cost tracking and analytics -- [Spend Logs](./spend_logs.md) - Detailed request-level spend logs +- [Spend Logs](./cost_tracking.md#-spend-logs-api---individual-transaction-logs) - Detailed request-level spend logs diff --git a/docs/my-website/docs/proxy/enterprise.md b/docs/my-website/docs/proxy/enterprise.md index 4b525837a20..09b103ca4a0 100644 --- a/docs/my-website/docs/proxy/enterprise.md +++ b/docs/my-website/docs/proxy/enterprise.md @@ -5,7 +5,7 @@ import TabItem from '@theme/TabItem'; # ✨ Enterprise Features :::tip -To get a license, get in touch with us [here](https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions) +To get a license, get in touch with us [here](https://enterprise.litellm.ai/demo) ::: diff --git a/docs/my-website/docs/proxy/guardrails/akto.md b/docs/my-website/docs/proxy/guardrails/akto.md new file mode 100644 index 00000000000..67ae741d11e --- /dev/null +++ b/docs/my-website/docs/proxy/guardrails/akto.md @@ -0,0 +1,139 @@ +# Akto + +## Overview +[Akto](https://www.akto.io/) provides API security guardrails and data ingestion for LLM traffic. + +Akto now uses a **two-entry guardrail pattern** in LiteLLM: +- `akto-validate` (`pre_call`) for request validation +- `akto-ingest` (`post_call`) for request/response ingestion + +There is no `on_flagged` setting anymore. + +Use these as two separate guardrails in `config.yaml`: +- `guardrail_name: "akto-validate"` +- `guardrail_name: "akto-ingest"` + +## 1. Get Your Akto Credentials + +Set up the Akto Guardrail API Service and grab: +- `AKTO_GUARDRAIL_API_BASE` — your Guardrail API Base URL +- `AKTO_API_KEY` — your API key + +## 2. Configure in `config.yaml` + +### Block + Ingest (recommended) + +Use both entries below. This gives you: +- pre-call block decision +- post-call ingestion for allowed traffic + +Keep these as two separate entries (`akto-validate` and `akto-ingest`). + +```yaml +guardrails: + - guardrail_name: "akto-validate" + litellm_params: + guardrail: akto + mode: pre_call + akto_base_url: os.environ/AKTO_GUARDRAIL_API_BASE + akto_api_key: os.environ/AKTO_API_KEY + default_on: true + unreachable_fallback: fail_closed # optional: fail_open | fail_closed (default: fail_closed) + guardrail_timeout: 5 # optional, default: 5 + akto_account_id: "1000000" # optional, env fallback: AKTO_ACCOUNT_ID + akto_vxlan_id: "0" # optional, env fallback: AKTO_VXLAN_ID + + - guardrail_name: "akto-ingest" + litellm_params: + guardrail: akto + mode: post_call + akto_base_url: os.environ/AKTO_GUARDRAIL_API_BASE + akto_api_key: os.environ/AKTO_API_KEY + default_on: true +``` + +### Monitor-only mode + +If you only want logging/ingestion and no blocking, keep only `akto-ingest`. + +```yaml +guardrails: + - guardrail_name: "akto-ingest" + litellm_params: + guardrail: akto + mode: post_call + akto_base_url: os.environ/AKTO_GUARDRAIL_API_BASE + akto_api_key: os.environ/AKTO_API_KEY + default_on: true +``` + +## 3. Test It + +```shell +curl -i http://localhost:4000/v1/chat/completions \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer " \ + -d '{ + "model": "gpt-3.5-turbo", + "messages": [ + {"role": "user", "content": "Hello, how are you?"} + ] + }' +``` + +If a request gets blocked: + +```json +{ + "error": { + "message": "Prompt injection detected", + "type": "None", + "param": "None", + "code": "403" + } +} +``` + +## 4. How It Works + +**Block + Ingest mode:** +``` +Request → LiteLLM → Akto guardrail check + → Allowed → forward to LLM → ingest response + → Blocked → ingest blocked marker → 403 error +``` + +**Monitor-only mode:** +``` +Request → LiteLLM → forward to LLM → get response + → Send to Akto (guardrails + ingest) → log only +``` + +## 5. Event behavior + +| Entry | LiteLLM hook | Akto call behavior | +|------|---|---| +| `akto-validate` | `pre_call` | Awaited call with `guardrails=true`, `ingest_data=false` | +| `akto-ingest` | `post_call` | Fire-and-forget call with `guardrails=true`, `ingest_data=true` | + +When blocked in `pre_call`, LiteLLM sends one fire-and-forget ingest payload with blocked metadata and returns `403`. + +## 6. Parameters + +| Parameter | Env Variable | Default | Description | +|-----------|-------------|---------|-------------| +| `akto_base_url` | `AKTO_GUARDRAIL_API_BASE` | *required* | Akto Guardrail API Base URL | +| `akto_api_key` | `AKTO_API_KEY` | *required* | API key (sent as `Authorization` header) | +| `akto_account_id` | `AKTO_ACCOUNT_ID` | `1000000` | Akto account id included in payload | +| `akto_vxlan_id` | `AKTO_VXLAN_ID` | `0` | Akto vxlan id included in payload | +| `unreachable_fallback` | — | `fail_closed` | `fail_open` or `fail_closed` | +| `guardrail_timeout` | — | `5` | Timeout in seconds | +| `default_on` | — | `true` (recommended) | Enables the guardrail entry by default | + +## 7. Error Handling + +| Scenario | `fail_closed` (default) | `fail_open` | +|----------|------------------------|-------------| +| Akto unreachable | ❌ Blocked (503) | ✅ Passes through | +| Akto returns error | ❌ Blocked (503) | ✅ Passes through | +| Guardrail says no | ❌ Blocked (403) | ❌ Blocked (403) | diff --git a/docs/my-website/docs/proxy/guardrails/aporia_api.md b/docs/my-website/docs/proxy/guardrails/aporia_api.md index ceafc19a1cc..e6ff0d5fed3 100644 --- a/docs/my-website/docs/proxy/guardrails/aporia_api.md +++ b/docs/my-website/docs/proxy/guardrails/aporia_api.md @@ -139,7 +139,7 @@ curl -i http://localhost:4000/v1/chat/completions \ :::info -✨ This is an Enterprise only feature [Contact us to get a free trial](https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions) +✨ This is an Enterprise only feature [Contact us to get a free trial](https://enterprise.litellm.ai/demo) ::: diff --git a/docs/my-website/docs/proxy/guardrails/custom_guardrail.md b/docs/my-website/docs/proxy/guardrails/custom_guardrail.md index c9115cf8265..37579ad870d 100644 --- a/docs/my-website/docs/proxy/guardrails/custom_guardrail.md +++ b/docs/my-website/docs/proxy/guardrails/custom_guardrail.md @@ -117,6 +117,14 @@ guardrails: ::: +:::note Streaming and post_call guardrails + +For **streaming responses**, `post_call` guardrails run on the fully assembled response **after** all chunks have been delivered to the client. This means `post_call` guardrails on streaming are **audit-only** — they can inspect and log the complete response, but cannot block content delivery. Guardrail results are recorded in `guardrail_information` within the logging payload for compliance and auditing. + +To filter or block streaming content in real-time, use `async_post_call_streaming_iterator_hook` instead, which processes chunks as they arrive. + +::: +
Advanced: Multiple modes with individual event hooks @@ -409,7 +417,7 @@ curl -i -X POST http://localhost:4000/v1/chat/completions \ :::info -✨ This is an Enterprise only feature [Contact us to get a free trial](https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions) +✨ This is an Enterprise only feature [Contact us to get a free trial](https://enterprise.litellm.ai/demo) ::: @@ -655,8 +663,8 @@ class myCustomGuardrail(CustomGuardrail): | `apply_guardrail` | Simple method to check and optionally modify text | ✅ | INPUT or OUTPUT | ✅ | ✅ | ✅ | | `async_pre_call_hook` | A hook that runs before the LLM API call | ✅ | INPUT | ✅ | ❌ | ✅ | | `async_moderation_hook` | A hook that runs during the LLM API call| ✅ | INPUT | ❌ | ❌ | ✅ | -| `async_post_call_success_hook` | A hook that runs after a successful LLM API call| ✅ | INPUT, OUTPUT | ❌ | ✅ | ✅ | -| `async_post_call_streaming_iterator_hook` | A hook that processes streaming responses | ✅ | OUTPUT | ❌ | ✅ | ✅ | +| `async_post_call_success_hook` | A hook that runs after a successful LLM API call. For streaming, runs on the assembled response after delivery (audit-only, cannot block). | ✅ | INPUT, OUTPUT | ❌ | ✅ | ✅ (non-streaming only) | +| `async_post_call_streaming_iterator_hook` | A hook that processes streaming responses in real-time (can filter/block chunks) | ✅ | OUTPUT | ❌ | ✅ | ✅ | ## Frequently Asked Questions diff --git a/docs/my-website/docs/proxy/guardrails/guardrails_ai.md b/docs/my-website/docs/proxy/guardrails/guardrails_ai.md index 55d586aee7b..19ae34014a4 100644 --- a/docs/my-website/docs/proxy/guardrails/guardrails_ai.md +++ b/docs/my-website/docs/proxy/guardrails/guardrails_ai.md @@ -59,7 +59,7 @@ curl -i http://localhost:4000/v1/chat/completions \ :::info -✨ This is an Enterprise only feature [Contact us to get a free trial](https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions) +✨ This is an Enterprise only feature [Contact us to get a free trial](https://enterprise.litellm.ai/demo) ::: diff --git a/docs/my-website/docs/proxy/health.md b/docs/my-website/docs/proxy/health.md index 2764a6f0d4f..530bea3d06b 100644 --- a/docs/my-website/docs/proxy/health.md +++ b/docs/my-website/docs/proxy/health.md @@ -314,6 +314,89 @@ general_settings: health_check_details: False ``` +## Health Check Driven Routing + +By default, background health checks are observability-only — they populate the `/health` endpoint but don't affect routing. Unhealthy deployments still receive traffic until request failures trigger cooldown. + +With `enable_health_check_routing: true`, the router **excludes deployments that failed their last background health check** before selecting a candidate. This gives you proactive failover instead of reactive cooldown. + +### How it works + +1. Background health checks run on their configured interval +2. After each cycle, every deployment is marked healthy or unhealthy +3. On each incoming request, the router filters out unhealthy deployments **before** cooldown filtering and load balancing +4. If all deployments are unhealthy, the filter is bypassed (safety net — never causes a total outage) +5. If health state is stale (older than `health_check_staleness_threshold`), it is ignored + +### Quick start + +```yaml +model_list: + - model_name: gpt-4 + litellm_params: + model: openai/gpt-4 + api_key: os.environ/OPENAI_API_KEY + - model_name: gpt-4 + litellm_params: + model: openai/gpt-4 + api_key: os.environ/OPENAI_API_KEY_SECONDARY + +general_settings: + background_health_checks: true + health_check_interval: 60 + enable_health_check_routing: true +``` + +### Configuration + +| Setting | Where | Default | Description | +|---------|-------|---------|-------------| +| `enable_health_check_routing` | `general_settings` | `false` | Enable/disable health-check-driven routing | +| `health_check_staleness_threshold` | `general_settings` | `health_check_interval * 2` | Seconds before health state is considered stale and ignored | +| `background_health_checks` | `general_settings` | `false` | Must be `true` for health check routing to work | +| `health_check_interval` | `general_settings` | `300` | Seconds between health check cycles | + +### Interaction with cooldown + +Health check filtering and cooldown are **additive**. A deployment can be excluded by either mechanism: + +- **Health check filter** — proactive, runs on the configured interval, excludes deployments that failed the last check +- **Cooldown** — reactive, triggered by request failures, excludes deployments for a short TTL + +This means request failures still provide fast detection between health check intervals. + +### Staleness + +If a health check result is older than `health_check_staleness_threshold`, it is ignored and the deployment is treated as eligible. This prevents stale data from permanently excluding a deployment if the health check loop stops or slows down. + +The default staleness threshold is `health_check_interval * 2`. For a 60s interval, health state expires after 120s. + +### Example: custom staleness + +```yaml +general_settings: + background_health_checks: true + health_check_interval: 30 + enable_health_check_routing: true + health_check_staleness_threshold: 90 # ignore health state older than 90s +``` + +### Debugging + +Run the proxy with `--detailed_debug` and look for: + +``` +health_check_routing_state_updated healthy=3 unhealthy=1 +``` + +This is logged after each health check cycle when routing state is written. + +If the safety net triggers (all deployments unhealthy), you'll see: + +``` +All deployments marked unhealthy by health checks, bypassing health filter +``` + ## Health Check Timeout The health check timeout is set in `litellm/constants.py` and defaults to 60 seconds. diff --git a/docs/my-website/docs/proxy/high_availability_control_plane.md b/docs/my-website/docs/proxy/high_availability_control_plane.md new file mode 100644 index 00000000000..324fba3a180 --- /dev/null +++ b/docs/my-website/docs/proxy/high_availability_control_plane.md @@ -0,0 +1,190 @@ +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; +import { ControlPlaneArchitecture } from '@site/src/components/ControlPlaneArchitecture'; + +# [BETA] High Availability Control Plane + +Deploy a single LiteLLM UI that manages multiple independent LiteLLM proxy instances, each with its own database, Redis, and master key. + +:::info + +This is an Enterprise feature. + +[Enterprise Pricing](https://www.litellm.ai/#pricing) + +[Get free 7-day trial key](https://www.litellm.ai/enterprise#trial) + +::: + +## Why This Architecture? + +In the [standard multi-region setup](./control_plane_and_data_plane.md), all instances share a single database and master key. This works, but introduces a shared dependency. If the database goes down, every instance is affected. + +The **High Availability Control Plane** takes a different approach: + +| | Shared Database (Standard) | High Availability Control Plane | +|---|---|---| +| **Database** | Single shared DB for all instances | Each instance has its own DB | +| **Redis** | Shared Redis | Each instance has its own Redis | +| **Master Key** | Same key across all instances | Each instance has its own key | +| **Failure isolation** | DB outage affects all instances | Failure is isolated to one instance | +| **User management** | Centralized, one user table | Independent, each worker manages its own users | +| **UI** | One UI per admin instance | Single control plane UI manages all workers | + +### Benefits + +- **True high availability**: no shared infrastructure means no single point of failure +- **Blast radius containment**: a misconfiguration or outage on one worker doesn't affect others +- **Regional isolation**: workers can run in different regions with data residency requirements +- **Simpler operations**: each worker is a self-contained LiteLLM deployment + +## Architecture + + + +The **control plane** is a LiteLLM instance that serves the admin UI and knows about all the workers. It does not proxy LLM requests, it is purely for administration. + +Each **worker** is a fully independent LiteLLM proxy that handles LLM requests for its region or team. Workers have their own users, keys, teams, and budgets. + +## Setup + +### 1. Control Plane Configuration + +The control plane needs a `worker_registry` that lists all worker instances. + +```yaml title="cp_config.yaml" +model_list: [] + +general_settings: + master_key: sk-1234 + database_url: os.environ/DATABASE_URL + +worker_registry: + - worker_id: "worker-a" + name: "Worker A" + url: "http://localhost:4001" + - worker_id: "worker-b" + name: "Worker B" + url: "http://localhost:4002" +``` + +Start the control plane: + +```bash +litellm --config cp_config.yaml --port 4000 +``` + +### 2. Worker Configuration + +Each worker needs `control_plane_url` in its `general_settings` to enable cross-origin authentication from the control plane UI. + +`PROXY_BASE_URL` must also be set for each worker so that SSO callback redirects resolve correctly. + + + + +```yaml title="worker_a_config.yaml" +model_list: [] + +general_settings: + master_key: sk-worker-a-1234 + database_url: os.environ/WORKER_A_DATABASE_URL + control_plane_url: "http://localhost:4000" +``` + +```bash +PROXY_BASE_URL=http://localhost:4001 litellm --config worker_a_config.yaml --port 4001 +``` + + + + +```yaml title="worker_b_config.yaml" +model_list: [] + +general_settings: + master_key: sk-worker-b-1234 + database_url: os.environ/WORKER_B_DATABASE_URL + control_plane_url: "http://localhost:4000" +``` + +```bash +PROXY_BASE_URL=http://localhost:4002 litellm --config worker_b_config.yaml --port 4002 +``` + + + + +:::important +Each worker must have its own `master_key` and `database_url`. The whole point of this architecture is that workers are independent. +::: + +### 3. SSO Configuration (Optional) + +SSO is configured on the **control plane** instance the same way as a standard LiteLLM proxy. See the [SSO setup guide](./admin_ui_sso.md) for full instructions. + +If using SSO, make sure to register each worker URL and the control plane URL as allowed callback URLs in your SSO provider's dashboard. + +## How It Works + +### Login Flow + +1. User visits the control plane UI (`http://localhost:4000/ui`) +2. The login page shows a **worker selector** dropdown listing all registered workers +3. User selects a worker (e.g. "Worker A") and logs in with username/password or SSO +4. The UI authenticates against the **selected worker** using the `/v3/login` endpoint +5. On success, the UI stores the worker's JWT and points all subsequent API calls at the worker +6. The user can now manage keys, teams, models, and budgets on that worker, all from the control plane UI + +### Switching Workers + +Once logged in, users can switch workers from the **navbar dropdown** without leaving the UI. Switching redirects back to the login page to authenticate against the new worker. + +### Discovery + +The control plane exposes a `/.well-known/litellm-ui-config` endpoint that the UI reads on load. This endpoint returns: +- `is_control_plane: true` +- The list of workers with their IDs, names, and URLs + +This is how the login page knows to show the worker selector. + +## Local Testing + +To try this out locally, start each instance in a separate terminal: + +```bash +# Terminal 1: Control Plane +litellm --config cp_config.yaml --port 4000 + +# Terminal 2: Worker A +PROXY_BASE_URL=http://localhost:4001 litellm --config worker_a_config.yaml --port 4001 + +# Terminal 3: Worker B +PROXY_BASE_URL=http://localhost:4002 litellm --config worker_b_config.yaml --port 4002 +``` + +Then open `http://localhost:4000/ui`. You should see the worker selector on the login page. + +## Configuration Reference + +### Control Plane Settings + +| Field | Location | Description | +|---|---|---| +| `worker_registry` | Top-level config | List of worker instances | +| `worker_registry[].worker_id` | Required | Unique identifier for the worker | +| `worker_registry[].name` | Required | Display name shown in the UI | +| `worker_registry[].url` | Required | Full URL of the worker instance | + +### Worker Settings + +| Field | Location | Description | +|---|---|---| +| `general_settings.control_plane_url` | Required | URL of the control plane instance. Enables `/v3/login` and `/v3/login/exchange` endpoints on this worker. | +| `PROXY_BASE_URL` | Environment variable | The worker's own external URL. Required for SSO callback redirects. | + +## Related Documentation + +- [Standard Multi-Region Setup](./control_plane_and_data_plane.md) - shared-database architecture for admin/worker split +- [SSO Setup](./admin_ui_sso.md) - configuring SSO for the admin UI +- [Production Deployment](./prod.md) - production best practices diff --git a/docs/my-website/docs/proxy/ip_address.md b/docs/my-website/docs/proxy/ip_address.md index 8f042d9f183..4c469b81e0b 100644 --- a/docs/my-website/docs/proxy/ip_address.md +++ b/docs/my-website/docs/proxy/ip_address.md @@ -3,7 +3,7 @@ :::info -You need a LiteLLM License to unlock this feature. [Grab time](https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions), to get one today! +You need a LiteLLM License to unlock this feature. [Grab time](https://enterprise.litellm.ai/demo), to get one today! ::: diff --git a/docs/my-website/docs/proxy/keys_teams_router_settings.md b/docs/my-website/docs/proxy/keys_teams_router_settings.md index ec59e8f271b..6a1744ca951 100644 --- a/docs/my-website/docs/proxy/keys_teams_router_settings.md +++ b/docs/my-website/docs/proxy/keys_teams_router_settings.md @@ -146,5 +146,5 @@ Test new router settings on specific keys or teams before applying globally: - [Router Settings Reference](./config_settings.md#router_settings---reference) - Complete reference of all router settings - [Load Balancing](./load_balancing.md) - Learn about routing strategies and load balancing - [Reliability](./reliability.md) - Configure fallbacks, retries, and error handling -- [Keys](./keys.md) - Manage API keys and their settings -- [Teams](./teams.md) - Organize keys into teams +- [Keys](./virtual_keys.md) - Manage API keys and their settings +- [Teams](./multi_tenant_architecture.md) - Organize keys into teams diff --git a/docs/my-website/docs/proxy/load_balancing.md b/docs/my-website/docs/proxy/load_balancing.md index 5bf39d179f6..93f3d944340 100644 --- a/docs/my-website/docs/proxy/load_balancing.md +++ b/docs/my-website/docs/proxy/load_balancing.md @@ -324,17 +324,58 @@ model_list: litellm_params: model: azure/gpt-4-fallback api_key: os.environ/AZURE_API_KEY_2 - order: 2 # 👈 Used when order=1 is unavailable - -router_settings: - enable_pre_call_checks: true # 👈 Required for 'order' to work + order: 2 # 👈 Used when order=1 fails ``` -:::important -The `order` parameter requires `enable_pre_call_checks: true` in `router_settings`. -::: +### How order-based fallback works -If `order=1` deployment is unavailable (e.g., rate-limited), the router falls back to `order=2` deployments. +When a request to an `order=1` deployment fails (connection error, 404, 429, etc.), the router automatically tries `order=2` deployments, then `order=3`, and so on. Each order level gets its own set of retries before escalating to the next. + +If all order levels are exhausted, the router falls through to any configured [model-level fallbacks](#fallbacks). + +```yaml +model_list: + - model_name: gpt-4 + litellm_params: + model: azure/gpt-4-primary + api_key: os.environ/AZURE_API_KEY + order: 1 + + - model_name: gpt-4 + litellm_params: + model: azure/gpt-4-secondary + api_key: os.environ/AZURE_API_KEY_2 + order: 2 + + - model_name: gpt-4-fallback + litellm_params: + model: openai/gpt-4 + api_key: os.environ/OPENAI_API_KEY + +router_settings: + fallbacks: + - gpt-4: + - gpt-4-fallback # tried after all order levels fail +``` + +The fallback chain for the above config: `order=1` → `order=2` → `gpt-4-fallback`. + +For 429 (rate limit) errors specifically, the failed deployment is immediately placed on cooldown. If all `order=1` deployments are on cooldown, the router picks `order=2` deployments directly during retries without waiting for the fallback path. + +### Team-scoped models and legacy `model_aliases` {#team-scoped-models-and-legacy-model_aliases} + +Team-scoped deployments are identified by `model_info.team_id` and `model_info.team_public_model_name`. Requests should use the **public** model name; the router resolves all sibling deployments (same public name, different `api_base` / `order`, etc.) for routing, failover, and deployment `order`. + +For router internals: when a `team_id` is in scope, optimized lookups key off `(team_id, team_public_model_name)`. If code passes an internal deployment id (e.g. `model_name__`) instead of the public name, routing still works via the usual deployment-name paths, but the team-specific fast path applies only to the public name. + +**Legacy teams:** Older proxy versions could persist `model_aliases` on the team row mapping a public name to a single internal deployment id (`model_name__`). On each request, pre-call logic may still rewrite `model` to that internal name **before** routing, which collapses to one deployment and can make newer sibling deployments unreachable. + +**Migration options:** + +1. **Recommended for upgrades:** Set environment variable `LITELLM_ENABLE_TEAM_STALE_ALIAS_BYPASS=true` so that when sibling team deployments exist for the public name, the stale alias rewrite is skipped and team-scoped routing (including `order` and failover) applies. See the [Environment variables](./config_settings) table in the proxy settings doc. +2. **Data cleanup:** Remove obsolete `model_aliases` entries for team public names from the team record in the database so only `team_public_model_name` + team model list drive access. + +If a stale alias is detected and the bypass is **not** enabled, the proxy may emit a **one-time** warning in logs explaining that sibling deployments may be unreachable until the flag is set or aliases are cleaned up. ### When You'll See Load Balancing in Action @@ -352,7 +393,7 @@ If `order=1` deployment is unavailable (e.g., rate-limited), the router falls ba When load balancing OpenAI's Responses API across deployments with **different API keys** (e.g., different Azure regions or organizations), encrypted content items (like `rs_...` reasoning items) can only be decrypted by the originating API key. -**Solution:** Use the `encrypted_content_affinity` pre-call check to automatically route follow-up requests containing encrypted items to the correct deployment: +**Solution:** Use the `encrypted_content_affinity` pre-call check (requires LiteLLM >= 1.82.3) to automatically route follow-up requests containing encrypted items to the correct deployment: ```yaml model_list: diff --git a/docs/my-website/docs/proxy/logging.md b/docs/my-website/docs/proxy/logging.md index 74a79776fbd..2f81498799a 100644 --- a/docs/my-website/docs/proxy/logging.md +++ b/docs/my-website/docs/proxy/logging.md @@ -1109,7 +1109,7 @@ Log LLM Logs to [Google Cloud Storage Buckets](https://cloud.google.com/storage? :::info -✨ This is an Enterprise only feature [Get Started with Enterprise here](https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions) +✨ This is an Enterprise only feature [Get Started with Enterprise here](https://enterprise.litellm.ai/demo) ::: @@ -1194,7 +1194,7 @@ Log LLM Logs/SpendLogs to [Google Cloud Storage PubSub Topic](https://cloud.goog :::info -✨ This is an Enterprise only feature [Get Started with Enterprise here](https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions) +✨ This is an Enterprise only feature [Get Started with Enterprise here](https://enterprise.litellm.ai/demo) ::: @@ -1497,7 +1497,7 @@ Log LLM Logs to [Azure Data Lake Storage](https://learn.microsoft.com/en-us/azur :::info -✨ This is an Enterprise only feature [Get Started with Enterprise here](https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions) +✨ This is an Enterprise only feature [Get Started with Enterprise here](https://enterprise.litellm.ai/demo) ::: diff --git a/docs/my-website/docs/proxy/model_compare_ui.md b/docs/my-website/docs/proxy/model_compare_ui.md index bd6f5414224..ee0376ed2fa 100644 --- a/docs/my-website/docs/proxy/model_compare_ui.md +++ b/docs/my-website/docs/proxy/model_compare_ui.md @@ -187,7 +187,7 @@ Use tags and multiple comparisons to run structured A/B tests: ## Related Features -- [Playground Chat UI](./playground.md) - Single model testing interface +- [Playground Chat UI](./ui.md) - Single model testing interface - [Model Management](./model_management.md) - Configure and manage models -- [Guardrails](./guardrails.md) - Set up safety filters +- [Guardrails](./guardrails/quick_start.md) - Set up safety filters - [AI Hub](./ai_hub.md) - Share models and agents with your organization diff --git a/docs/my-website/docs/proxy/multiple_admins.md b/docs/my-website/docs/proxy/multiple_admins.md index 8d39674df19..83d0c5863df 100644 --- a/docs/my-website/docs/proxy/multiple_admins.md +++ b/docs/my-website/docs/proxy/multiple_admins.md @@ -20,7 +20,7 @@ LiteLLM tracks changes to the following entities and actions: :::tip -Requires Enterprise License, Get in touch with us [here](https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions) +Requires Enterprise License, Get in touch with us [here](https://enterprise.litellm.ai/demo) ::: @@ -56,6 +56,40 @@ On the LiteLLM UI, navigate to Logs -> Audit Logs. You should see the audit log /> +## Export Audit Logs to External Storage + +You can export audit logs to an external storage backend (e.g. S3) in addition to storing them in the database. Logs are batched and uploaded asynchronously, so they do not block your proxy requests. + +### S3 Example + +Add `audit_log_callbacks` and `s3_callback_params` to your `litellm_settings`: + +```yaml +litellm_settings: + store_audit_logs: true + audit_log_callbacks: ["s3_v2"] + s3_callback_params: + s3_bucket_name: my-audit-logs-bucket # AWS Bucket Name + s3_region_name: us-west-2 # AWS Region + s3_aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID + s3_aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY + s3_path: litellm-audit # [OPTIONAL] prefix path in the bucket +``` + +Audit logs are written as JSON files to: + +``` +s3:///audit_logs//_.json +# or, when s3_path is set: +s3:////audit_logs//_.json +``` + +:::info + +Both `store_audit_logs: true` and `audit_log_callbacks` must be set. If `store_audit_logs` is not enabled, the callbacks will not fire. + +::: + ## Advanced ### Attribute Management changes to Users diff --git a/docs/my-website/docs/proxy/oauth2.md b/docs/my-website/docs/proxy/oauth2.md index 41c4110e447..204a01538cc 100644 --- a/docs/my-website/docs/proxy/oauth2.md +++ b/docs/my-website/docs/proxy/oauth2.md @@ -4,7 +4,7 @@ Use this if you want to use an Oauth2.0 token to make `/chat`, `/embeddings` req :::info -This is an Enterprise Feature - [get in touch with us if you want a free trial to test if this feature meets your needs]((https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions)) +This is an Enterprise Feature - [get in touch with us if you want a free trial to test if this feature meets your needs]((https://enterprise.litellm.ai/demo)) ::: diff --git a/docs/my-website/docs/proxy/pass_through.md b/docs/my-website/docs/proxy/pass_through.md index f47d7064140..700bfb0831d 100644 --- a/docs/my-website/docs/proxy/pass_through.md +++ b/docs/my-website/docs/proxy/pass_through.md @@ -422,6 +422,5 @@ general_settings: [Community Discord 💭](https://discord.gg/wuPM9dRgDw) -Our numbers 📞 +1 (770) 8783-106 / ‭+1 (412) 618-6238‬ Our emails ✉️ ishaan@berri.ai / krrish@berri.ai diff --git a/docs/my-website/docs/proxy/prod.md b/docs/my-website/docs/proxy/prod.md index 26cb484cbe9..d40a0343106 100644 --- a/docs/my-website/docs/proxy/prod.md +++ b/docs/my-website/docs/proxy/prod.md @@ -47,7 +47,7 @@ export LITELLM_LOG="ERROR" :::info -Need Help or want dedicated support ? Talk to a founder [here]: (https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions) +Need Help or want dedicated support ? Talk to a founder [here]: (https://enterprise.litellm.ai/demo) ::: diff --git a/docs/my-website/docs/proxy/prompt_management.md b/docs/my-website/docs/proxy/prompt_management.md index 08307ba99ec..5a3e411e984 100644 --- a/docs/my-website/docs/proxy/prompt_management.md +++ b/docs/my-website/docs/proxy/prompt_management.md @@ -311,7 +311,7 @@ litellm_settings: 1. **At Startup**: When the proxy starts, it reads the `prompts` field from `config.yaml` 2. **Initialization**: Each prompt is initialized based on its `prompt_integration` type 3. **In-Memory Storage**: Prompts are stored in the `IN_MEMORY_PROMPT_REGISTRY` -4. **Access**: Use these prompts via the `/v1/chat/completions` endpoint with `prompt_id` in the request +4. **Access**: Use these prompts via `/v1/chat/completions` or `/v1/responses` with `prompt_id` in the request ### Using Config-Loaded Prompts @@ -331,6 +331,23 @@ curl -L -X POST 'http://0.0.0.0:4000/v1/chat/completions' \ }' ``` +You can also use the same `prompt_id` with the Responses API: + +```bash +curl -L -X POST 'http://0.0.0.0:4000/v1/responses' \ +-H 'Content-Type: application/json' \ +-H 'Authorization: Bearer sk-1234' \ +-d '{ + "model": "gpt-4o", + "prompt_id": "coding_assistant", + "prompt_variables": { + "language": "python", + "task": "create a web scraper" + }, + "input": [] +}' +``` + ### Prompt Schema Reference Each prompt in the `prompts` list requires: diff --git a/docs/my-website/docs/proxy/public_routes.md b/docs/my-website/docs/proxy/public_routes.md index d5f3941751f..e53548349dc 100644 --- a/docs/my-website/docs/proxy/public_routes.md +++ b/docs/my-website/docs/proxy/public_routes.md @@ -5,7 +5,7 @@ import TabItem from '@theme/TabItem'; :::info -Requires a LiteLLM Enterprise License. [Get a free trial](https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions). +Requires a LiteLLM Enterprise License. [Get a free trial](https://enterprise.litellm.ai/demo). ::: diff --git a/docs/my-website/docs/proxy/sync_anthropic_beta_headers.md b/docs/my-website/docs/proxy/sync_anthropic_beta_headers.md index e1645082d97..0373d20c879 100644 --- a/docs/my-website/docs/proxy/sync_anthropic_beta_headers.md +++ b/docs/my-website/docs/proxy/sync_anthropic_beta_headers.md @@ -125,4 +125,4 @@ curl -X DELETE "https://your-proxy-url/schedule/anthropic_beta_headers_reload" \ ## Related - [Model Cost Map Sync](./sync_models_github.md) - Auto-sync model pricing data -- [Anthropic Beta Headers](../completion/anthropic.md#beta-features) - Using Anthropic beta features +- [Anthropic Beta Headers](../providers/anthropic.md) - Using Anthropic beta features diff --git a/docs/my-website/docs/proxy/tag_routing.md b/docs/my-website/docs/proxy/tag_routing.md index a1ae52e5e45..57d16a59b54 100644 --- a/docs/my-website/docs/proxy/tag_routing.md +++ b/docs/my-website/docs/proxy/tag_routing.md @@ -315,7 +315,7 @@ LiteLLM Proxy supports team-based tag routing, allowing you to associate specifi :::info -This is an enterprise feature, [Contact us here to get a free trial](https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions) +This is an enterprise feature, [Contact us here to get a free trial](https://enterprise.litellm.ai/demo) ::: diff --git a/docs/my-website/docs/proxy/team_logging.md b/docs/my-website/docs/proxy/team_logging.md index 2ad7e2a4a8e..3f57d0d6d8b 100644 --- a/docs/my-website/docs/proxy/team_logging.md +++ b/docs/my-website/docs/proxy/team_logging.md @@ -26,7 +26,7 @@ Team 3 -> Disabled Logging (for GDPR compliance) :::info -✨ This is an Enterprise only feature [Get Started with Enterprise here](https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions) +✨ This is an Enterprise only feature [Get Started with Enterprise here](https://enterprise.litellm.ai/demo) ::: @@ -248,7 +248,7 @@ Use the `/key/generate` or `/key/update` endpoints to add logging callbacks to a :::info -✨ This is an Enterprise only feature [Get Started with Enterprise here](https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions) +✨ This is an Enterprise only feature [Get Started with Enterprise here](https://enterprise.litellm.ai/demo) ::: diff --git a/docs/my-website/docs/proxy/team_model_add.md b/docs/my-website/docs/proxy/team_model_add.md index 7db59a3300e..bb4238055b5 100644 --- a/docs/my-website/docs/proxy/team_model_add.md +++ b/docs/my-website/docs/proxy/team_model_add.md @@ -5,7 +5,7 @@ This is an Enterprise feature. [Enterprise Pricing](https://www.litellm.ai/#pricing) -[Contact us here to get a free trial](https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions) +[Contact us here to get a free trial](https://enterprise.litellm.ai/demo) ::: diff --git a/docs/my-website/docs/proxy/token_auth.md b/docs/my-website/docs/proxy/token_auth.md index 7364ae0fb56..bc6fde7c840 100644 --- a/docs/my-website/docs/proxy/token_auth.md +++ b/docs/my-website/docs/proxy/token_auth.md @@ -11,7 +11,7 @@ Use JWT's to auth admins / users / projects into the proxy. [Enterprise Pricing](https://www.litellm.ai/#pricing) -[Contact us here to get a free trial](https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions) +[Contact us here to get a free trial](https://enterprise.litellm.ai/demo) ::: diff --git a/docs/my-website/docs/proxy/ui_store_model_db_setting.md b/docs/my-website/docs/proxy/ui_store_model_db_setting.md index 5f860137d0f..4b0bc690f9a 100644 --- a/docs/my-website/docs/proxy/ui_store_model_db_setting.md +++ b/docs/my-website/docs/proxy/ui_store_model_db_setting.md @@ -87,6 +87,6 @@ Change the setting from the UI and have it take effect immediately—perfect for ## Related Documentation -- [Admin UI Overview](./ui_overview.md) – General guide to the LiteLLM Admin UI -- [Models and Endpoints](./models_and_endpoints.md) – Managing models and API endpoints +- [Admin UI Overview](./ui.md) – General guide to the LiteLLM Admin UI +- [Models and Endpoints](./model_management.md) – Managing models and API endpoints - [Config Settings](./config_settings.md) – `store_model_in_db` in `general_settings` diff --git a/docs/my-website/docs/proxy/user_onboarding.md b/docs/my-website/docs/proxy/user_onboarding.md index baa241d6cdf..ecbdc11db43 100644 --- a/docs/my-website/docs/proxy/user_onboarding.md +++ b/docs/my-website/docs/proxy/user_onboarding.md @@ -79,4 +79,4 @@ curl -X POST http://localhost:4000/v1/chat/completions \ ## See Also - [Proxy Quick Start](./quick_start.md) - [User Management](./users.md) -- [Key Management](./key_management.md) +- [Key Management](./virtual_keys.md) diff --git a/docs/my-website/docs/proxy_server.md b/docs/my-website/docs/proxy_server.md index e23d64e443b..7b6f15a604b 100644 --- a/docs/my-website/docs/proxy_server.md +++ b/docs/my-website/docs/proxy_server.md @@ -813,5 +813,4 @@ Thread Stats Avg Stdev Max +/- Stdev - [Schedule Demo 👋](https://calendly.com/d/4mp-gd3-k5k/berriai-1-1-onboarding-litellm-hosted-version) - [Community Discord 💭](https://discord.gg/wuPM9dRgDw) -- Our numbers 📞 +1 (770) 8783-106 / ‭+1 (412) 618-6238‬ - Our emails ✉️ ishaan@berri.ai / krrish@berri.ai diff --git a/docs/my-website/docs/reasoning_content.md b/docs/my-website/docs/reasoning_content.md index 8bf59f66a33..6e6a30cdb49 100644 --- a/docs/my-website/docs/reasoning_content.md +++ b/docs/my-website/docs/reasoning_content.md @@ -13,6 +13,7 @@ Supported Providers: - Deepseek (`deepseek/`) - Anthropic API (`anthropic/`) - Bedrock (Anthropic + Deepseek + GPT-OSS) (`bedrock/`) +- OpenAI Responses API (`openai/responses/`) - Vertex AI (Anthropic) (`vertexai/`) - OpenRouter (`openrouter/`) - XAI (`xai/`) @@ -594,9 +595,26 @@ Expected Response :::tip gpt-5.4: reasoning_effort + function tools -LiteLLM drops `reasoning_effort` from `gpt-5.4` requests to `litellm.completion()` that include tools, since that combination is supported in the Responses API. +When `gpt-5.4+` requests to `litellm.completion()` include both `reasoning_effort` and `tools`, LiteLLM **automatically routes** the request through the Responses API bridge. This works for both **OpenAI** (`openai/gpt-5.4`) and **Azure** (`azure/gpt-5.4`) providers — no extra configuration needed. -If you need reasoning **and** tools together, use `openai/responses/gpt-5.4` to route through the Responses API instead. See [Responses API Bridge](/docs/providers/openai#openai-chat-completion-to-responses-api-bridge) for details. +You can also route explicitly via `openai/responses/gpt-5.4` or `azure/responses/gpt-5.4`. See [Responses API Bridge](/docs/providers/openai#openai-chat-completion-to-responses-api-bridge) for details. + +**Azure custom deployment names:** Auto-routing relies on the deployment name matching the `gpt-5.4*` pattern. If you use a custom deployment name (e.g. `"my-reasoning-model"`), enable routing via: + +**SDK:** +```python +litellm.completion(model="azure/responses/my-reasoning-model", ...) +``` + +**Proxy config:** +```yaml +model_list: + - model_name: my-reasoning-model + litellm_params: + model: azure/my-reasoning-model + model_info: + mode: responses +``` ::: @@ -683,3 +701,69 @@ response = litellm.completion( reasoning_effort={"effort": "low", "summary": "detailed"}, # Explicit control ) ``` + +### Summary Preservation via `/v1/messages` Adapter + +When using the Anthropic `/v1/messages` adapter to route non-Claude models (e.g., `openai/gpt-5.1`), the `thinking.summary` value is preserved and forwarded to the downstream provider. For example: + +```python +import litellm + +response = await litellm.anthropic.messages.acreate( + model="openai/gpt-5.1", + messages=[{"role": "user", "content": "Hello"}], + max_tokens=8096, + thinking={"type": "enabled", "budget_tokens": 5000, "summary": "concise"}, +) +# The summary="concise" is preserved when routing to OpenAI's Responses API +``` + +### Enabling Default Summary Injection for `/v1/messages` Adapter + +When the Anthropic `/v1/messages` adapter translates `thinking` parameters to OpenAI `reasoning_effort` for non-Claude models, you can opt-in to automatic `summary="detailed"` injection using the `reasoning_auto_summary` flag. This ensures that reasoning text is returned in the response (matching the Anthropic thinking behavior). + +To **enable** this default injection, use the `reasoning_auto_summary` flag: + + + + +```python +import litellm + +# Enable default summary="detailed" injection +litellm.reasoning_auto_summary = True + +response = await litellm.anthropic.messages.acreate( + model="openai/gpt-5.1", + messages=[{"role": "user", "content": "Hello"}], + max_tokens=8096, + thinking={"type": "enabled", "budget_tokens": 5000}, +) +# summary="detailed" will be automatically added to reasoning_effort +``` + + + + + +```bash +export LITELLM_REASONING_AUTO_SUMMARY=true +``` + + + + + +```yaml +litellm_settings: + reasoning_auto_summary: true +``` + + + + +:::info + +This flag only affects the automatic injection of `summary="detailed"` when no user-provided summary is present. If you explicitly pass `thinking.summary` (e.g., `"concise"` or `"auto"`), your value is always preserved regardless of this flag. + +::: diff --git a/docs/my-website/docs/response_api.md b/docs/my-website/docs/response_api.md index fb55ae9f9d0..0c428000c72 100644 --- a/docs/my-website/docs/response_api.md +++ b/docs/my-website/docs/response_api.md @@ -1160,12 +1160,12 @@ follow_up = await router.aresponses( To enable session continuity for Responses API in your LiteLLM proxy, set `optional_pre_call_checks` in your proxy config.yaml. - `responses_api_deployment_check`: high priority routing when `previous_response_id` is provided -- `encrypted_content_affinity`: **[Recommended]** content-aware routing for encrypted items (e.g., `rs_...` reasoning items) +- `encrypted_content_affinity`: **[Recommended]** content-aware routing for encrypted items (e.g., `rs_...` reasoning items) (**requires LiteLLM >= 1.82.3**) - `session_affinity`: sticky sessions based on session id (takes priority over `deployment_affinity`) - `deployment_affinity`: sticky sessions based on user key (applies even without `previous_response_id`) :::tip Recommended: Use `encrypted_content_affinity` -For Responses API with load balancing across deployments with **different API keys**, use `encrypted_content_affinity` instead of `deployment_affinity`. It only pins requests that contain encrypted content, avoiding quota reduction while preventing `invalid_encrypted_content` errors. +For Responses API with load balancing across deployments with **different API keys**, use `encrypted_content_affinity` instead of `deployment_affinity`. It only pins requests that contain encrypted content, avoiding quota reduction while preventing `invalid_encrypted_content` errors. (Requires LiteLLM >= 1.82.3.) ::: Notes: @@ -1364,6 +1364,85 @@ litellm --config config.yaml | `deployment_affinity` | Simple sticky sessions | All requests from same API key | ❌ Reduces quota by # of users | +## Per-Model-Group Affinity Configuration + +By default, `optional_pre_call_checks` applies globally to all model groups. Use `model_group_affinity_config` when you want different affinity behavior per model group — for example, enabling stickiness only for models spread across providers (Azure + Bedrock) while leaving single-provider groups free to load-balance. + +Groups not listed fall back to the global `optional_pre_call_checks` settings. + + + + +```python +router = litellm.Router( + model_list=[ + { + "model_name": "gpt-4", + "litellm_params": {"model": "azure/gpt-4", "api_key": "...", "api_base": "https://endpoint1.openai.azure.com"}, + }, + { + "model_name": "gpt-4", + "litellm_params": {"model": "bedrock/anthropic.claude-v2", "aws_region_name": "us-east-1"}, + }, + { + "model_name": "text-embedding-ada-002", + "litellm_params": {"model": "azure/text-embedding-ada-002", "api_key": "...", "api_base": "https://endpoint1.openai.azure.com"}, + }, + { + "model_name": "text-embedding-ada-002", + "litellm_params": {"model": "azure/text-embedding-ada-002", "api_key": "...", "api_base": "https://endpoint2.openai.azure.com"}, + }, + ], + # gpt-4: cross-provider (Azure + Bedrock) — enable deployment affinity + # text-embedding-ada-002: same provider — no affinity, let it load balance freely + model_group_affinity_config={ + "gpt-4": ["deployment_affinity", "responses_api_deployment_check"], + }, +) +``` + + + + +```yaml title="config.yaml" +model_list: + - model_name: gpt-4 + litellm_params: + model: azure/gpt-4 + api_key: os.environ/AZURE_API_KEY_1 + api_base: https://endpoint1.openai.azure.com + + - model_name: gpt-4 + litellm_params: + model: bedrock/anthropic.claude-v2 + aws_region_name: us-east-1 + + - model_name: text-embedding-ada-002 + litellm_params: + model: azure/text-embedding-ada-002 + api_key: os.environ/AZURE_API_KEY_1 + api_base: https://endpoint1.openai.azure.com + + - model_name: text-embedding-ada-002 + litellm_params: + model: azure/text-embedding-ada-002 + api_key: os.environ/AZURE_API_KEY_2 + api_base: https://endpoint2.openai.azure.com + +router_settings: + # gpt-4: cross-provider — enable stickiness + # text-embedding-ada-002: not listed — load balances freely + model_group_affinity_config: + "gpt-4": + - deployment_affinity + - responses_api_deployment_check +``` + + + + +**Supported values:** `deployment_affinity`, `responses_api_deployment_check`, `session_affinity` + ## Calling non-Responses API endpoints (`/responses` to `/chat/completions` Bridge) LiteLLM allows you to call non-Responses API models via a bridge to LiteLLM's `/chat/completions` endpoint. This is useful for calling Anthropic, Gemini and even non-Responses API OpenAI models. @@ -1556,6 +1635,12 @@ curl -X POST "http://localhost:4000/v1/responses" \ }' ``` +## File Search (Vector Stores) + +For full `file_search` usage (native + emulated fallback), SDK/Proxy examples, architecture diagram, and Q&A, see: + +- [`File Search in the Responses API — E2E Testing Guide`](/docs/tutorials/file_search_responses_api) + ## Session Management LiteLLM Proxy supports session management for all supported models. This allows you to store and fetch conversation history (state) in LiteLLM Proxy. diff --git a/docs/my-website/docs/routing.md b/docs/my-website/docs/routing.md index 67e7f681147..5aa655ae212 100644 --- a/docs/my-website/docs/routing.md +++ b/docs/my-website/docs/routing.md @@ -842,6 +842,8 @@ Traffic mirroring allows you to "mimic" production traffic to a secondary (silen Set `order` in `litellm_params` to prioritize deployments. Lower values = higher priority. When multiple deployments share the same `order`, the routing strategy picks among them. +When a request to an `order=1` deployment fails (connection error, 404, 429, etc.), the router automatically tries `order=2` deployments, then `order=3`, and so on. Each order level gets its own set of retries before escalating to the next. If all order levels are exhausted, the router falls through to any configured [fallbacks](#fallbacks). + @@ -862,18 +864,14 @@ model_list = [ "litellm_params": { "model": "azure/gpt-4-fallback", "api_key": os.getenv("AZURE_API_KEY_2"), - "order": 2, # 👈 Used when order=1 is unavailable + "order": 2, # 👈 Tried when order=1 fails }, }, ] -router = Router(model_list=model_list, enable_pre_call_checks=True) # 👈 Required for 'order' to work +router = Router(model_list=model_list) ``` -:::important -The `order` parameter requires `enable_pre_call_checks=True` to be set on the Router. -::: - @@ -889,10 +887,7 @@ model_list: litellm_params: model: azure/gpt-4-fallback api_key: os.environ/AZURE_API_KEY_2 - order: 2 # 👈 Used when order=1 is unavailable - -router_settings: - enable_pre_call_checks: true # 👈 Required for 'order' to work + order: 2 # 👈 Tried when order=1 fails ``` diff --git a/docs/my-website/docs/secret.md b/docs/my-website/docs/secret.md index c5c80311475..57f576fd56a 100644 --- a/docs/my-website/docs/secret.md +++ b/docs/my-website/docs/secret.md @@ -6,7 +6,7 @@ [Enterprise Pricing](https://www.litellm.ai/#pricing) -[Contact us here to get a free trial](https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions) +[Contact us here to get a free trial](https://enterprise.litellm.ai/demo) ::: diff --git a/docs/my-website/docs/secret_managers/aws_kms.md b/docs/my-website/docs/secret_managers/aws_kms.md index 7f69d91fe87..806223a2539 100644 --- a/docs/my-website/docs/secret_managers/aws_kms.md +++ b/docs/my-website/docs/secret_managers/aws_kms.md @@ -6,7 +6,7 @@ [Enterprise Pricing](https://www.litellm.ai/#pricing) -[Contact us here to get a free trial](https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions) +[Contact us here to get a free trial](https://enterprise.litellm.ai/demo) ::: diff --git a/docs/my-website/docs/secret_managers/aws_secret_manager.md b/docs/my-website/docs/secret_managers/aws_secret_manager.md index c49797a15dd..a7e24ea69ae 100644 --- a/docs/my-website/docs/secret_managers/aws_secret_manager.md +++ b/docs/my-website/docs/secret_managers/aws_secret_manager.md @@ -9,7 +9,7 @@ import TabItem from '@theme/TabItem'; [Enterprise Pricing](https://www.litellm.ai/#pricing) -[Contact us here to get a free trial](https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions) +[Contact us here to get a free trial](https://enterprise.litellm.ai/demo) ::: diff --git a/docs/my-website/docs/secret_managers/azure_key_vault.md b/docs/my-website/docs/secret_managers/azure_key_vault.md index 81aeaa32159..4ea53d2ea9e 100644 --- a/docs/my-website/docs/secret_managers/azure_key_vault.md +++ b/docs/my-website/docs/secret_managers/azure_key_vault.md @@ -6,7 +6,7 @@ [Enterprise Pricing](https://www.litellm.ai/#pricing) -[Contact us here to get a free trial](https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions) +[Contact us here to get a free trial](https://enterprise.litellm.ai/demo) ::: diff --git a/docs/my-website/docs/secret_managers/cyberark.md b/docs/my-website/docs/secret_managers/cyberark.md index 0a17c0afc30..cd7c0ea5d25 100644 --- a/docs/my-website/docs/secret_managers/cyberark.md +++ b/docs/my-website/docs/secret_managers/cyberark.md @@ -8,7 +8,7 @@ import Image from '@theme/IdealImage'; [Enterprise Pricing](https://www.litellm.ai/#pricing) -[Contact us here to get a free trial](https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions) +[Contact us here to get a free trial](https://enterprise.litellm.ai/demo) ::: diff --git a/docs/my-website/docs/secret_managers/google_kms.md b/docs/my-website/docs/secret_managers/google_kms.md index 31fd6195bdb..152ecbaae80 100644 --- a/docs/my-website/docs/secret_managers/google_kms.md +++ b/docs/my-website/docs/secret_managers/google_kms.md @@ -6,7 +6,7 @@ [Enterprise Pricing](https://www.litellm.ai/#pricing) -[Contact us here to get a free trial](https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions) +[Contact us here to get a free trial](https://enterprise.litellm.ai/demo) ::: diff --git a/docs/my-website/docs/secret_managers/google_secret_manager.md b/docs/my-website/docs/secret_managers/google_secret_manager.md index 81878b7e398..f3e7367e8a4 100644 --- a/docs/my-website/docs/secret_managers/google_secret_manager.md +++ b/docs/my-website/docs/secret_managers/google_secret_manager.md @@ -6,7 +6,7 @@ [Enterprise Pricing](https://www.litellm.ai/#pricing) -[Contact us here to get a free trial](https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions) +[Contact us here to get a free trial](https://enterprise.litellm.ai/demo) ::: diff --git a/docs/my-website/docs/secret_managers/hashicorp_vault.md b/docs/my-website/docs/secret_managers/hashicorp_vault.md index 52d9b556200..11e25e88a7d 100644 --- a/docs/my-website/docs/secret_managers/hashicorp_vault.md +++ b/docs/my-website/docs/secret_managers/hashicorp_vault.md @@ -8,7 +8,7 @@ import Image from '@theme/IdealImage'; [Enterprise Pricing](https://www.litellm.ai/#pricing) -[Contact us here to get a free trial](https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions) +[Contact us here to get a free trial](https://enterprise.litellm.ai/demo) ::: diff --git a/docs/my-website/docs/secret_managers/overview.md b/docs/my-website/docs/secret_managers/overview.md index bf7386ab89c..f02362f4932 100644 --- a/docs/my-website/docs/secret_managers/overview.md +++ b/docs/my-website/docs/secret_managers/overview.md @@ -8,7 +8,7 @@ import Image from '@theme/IdealImage'; [Enterprise Pricing](https://www.litellm.ai/#pricing) -[Contact us here to get a free trial](https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions) +[Contact us here to get a free trial](https://enterprise.litellm.ai/demo) ::: diff --git a/docs/my-website/docs/troubleshoot.md b/docs/my-website/docs/troubleshoot.md index 1539e1959f7..1afa35df8e4 100644 --- a/docs/my-website/docs/troubleshoot.md +++ b/docs/my-website/docs/troubleshoot.md @@ -50,7 +50,6 @@ Full error logs, stack traces, and any images from service metrics (CPU, memory, [Community Discord 💭](https://discord.gg/wuPM9dRgDw) [Community Slack 💭](https://www.litellm.ai/support) -Our numbers 📞 +1 (770) 8783-106 / +1 (412) 618-6238 Our emails ✉️ ishaan@berri.ai / krrish@berri.ai diff --git a/docs/my-website/docs/tutorials/claude_code_plugin_marketplace.md b/docs/my-website/docs/tutorials/claude_code_plugin_marketplace.md index 9d93c717c4f..d8175f51aca 100644 --- a/docs/my-website/docs/tutorials/claude_code_plugin_marketplace.md +++ b/docs/my-website/docs/tutorials/claude_code_plugin_marketplace.md @@ -37,7 +37,7 @@ Click **+ Add New Plugin** to register a plugin in your marketplace. Enter the plugin information: - **Name**: Plugin identifier (kebab-case, e.g., `my-plugin`) -- **Source Type**: Choose GitHub or URL +- **Source Type**: Choose GitHub, Git URL, or Git Subdir - **Repository/URL**: The git source (e.g., `org/repo` for GitHub) - **Version**: Semantic version (optional) - **Description**: What the plugin does @@ -216,6 +216,22 @@ curl -X DELETE http://localhost:4000/claude-code/plugins/my-plugin \ Use this format for GitLab, Bitbucket, or self-hosted git repositories. + + + +```json +{ + "name": "my-plugin", + "source": { + "source": "git-subdir", + "url": "https://github.com/org/repo.git", + "path": "plugins/my-plugin" + } +} +``` + +Use this format when your plugin lives in a subdirectory of a git repository. The `path` field must be a relative path of slash-separated segments (alphanumeric, dots, hyphens, underscores only). + diff --git a/docs/my-website/docs/tutorials/compare_llms.md b/docs/my-website/docs/tutorials/compare_llms.md index 02877b46607..0252263a16e 100644 --- a/docs/my-website/docs/tutorials/compare_llms.md +++ b/docs/my-website/docs/tutorials/compare_llms.md @@ -82,7 +82,7 @@ Benchmark Results for 'When will BerriAI IPO?': +-----------------+----------------------------------------------------------------------------------+---------------------------+------------+ ``` ## Support -**🤝 Schedule a 1-on-1 Session:** Book a [1-on-1 session](https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions) with Krrish and Ishaan, the founders, to discuss any issues, provide feedback, or explore how we can improve LiteLLM for you. +**🤝 Schedule a 1-on-1 Session:** Book a [1-on-1 session](https://enterprise.litellm.ai/demo) with Krrish and Ishaan, the founders, to discuss any issues, provide feedback, or explore how we can improve LiteLLM for you. B[LiteLLM Responses API] + B --> C{Provider supports native file_search?} + + C -->|Yes| D[Native passthrough path] + D --> D1[Decode unified vector_store_id if needed] + D1 --> D2[Forward request to provider unchanged] + D2 --> D3[Provider performs file_search] + D3 --> Z[OpenAI-compatible output] + + C -->|No| E[Emulated fallback path] + E --> E1[Convert file_search to litellm_file_search function tool] + E1 --> E2[First model call returns tool call with one or more queries] + E2 --> E3[LiteLLM executes vector search for each query] + E3 --> E4[Second model call with tool_result context] + E4 --> E5[Synthesize file_search_call + message + citations] + E5 --> Z[OpenAI-compatible output] +``` + + + +## Prerequisites + +```bash +pip install 'litellm[proxy]' +export OPENAI_API_KEY="sk-..." # for native path +export ANTHROPIC_API_KEY="sk-ant-..." # for emulated path +``` + + + +## Example response shape + +## Validating the Output Format + +Regardless of which path ran, the response always follows the OpenAI Responses API format: + +```json +{ + "output": [ + { + "type": "file_search_call", + "id": "fs_abc123", + "status": "completed", + "queries": ["What does LiteLLM support?"], + "search_results": null + }, + { + "type": "message", + "role": "assistant", + "content": [ + { + "type": "output_text", + "text": "LiteLLM is a unified interface...", + "annotations": [ + { + "type": "file_citation", + "index": 150, + "file_id": "file-xxxx", + "filename": "knowledge.txt" + } + ] + } + ] + } + ] +} +``` + +**Validation script:** + +```python showLineNumbers title="Validate response structure" +def validate_file_search_response(response): + """Assert that response follows OpenAI file_search output format.""" + output = response.output + assert len(output) >= 2, "Expected at least 2 output items" + + # First item: file_search_call + fs_call = output[0] + fs_type = fs_call["type"] if isinstance(fs_call, dict) else fs_call.type + assert fs_type == "file_search_call", f"Expected file_search_call, got {fs_type}" + + fs_status = fs_call["status"] if isinstance(fs_call, dict) else fs_call.status + assert fs_status == "completed" + + # Second item: message + msg = output[1] + msg_type = msg["type"] if isinstance(msg, dict) else msg.type + assert msg_type == "message" + + content = msg["content"] if isinstance(msg, dict) else msg.content + assert len(content) > 0 + text_block = content[0] + text = text_block["text"] if isinstance(text_block, dict) else text_block.text + assert isinstance(text, str) and len(text) > 0 + + print("✅ Response structure valid") + print(f" Queries: {fs_call['queries'] if isinstance(fs_call, dict) else fs_call.queries}") + print(f" Answer length: {len(text)} chars") + annotations = text_block["annotations"] if isinstance(text_block, dict) else text_block.annotations + print(f" Citations: {len(annotations)}") + +validate_file_search_response(response) +``` + + + +## Q&A + +- **Why do I see `UnsupportedParamsError`?** This usually means `file_search` was passed to a provider that does not support it natively and emulation could not route correctly. Check: + - The model string is valid (for example, `anthropic/claude-sonnet-4-5`). + - `custom_llm_provider` resolves correctly so LiteLLM can load the provider config. +- **Why does vector search return no results?** Common causes: + - The vector store ID is wrong or has no files attached. + - In LiteLLM-managed stores, file ingestion is not complete (`status != completed`). + - The query is too narrow; try a broader query. +- **Why am I getting `403 Access denied` on vector store calls?** The caller does not have access to that vector store. + - The store may belong to another team. + - Use an admin/proxy key if your setup requires cross-team access. +- **Why are `annotations` empty in emulated mode?** `file_citation` annotations require `file_id` metadata in search results. If your vector backend does not return file-level metadata, the answer text is still generated but citations can be empty. + + + +## What to check next + +- [File Search reference in Responses API docs](/docs/response_api#file-search-vector-stores) — full API reference +- [Vector Store management](/docs/vector_store_files) — create and manage vector stores +- [Managed vector stores](/docs/providers/bedrock_vector_store) — provider-specific setup diff --git a/docs/my-website/docs/tutorials/index.md b/docs/my-website/docs/tutorials/index.md new file mode 100644 index 00000000000..7f80cc760ee --- /dev/null +++ b/docs/my-website/docs/tutorials/index.md @@ -0,0 +1,98 @@ +--- +title: Tutorials +sidebar_label: Overview +--- + +import NavigationCards from '@site/src/components/NavigationCards'; + +**Tutorials** are step-by-step walkthroughs for integrating LiteLLM with external tools, frameworks, and services — or building complete end-to-end workflows. + +> Need help choosing the right path before you start? See [Learn →](/docs/learn) + +--- + +## Getting Started + + + +--- + +## Integrations + + + +--- + +## Proxy + + + +--- + +## Observability & Evaluation + + diff --git a/docs/my-website/docs/tutorials/installation.md b/docs/my-website/docs/tutorials/installation.md index ecaed0bec9d..cf39c55bee6 100644 --- a/docs/my-website/docs/tutorials/installation.md +++ b/docs/my-website/docs/tutorials/installation.md @@ -1,7 +1,3 @@ ---- -displayed_sidebar: tutorialSidebar ---- - # Set up environment Let's get the necessary keys to set up our demo environment. @@ -11,7 +7,5 @@ Every LLM provider needs API keys (e.g. `OPENAI_API_KEY`). You can get API keys Let's get them for our demo! **OpenAI**: https://platform.openai.com/account/api-keys -**Cohere**: https://dashboard.cohere.com/welcome/login?redirect_uri=%2Fapi-keys (no credit card required) +**Cohere**: https://dashboard.cohere.com/welcome/login?redirect_uri=%2Fapi-keys (no credit card required) **AI21**: https://studio.ai21.com/account/api-key (no credit card required) - - diff --git a/docs/my-website/docs/tutorials/opencode_integration.md b/docs/my-website/docs/tutorials/opencode_integration.md index e55367833f2..35e00a1de50 100644 --- a/docs/my-website/docs/tutorials/opencode_integration.md +++ b/docs/my-website/docs/tutorials/opencode_integration.md @@ -253,7 +253,7 @@ model_list: litellm_params: model: openai/gpt-4 api_key: os.environ/OPENAI_API_KEY - + - model_name: gpt-4o litellm_params: model: openai/gpt-4o @@ -264,7 +264,7 @@ model_list: litellm_params: model: anthropic/claude-3-5-sonnet-20241022 api_key: os.environ/ANTHROPIC_API_KEY - + # DeepSeek models - model_name: deepseek-chat litellm_params: @@ -272,6 +272,19 @@ model_list: api_key: os.environ/DEEPSEEK_API_KEY ``` +### Dropping OpenCode-specific parameters + +OpenCode sends a `reasoningSummary` parameter with reasoning-capable models such as `gpt-5`. This parameter is not supported by the Chat Completions API and will cause errors. Add `additional_drop_params` to every model entry in your `model_list` that will receive requests from OpenCode with reasoning enabled: + +```yaml +model_list: + - model_name: gpt-5 + litellm_params: + model: openai/gpt-5 + api_key: os.environ/OPENAI_API_KEY + additional_drop_params: ["reasoningSummary"] +``` + ## Troubleshooting **OpenCode not connecting:** @@ -294,6 +307,16 @@ model_list: - Validate JSON syntax using a JSON validator - Ensure the `$schema` URL is accessible +**`Unknown parameter: 'reasoningSummary'` error:** +- OpenCode sends a `reasoningSummary` parameter that is not supported by the Chat Completions API. Add `additional_drop_params: ["reasoningSummary"]` to each affected model entry in your `litellm_params`: + ```yaml + - model_name: gpt-5 + litellm_params: + model: openai/gpt-5 + api_key: os.environ/OPENAI_API_KEY + additional_drop_params: ["reasoningSummary"] + ``` + ## Tips - Add more models to the config as needed - they'll appear in `/models` diff --git a/docs/my-website/docs/tutorials/vertex_ai_pay_go.md b/docs/my-website/docs/tutorials/vertex_ai_pay_go.md new file mode 100644 index 00000000000..87197e5bad5 --- /dev/null +++ b/docs/my-website/docs/tutorials/vertex_ai_pay_go.md @@ -0,0 +1,151 @@ +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +# Vertex AI PayGo and Priority + +## Priority PayGo + +LiteLLM supports Priority PayGo. +Send a priority header, get priority queueing, and pay priority token rates. + +:::info Which models support Priority PayGo? +As of this writing: `gemini/gemini-2.5-pro`, `vertex_ai/gemini-3-pro-preview`, `vertex_ai/gemini-3.1-pro-preview`, `vertex_ai/gemini-3-flash-preview`, and their variants. +Check `supports_service_tier: true` in LiteLLM's [model pricing JSON](https://github.com/BerriAI/litellm/blob/main/model_prices_and_context_window.json). +::: + +### Send a priority request + +Use this header: + +`X-Vertex-AI-LLM-Shared-Request-Type: priority` + + + + +```python +import litellm + +response = litellm.completion( + model="vertex_ai/gemini-3-pro-preview", + messages=[{"role": "user", "content": "Summarize the Gettysburg Address."}], + vertex_project="YOUR_PROJECT_ID", + vertex_location="us-central1", + extra_headers={"X-Vertex-AI-LLM-Shared-Request-Type": "priority"}, +) + +print(response.choices[0].message.content) +``` + + + + +```yaml title="config.yaml" +model_list: + - model_name: gemini-priority + litellm_params: + model: vertex_ai/gemini-3-pro-preview + vertex_project: "YOUR_PROJECT_ID" + vertex_location: "us-central1" + vertex_credentials: os.environ/GOOGLE_APPLICATION_CREDENTIALS + extra_headers: + X-Vertex-AI-LLM-Shared-Request-Type: priority +``` + +```bash +curl http://localhost:4000/v1/chat/completions \ + -H "Authorization: Bearer sk-your-key" \ + -H "Content-Type: application/json" \ + -d '{"model": "gemini-priority", "messages": [{"role": "user", "content": "Hello"}]}' +``` + + + + +Use `x-pass-` so LiteLLM forwards provider-specific headers. + +```bash +MODEL_ID="gemini-3-pro-preview-0325" +PROJECT_ID="YOUR_PROJECT_ID" + +curl -X POST \ + "${LITELLM_PROXY_BASE_URL}/vertex_ai/v1/projects/${PROJECT_ID}/locations/global/publishers/google/models/${MODEL_ID}:generateContent" \ + -H "Authorization: Bearer sk-your-litellm-key" \ + -H "Content-Type: application/json" \ + -H "x-pass-X-Vertex-AI-LLM-Shared-Request-Type: priority" \ + -d '{"contents": [{"role": "user", "parts": [{"text": "Hello!"}]}]}' +``` + + + + +### How cost tracking works + +![Vertex AI Priority PayGo Cost Tracking Flow](/img/vertex_cost_tracking_flow.svg) + +**`trafficType` → `service_tier` mapping** + +| `usageMetadata.trafficType` | `service_tier` | Pricing keys used | +|---|---|---| +| `ON_DEMAND` | `None` | `input_cost_per_token` | +| `ON_DEMAND_PRIORITY` | `"priority"` | `input_cost_per_token_priority` | +| `FLEX` / `BATCH` | `"flex"` | `input_cost_per_token_flex` | + +If a tier-specific key is missing, LiteLLM falls back to standard pricing keys. + +--- + +## Standard PayGo vs Provisioned Throughput + +This is a different header from priority routing: + +| Header value | Behavior | +|---|---| +| `X-Vertex-AI-LLM-Request-Type: shared` | Force standard PayGo (bypass PT) | +| `X-Vertex-AI-LLM-Request-Type: dedicated` | Force Provisioned Throughput only (`429` if exhausted) | + +### Native route example + +```python +import litellm + +response = litellm.completion( + model="vertex_ai/gemini-2.0-flash", + messages=[{"role": "user", "content": "Hello!"}], + vertex_project="YOUR_PROJECT_ID", + vertex_location="us-central1", + extra_headers={"X-Vertex-AI-LLM-Request-Type": "shared"}, +) +``` + +### Pass-through example + +```bash +MODEL_ID="gemini-2.0-flash-001" +PROJECT_ID="YOUR_PROJECT_ID" + +curl -X POST \ + "${LITELLM_PROXY_BASE_URL}/vertex_ai/v1/projects/${PROJECT_ID}/locations/global/publishers/google/models/${MODEL_ID}:generateContent" \ + -H "Authorization: Bearer sk-your-litellm-key" \ + -H "Content-Type: application/json" \ + -H "x-pass-X-Vertex-AI-LLM-Request-Type: shared" \ + -d '{ + "contents": [{"role": "user", "parts": [{"text": "Hello!"}]}] + }' +``` + +--- + +## Troubleshooting + +**Q: What does `403 Permission denied` or `IAM_PERMISSION_DENIED` mean?** +A: The service account or Application Default Credentials (ADC) user does not have the `roles/aiplatform.user` role. To resolve this, re-run the `gcloud projects add-iam-policy-binding`. + +**Q: What should I do if I get a `429 Quota exceeded` error?** +A: This means you've hit the per-region QPM (queries per minute) or TPM (tokens per minute) quota. You can: +- Request a quota increase from the [GCP Quotas console](https://console.cloud.google.com/iam-admin/quotas) +- Add more regions to your LiteLLM configuration for load balancing +- Upgrade to [Provisioned Throughput](https://cloud.google.com/vertex-ai/generative-ai/docs/provisioned-throughput) for guaranteed capacity + +**Q: How do I fix the `VERTEXAI_PROJECT not set` error?** +A: Either pass the `vertex_project` parameter explicitly in your LiteLLM call, or set the `VERTEXAI_PROJECT` environment variable before running your code. + diff --git a/docs/my-website/docs/vertex_batch_passthrough.md b/docs/my-website/docs/vertex_batch_passthrough.md index 3203d7d792a..17ffc1e6dbc 100644 --- a/docs/my-website/docs/vertex_batch_passthrough.md +++ b/docs/my-website/docs/vertex_batch_passthrough.md @@ -155,6 +155,6 @@ Common error scenarios and their solutions: ## Related Documentation -- [Vertex AI Provider Documentation](./vertex.md) -- [General Batches API Documentation](../batches.md) -- [Cost Tracking and Monitoring](../observability/telemetry.md) +- [Vertex AI Provider Documentation](./providers/vertex.md) +- [General Batches API Documentation](./batches.md) +- [Cost Tracking and Monitoring](./observability/telemetry.md) diff --git a/docs/my-website/docusaurus.config.js b/docs/my-website/docusaurus.config.js index 32d5d800b71..9e1fae6a34f 100644 --- a/docs/my-website/docusaurus.config.js +++ b/docs/my-website/docusaurus.config.js @@ -2,9 +2,9 @@ // Note: type annotations allow type checking and IDEs autocompletion // @ts-ignore -const lightCodeTheme = require('prism-react-renderer/themes/github'); +const lightCodeTheme = require('prism-react-renderer/themes/vsLight'); // @ts-ignore -const darkCodeTheme = require('prism-react-renderer/themes/dracula'); +const darkCodeTheme = require('prism-react-renderer/themes/nightOwl'); const inkeepConfig = { baseSettings: { @@ -87,18 +87,88 @@ const config = { }, ], [ - '@docusaurus/plugin-content-blog', + '@docusaurus/plugin-content-docs', { - id: 'release_notes', + id: 'release-notes', path: './release_notes', routeBasePath: 'release_notes', - blogTitle: 'Release Notes', - blogSidebarTitle: 'Releases', - blogSidebarCount: 'ALL', - postsPerPage: 'ALL', - showReadingTime: false, - sortPosts: 'descending', - include: ['**/*.{md,mdx}'], + sidebarPath: require.resolve('./sidebars-release-notes.js'), + async sidebarItemsGenerator({defaultSidebarItemsGenerator, docs, ...args}) { + const items = await defaultSidebarItemsGenerator({docs, ...args}); + + // Build map of doc id -> year from frontmatter date + const docYearMap = {}; + for (const doc of docs) { + const date = doc.frontMatter && doc.frontMatter.date; + if (date) { + const year = new Date(date).getFullYear(); + docYearMap[doc.id] = year; + } + } + + function parseVersion(str) { + const match = (str || '').match(/v?(\d+)\.(\d+)\.(\d+)/); + if (!match) return [0, 0, 0]; + return [parseInt(match[1]), parseInt(match[2]), parseInt(match[3])]; + } + function compareVersionsDesc(a, b) { + const [aMaj, aMin, aPatch] = parseVersion(a.label || a.id || ''); + const [bMaj, bMin, bPatch] = parseVersion(b.label || b.id || ''); + if (bMaj !== aMaj) return bMaj - aMaj; + if (bMin !== aMin) return bMin - aMin; + return bPatch - aPatch; + } + + // Flatten and transform doc items (filter index, shorten labels) + function flattenDocs(list) { + const result = []; + for (const item of list) { + if (item.type === 'doc' && item.id === 'index') continue; + if (item.type === 'doc') { + const label = item.id.replace(/\/index$/, ''); + result.push({...item, label}); + } else if (item.type === 'category') { + if (item.link && item.link.type === 'doc' && item.link.id !== 'index') { + const id = item.link.id; + const label = id.replace(/\/index$/, ''); + result.push({type: 'doc', id, label}); + } else { + result.push(...flattenDocs(item.items)); + } + } + } + return result; + } + + const docItems = flattenDocs(items); + + // Group by year + const byYear = {}; + for (const item of docItems) { + const year = docYearMap[item.id] || 'Other'; + if (!byYear[year]) byYear[year] = []; + byYear[year].push(item); + } + + // Sort each year's items by version descending + for (const year of Object.keys(byYear)) { + byYear[year].sort(compareVersionsDesc); + } + + // Build categories sorted by year descending + const years = Object.keys(byYear).sort((a, b) => { + // Object.keys() returns strings; avoid numeric subtraction type errors. + const na = Number.parseInt(a, 10); + const nb = Number.parseInt(b, 10); + return nb - na; + }); + return years.map(year => ({ + type: 'category', + label: String(year), + collapsed: year !== String(years[0]), + items: byYear[year], + })); + }, }, ], [ @@ -130,6 +200,20 @@ const config = { }; }, }), + // Ensure gtag exists before the GA script loads. + () => ({ + name: 'gtag-shim', + injectHtmlTags() { + return { + headTags: [ + { + tagName: 'script', + innerHTML: `window.dataLayer=window.dataLayer||[];function gtag(){dataLayer.push(arguments);}if(!window.gtag){window.gtag=gtag;}`, + }, + ], + }; + }, + }), ], presets: [ @@ -137,10 +221,13 @@ const config = { 'classic', /** @type {import('@docusaurus/preset-classic').Options} */ ({ - gtag: { - trackingID: 'G-K7K215ZVNC', - anonymizeIP: true, - }, + gtag: + process.env.NODE_ENV === 'production' + ? { + trackingID: 'G-K7K215ZVNC', + anonymizeIP: true, + } + : undefined, docs: { sidebarPath: require.resolve('./sidebars.js'), }, @@ -181,34 +268,39 @@ const config = { label: 'Docs', }, { + type: 'docSidebar', + sidebarId: 'learnSidebar', + position: 'left', + label: 'Learn', + }, + { + type: 'docSidebar', sidebarId: 'integrationsSidebar', position: 'left', label: 'Integrations', - to: "docs/integrations" }, { - sidebarId: 'tutorialSidebar', position: 'left', label: 'Enterprise', to: "docs/enterprise" }, - { to: '/release_notes', label: 'Release Notes', position: 'left' }, { to: '/blog', label: 'Blog', position: 'left' }, - { - href: 'https://models.litellm.ai/', - label: '💸 LLM Model Cost Map', - position: 'right', - }, { href: 'https://github.com/BerriAI/litellm', - label: 'GitHub', position: 'right', + className: 'header-github-link', + 'aria-label': 'GitHub repository', }, { href: 'https://www.litellm.ai/support', - label: 'Slack/Discord', position: 'right', - } + className: 'header-discord-link', + 'aria-label': 'Discord / Slack community', + }, + { + type: 'search', + position: 'right', + }, ], }, footer: { diff --git a/docs/my-website/img/ci_cd_architecture.png b/docs/my-website/img/ci_cd_architecture.png new file mode 100644 index 00000000000..111567c11b0 Binary files /dev/null and b/docs/my-website/img/ci_cd_architecture.png differ diff --git a/docs/my-website/img/hero.png b/docs/my-website/img/hero.png new file mode 100644 index 00000000000..9f77a28d718 Binary files /dev/null and b/docs/my-website/img/hero.png differ diff --git a/docs/my-website/img/isolated_ci_cd_environments.png b/docs/my-website/img/isolated_ci_cd_environments.png new file mode 100644 index 00000000000..347523f0fab Binary files /dev/null and b/docs/my-website/img/isolated_ci_cd_environments.png differ diff --git a/docs/my-website/img/mcp_zero_trust_gateway.png b/docs/my-website/img/mcp_zero_trust_gateway.png new file mode 100644 index 00000000000..3955cef0553 Binary files /dev/null and b/docs/my-website/img/mcp_zero_trust_gateway.png differ diff --git a/docs/my-website/img/security_update_march_2026/proxy_version.png b/docs/my-website/img/security_update_march_2026/proxy_version.png new file mode 100644 index 00000000000..c5d03d6a636 Binary files /dev/null and b/docs/my-website/img/security_update_march_2026/proxy_version.png differ diff --git a/docs/my-website/img/shared_ci_cd_environment.png b/docs/my-website/img/shared_ci_cd_environment.png new file mode 100644 index 00000000000..e54e11faa85 Binary files /dev/null and b/docs/my-website/img/shared_ci_cd_environment.png differ diff --git a/docs/my-website/package.json b/docs/my-website/package.json index 20462de2dd7..4b41a7ef46a 100644 --- a/docs/my-website/package.json +++ b/docs/my-website/package.json @@ -90,7 +90,7 @@ "tar-fs": ">=2.1.4", "webpack-dev-middleware": ">=5.3.4", "braces": ">=3.0.3", - "axios": ">=0.30.2", + "axios": "1.13.6", "webpack": ">=5.94.0", "serve-static": ">=1.16.0", "path-to-regexp": ">=0.1.12", diff --git a/docs/my-website/release_notes/index.md b/docs/my-website/release_notes/index.md new file mode 100644 index 00000000000..e2b7edf3222 --- /dev/null +++ b/docs/my-website/release_notes/index.md @@ -0,0 +1,52 @@ +--- +title: Release Notes +sidebar_label: Overview +slug: / +--- + +# Release Notes + +LiteLLM ships new releases regularly with new provider support, performance improvements, and enterprise features. Use the sidebar to browse all releases. + +## Latest Release + +### [v1.82.3 — Nebius AI, gpt-5.4, Gemini 3.x, FLUX Kontext, and 116 New Models](/release_notes/v1.82.3/v1-82-3) + +_March 16, 2026_ + +116 new models including Nebius AI, gpt-5.4, Gemini 3.x, and FLUX Kontext. + +--- + +## Recent Releases + +| Version | Date | Highlights | +| ----------------------------------- | ------------ | ---------------------------------------------------------- | +| [v1.82.0](/release_notes/v1.82.0/v1-82-0) | Feb 28, 2026 | Realtime Guardrails, Projects Management, and 10+ Performance Optimizations | +| [v1.81.14](/release_notes/v1.81.14/v1-81-14) | Feb 21, 2026 | New Gateway Level Guardrails & Compliance Playground | +| [v1.81.12](/release_notes/v1.81.12/v1-81-12) | Feb 14, 2026 | Guardrail Policy Templates & Action Builder | +| [v1.81.9](/release_notes/v1.81.9/v1-81-9) | Feb 7, 2026 | Control which MCP Servers are exposed on the Internet | +| [v1.81.6](/release_notes/v1.81.6/v1-81-6) | Jan 31, 2026 | Logs v2 with Tool Call Tracing | +| [v1.81.3](/release_notes/v1.81.3-stable/v1-81-3) | Jan 26, 2026 | Performance — 25% CPU Usage Reduction | +| [v1.81.0](/release_notes/v1.81.0/v1-81-0) | Jan 18, 2026 | Claude Code — Web Search Across All Providers | +| [v1.80.15](/release_notes/v1.80.15/v1-80-15) | Jan 10, 2026 | Manus API Support | +| [v1.80.8](/release_notes/v1.80.8-stable/v1-80-8) | Dec 6, 2025 | Introducing A2A Agent Gateway | +| [v1.80.5](/release_notes/v1.80.5-stable/v1-80-5) | Nov 22, 2025 | Gemini 3.0 Support | +| [v1.80.0](/release_notes/v1.80.0-stable/v1-80-0) | Nov 15, 2025 | Introducing Agent Hub: Register, Publish, and Share Agents | +| [v1.79.3](/release_notes/v1.79.3-stable/v1-79-3) | Nov 8, 2025 | Built-in Guardrails on AI Gateway | +| [v1.79.0](/release_notes/v1.79.0-stable/v1-79-0) | Oct 26, 2025 | Search APIs | +| [v1.78.5](/release_notes/v1.78.5-stable/v1-78-5) | Oct 18, 2025 | Native OCR Support | +| [v1.78.0](/release_notes/v1.78.0-stable/v1-78-0) | Oct 11, 2025 | MCP Gateway: Control Tool Access by Team, Key | +| [v1.77.7](/release_notes/v1.77.7-stable/v1-77-7) | Oct 4, 2025 | 2.9x Lower Median Latency | +| [v1.77.5](/release_notes/v1.77.5-stable/v1-77-5) | Sep 29, 2025 | MCP OAuth 2.0 Support | +| [v1.77.3](/release_notes/v1.77.3-stable/v1-77-3) | Sep 21, 2025 | Priority Based Rate Limiting | + +--- + +## Stay Updated + +- **GitHub**: Watch the [BerriAI/litellm](https://github.com/BerriAI/litellm) repository for release notifications +- **Discord**: Join our [community](https://discord.com/invite/wuPM9dRgDw) for announcements +- **Twitter**: Follow [@LiteLLM](https://twitter.com/LiteLLM) + +Use the sidebar to browse the full release history. diff --git a/docs/my-website/release_notes/v1.63.14/index.md b/docs/my-website/release_notes/v1.63.14/index.md index 1ac713fc2d5..d34b2c7b335 100644 --- a/docs/my-website/release_notes/v1.63.14/index.md +++ b/docs/my-website/release_notes/v1.63.14/index.md @@ -62,10 +62,10 @@ Here's a Demo Instance to test changes: - Infer aws region from bedrock application profile id - (`arn:aws:bedrock:us-east-1:...`) - Ollama - support calling via `/v1/completions` [Get Started](../../docs/providers/ollama#using-ollama-fim-on-v1completions) - Bedrock - support `us.deepseek.r1-v1:0` model name [Docs](../../docs/providers/bedrock#supported-aws-bedrock-models) -- OpenRouter - `OPENROUTER_API_BASE` env var support [Docs](../../docs/providers/openrouter.md) +- OpenRouter - `OPENROUTER_API_BASE` env var support [Docs](../../docs/providers/openrouter) - Azure - add audio model parameter support - [Docs](../../docs/providers/azure#azure-audio-model) - OpenAI - PDF File support [Docs](../../docs/completion/document_understanding#openai-file-message-type) -- OpenAI - o1-pro Responses API streaming support [Docs](../../docs/response_api.md#streaming) +- OpenAI - o1-pro Responses API streaming support [Docs](../../docs/response_api#streaming) - [BETA] MCP - Use MCP Tools with LiteLLM SDK [Docs](../../docs/mcp) 2. **Bug Fixes** @@ -102,7 +102,7 @@ Here's a Demo Instance to test changes: - fix logging to just log the LLM I/O [PR](https://github.com/BerriAI/litellm/pull/9353) - Dynamic API Key/Space param support [Get Started](../../docs/observability/arize_integration#pass-arize-spacekey-per-request) - StandardLoggingPayload - Log litellm_model_name in payload. Allows knowing what the model sent to API provider was [Get Started](../../docs/proxy/logging_spec#standardlogginghiddenparams) -- Prompt Management - Allow building custom prompt management integration [Get Started](../../docs/proxy/custom_prompt_management.md) +- Prompt Management - Allow building custom prompt management integration [Get Started](../../docs/proxy/custom_prompt_management) ## Performance / Reliability improvements @@ -128,4 +128,4 @@ Here's a Demo Instance to test changes: ## Complete Git Diff -[Here's the complete git diff](https://github.com/BerriAI/litellm/compare/v1.63.11-stable...v1.63.14.rc) \ No newline at end of file +[Here's the complete git diff](https://github.com/BerriAI/litellm/compare/v1.63.11-stable...v1.63.14.rc) diff --git a/docs/my-website/release_notes/v1.63.2-stable/index.md b/docs/my-website/release_notes/v1.63.2-stable/index.md index a248aa94342..18233f25c21 100644 --- a/docs/my-website/release_notes/v1.63.2-stable/index.md +++ b/docs/my-website/release_notes/v1.63.2-stable/index.md @@ -64,15 +64,15 @@ Here's a Demo Instance to test changes: 9. Bedrock - handle thinking blocks in assistant message. [Get Started](https://docs.litellm.ai/docs/providers/bedrock#usage---thinking--reasoning-content) 10. Anthropic - Return `signature` on streaming. [Get Started](https://docs.litellm.ai/docs/providers/bedrock#usage---thinking--reasoning-content) - Note: We've also migrated from `signature_delta` to `signature`. [Read more](https://docs.litellm.ai/release_notes/v1.63.0) -11. Support format param for specifying image type. [Get Started](../../docs/completion/vision.md#explicitly-specify-image-type) -12. Anthropic - `/v1/messages` endpoint - `thinking` param support. [Get Started](../../docs/anthropic_unified.md) +11. Support format param for specifying image type. [Get Started](../../docs/completion/vision#explicitly-specify-image-type) +12. Anthropic - `/v1/messages` endpoint - `thinking` param support. [Get Started](../../docs/anthropic_unified) - Note: this refactors the [BETA] unified `/v1/messages` endpoint, to just work for the Anthropic API. 13. Vertex AI - handle $id in response schema when calling vertex ai. [Get Started](https://docs.litellm.ai/docs/providers/vertex#json-schema) ## Spend Tracking Improvements 1. Batches API - Fix cost calculation to run on retrieve_batch. [Get Started](https://docs.litellm.ai/docs/batches) -2. Batches API - Log batch models in spend logs / standard logging payload. [Get Started](../../docs/proxy/logging_spec.md#standardlogginghiddenparams) +2. Batches API - Log batch models in spend logs / standard logging payload. [Get Started](../../docs/proxy/logging_spec#standardlogginghiddenparams) ## Management Endpoints / UI @@ -109,4 +109,4 @@ Here's a Demo Instance to test changes: ## Complete Git Diff -[Here's the complete git diff](https://github.com/BerriAI/litellm/compare/v1.61.20-stable...v1.63.2-stable) \ No newline at end of file +[Here's the complete git diff](https://github.com/BerriAI/litellm/compare/v1.61.20-stable...v1.63.2-stable) diff --git a/docs/my-website/release_notes/v1.80.15/index.md b/docs/my-website/release_notes/v1.80.15/index.md index 4037a0d9b5d..15b49822965 100644 --- a/docs/my-website/release_notes/v1.80.15/index.md +++ b/docs/my-website/release_notes/v1.80.15/index.md @@ -53,7 +53,7 @@ pip install litellm==1.80.15 - **MCP Global Mode** - [Configure MCP servers globally with visibility controls](../../docs/mcp) - **Interactions API Bridge** - [Use all LiteLLM providers with the Interactions API](../../docs/interactions) - **RAG Query Endpoint** - [New RAG Search/Query endpoint for retrieval-augmented generation](../../docs/search/index) -- **UI Usage - Endpoint Activity** - [Users can now see Endpoint Activity Metrics in the UI](../../docs/proxy/endpoint_activity.md) +- **UI Usage - Endpoint Activity** - [Users can now see Endpoint Activity Metrics in the UI](../../docs/proxy/endpoint_activity) - **50% Overhead Reduction** - LiteLLM now sends 2.5× more requests to LLM providers @@ -640,4 +640,3 @@ Users can now see Endpoint Activity Metrics in the UI. **[View complete changelog on GitHub](https://github.com/BerriAI/litellm/compare/v1.80.11.rc.1...v1.80.15-stable.1)** - diff --git a/docs/my-website/release_notes/v1.81.0/index.md b/docs/my-website/release_notes/v1.81.0/index.md index e61d7d2d593..5953c572a7a 100644 --- a/docs/my-website/release_notes/v1.81.0/index.md +++ b/docs/my-website/release_notes/v1.81.0/index.md @@ -48,7 +48,7 @@ pip install litellm==1.81.0 - **Claude Code** - Support for using web search across Bedrock, Vertex AI, and all LiteLLM providers - **Major Change** - [50MB limit on image URL downloads](#major-change---chatcompletions-image-url-download-size-limit) to improve reliability - **Performance** - [25% CPU Usage Reduction](#performance---25-cpu-usage-reduction) by removing premature model.dump() calls from the hot path -- **Deleted Keys Audit Table on UI** - [View deleted keys and teams for audit purposes](../../docs/proxy/deleted_keys_teams.md) with spend and budget information at the time of deletion +- **Deleted Keys Audit Table on UI** - [View deleted keys and teams for audit purposes](../../docs/proxy/deleted_keys_teams) with spend and budget information at the time of deletion --- @@ -166,7 +166,7 @@ LiteLLM now reduces CPU usage by removing premature `model.dump()` calls from th -LiteLLM now provides a comprehensive audit table for deleted API keys and teams directly in the UI. This feature allows you to easily track the spend of deleted keys, view their associated team information, and maintain accurate financial records for auditing and compliance purposes. The table displays key details including key aliases, team associations, and spend information captured at the time of deletion. For more information on how to use this feature, see the [Deleted Keys & Teams documentation](../../docs/proxy/deleted_keys_teams.md). +LiteLLM now provides a comprehensive audit table for deleted API keys and teams directly in the UI. This feature allows you to easily track the spend of deleted keys, view their associated team information, and maintain accurate financial records for auditing and compliance purposes. The table displays key details including key aliases, team associations, and spend information captured at the time of deletion. For more information on how to use this feature, see the [Deleted Keys & Teams documentation](../../docs/proxy/deleted_keys_teams). --- diff --git a/docs/my-website/release_notes/v1.81.12.md b/docs/my-website/release_notes/v1.81.12/index.md similarity index 99% rename from docs/my-website/release_notes/v1.81.12.md rename to docs/my-website/release_notes/v1.81.12/index.md index 0b7c1e146ab..1bbea5b82bc 100644 --- a/docs/my-website/release_notes/v1.81.12.md +++ b/docs/my-website/release_notes/v1.81.12/index.md @@ -62,13 +62,13 @@ This release fixes out-of-memory (OOM) risks from unbounded `asyncio.Queue()` us This release adds a visual action builder for guardrail policies with conditional execution support. You can now chain guardrails into multi-step pipelines — if a simple guardrail fails, route to an advanced one instead of immediately blocking. Each step has configurable ON PASS and ON FAIL actions (Next Step, Block, or Allow), and you can test the full pipeline with a sample message before saving. -![Guardrail Action Builder](../img/release_notes/guard_actions.png) +![Guardrail Action Builder](../../img/release_notes/guard_actions.png) ### Access Groups Access Groups simplify defining resource access across your organization. One group can grant access to models, MCP servers, and agents—simply attach it to a key or team. Create groups in the Admin UI, define which resources each group includes, then assign the group when creating keys or teams. Updates to a group apply automatically to all attached keys and teams. - + ## New Providers and Endpoints diff --git a/docs/my-website/release_notes/v1.81.14.md b/docs/my-website/release_notes/v1.81.14/index.md similarity index 99% rename from docs/my-website/release_notes/v1.81.14.md rename to docs/my-website/release_notes/v1.81.14/index.md index c342bc47ee9..92c22bc0ea3 100644 --- a/docs/my-website/release_notes/v1.81.14.md +++ b/docs/my-website/release_notes/v1.81.14/index.md @@ -56,7 +56,7 @@ pip install litellm==1.81.14 AI Platform Admins can now browse built-in and partner guardrails from the Guardrail Garden. Guardrails are organized by use case — blocking financial advice, filtering insults, detecting competitor mentions, and more — so you can find the right one and deploy it in a few clicks. -![Guardrail Garden](../img/release_notes/guardrail_garden.png) +![Guardrail Garden](../../img/release_notes/guardrail_garden.png) ### 3 New Built-in Guardrails @@ -72,7 +72,7 @@ These guardrails are built for production and on our benchmarks had a 100% Recal Previously, the `store_model_in_db` setting could only be configured in `proxy_config.yaml` under `general_settings`, requiring a proxy restart to take effect. Now you can enable or disable this setting directly from the Admin UI without any restarts. This is especially useful for cloud deployments where you don't have direct access to config files or want to avoid downtime. Enable `store_model_in_db` to move model definitions from your YAML into the database—reducing config complexity, improving scalability, and enabling dynamic model management across multiple proxy instances. -![Store model in DB Setting](../img/ui_store_model_in_db.png) +![Store model in DB Setting](../../img/ui_store_model_in_db.png) #### Eval results @@ -91,14 +91,14 @@ We benchmarked our new built-in guardrails against labeled datasets before shipp The Compliance Playground lets you test any guardrail against our pre-built eval datasets or your own custom datasets, so you can see precision, recall, and false positive rate before rolling it out to production. -![Compliance Playground](../img/release_notes/compliance_playground.png) +![Compliance Playground](../../img/release_notes/compliance_playground.png) --- ## Performance & Reliability — Up to 13% Lower Latency - + This release cuts latency across all percentiles through 20+ micro-optimizations across logging, cost calculation, routing, and connection management. See [benchmarking](../../docs/benchmarks) for more info about how to benchmark yourself. diff --git a/docs/my-website/release_notes/v1.81.6.md b/docs/my-website/release_notes/v1.81.6/index.md similarity index 100% rename from docs/my-website/release_notes/v1.81.6.md rename to docs/my-website/release_notes/v1.81.6/index.md diff --git a/docs/my-website/release_notes/v1.81.9.md b/docs/my-website/release_notes/v1.81.9/index.md similarity index 99% rename from docs/my-website/release_notes/v1.81.9.md rename to docs/my-website/release_notes/v1.81.9/index.md index 80be4179b46..d11b52e892c 100644 --- a/docs/my-website/release_notes/v1.81.9.md +++ b/docs/my-website/release_notes/v1.81.9/index.md @@ -81,7 +81,7 @@ This release makes it safe to expose MCP servers on the public internet by addin [Get started](../../docs/mcp_public_internet) @@ -92,7 +92,7 @@ Set a soft budget on any team to receive email alerts when spending crosses the [Get started](../../docs/proxy/ui_team_soft_budget_alerts) diff --git a/docs/my-website/release_notes/v1.82.0.md b/docs/my-website/release_notes/v1.82.0/index.md similarity index 100% rename from docs/my-website/release_notes/v1.82.0.md rename to docs/my-website/release_notes/v1.82.0/index.md diff --git a/docs/my-website/release_notes/v1.82.3.md b/docs/my-website/release_notes/v1.82.3/index.md similarity index 58% rename from docs/my-website/release_notes/v1.82.3.md rename to docs/my-website/release_notes/v1.82.3/index.md index 15df33fdb80..20be8826718 100644 --- a/docs/my-website/release_notes/v1.82.3.md +++ b/docs/my-website/release_notes/v1.82.3/index.md @@ -47,6 +47,10 @@ pip install litellm==1.82.3 - **FLUX Kontext image editing** — `flux-kontext-pro` and `flux-kontext-max` added to Black Forest Labs, alongside `flux-pro-1.0-fill` and `flux-pro-1.0-expand` for inpainting and outpainting - **116 new models, 132 deprecated models cleaned up** — Major model map refresh including Mistral Magistral, Dashscope Qwen3 VL, xAI Grok via Azure AI, ZAI GLM-5, Serper Search; removal of OpenAI GPT-3.5/GPT-4 legacy variants, Gemini 1.5, and Vertex AI PaLM2 - **SageMaker Nova provider** — [New `sagemaker_nova` provider for Amazon Nova models on SageMaker](../../docs/providers/aws_sagemaker) - [PR #21542](https://github.com/BerriAI/litellm/pull/21542) +- **Hashicorp Vault secret manager** — Config override backend powered by Hashicorp Vault, with full UI for managing vault-sourced credentials - [PR #22939](https://github.com/BerriAI/litellm/pull/22939), [PR #23036](https://github.com/BerriAI/litellm/pull/23036) +- **Responses API WebSocket streaming** — Real-time WebSocket streaming for the Responses API, including support across all providers - [PR #22559](https://github.com/BerriAI/litellm/pull/22559), [PR #22771](https://github.com/BerriAI/litellm/pull/22771) +- **Org Admin RBAC expansion** — Org Admins can now access team management endpoints, view and invite internal users, and manage team membership without requiring a global admin role - [PR #23085](https://github.com/BerriAI/litellm/pull/23085), [PR #23080](https://github.com/BerriAI/litellm/pull/23080) +- **Guardrail mode defaults and tag-based modes** — Set a default guardrail mode list globally, and specify a list of modes in tag-based guardrail configs - [PR #22676](https://github.com/BerriAI/litellm/pull/22676), [PR #23020](https://github.com/BerriAI/litellm/pull/23020) - **Secret redaction in logs** — API keys, tokens, and credentials automatically scrubbed from all proxy log output. Enabled by default; opt out with `LITELLM_DISABLE_REDACT_SECRETS=true` - [PR #23668](https://github.com/BerriAI/litellm/pull/23668) - **Streaming stability fix** — Critical fix for `RuntimeError: Cannot send a request, as the client has been closed.` crashes after ~1 hour in production - [PR #22926](https://github.com/BerriAI/litellm/pull/22926) @@ -54,7 +58,7 @@ pip install litellm==1.82.3 ## New Providers and Endpoints -### New Providers (5 new providers) +### New Providers (7 new providers) | Provider | Supported LiteLLM Endpoints | Description | | -------- | --------------------------- | ----------- | @@ -63,6 +67,8 @@ pip install litellm==1.82.3 | [Black Forest Labs](../../docs/providers/black_forest_labs) (`black_forest_labs/`) | `/images/generations`, `/images/edits` | FLUX image generation and editing — Kontext Pro/Max, Pro 1.0 Fill/Expand | | [Serper](../../docs/providers/serper) (`serper/`) | `/search` | Web search via Serper API | | [SageMaker Nova](../../docs/providers/aws_sagemaker) (`sagemaker_nova/`) | `/chat/completions` | Amazon Nova models via SageMaker endpoint | +| [Google Search API](../../docs/providers/google_search) (`google_search/`) | `/search` | Google Search API integration - [PR #22752](https://github.com/BerriAI/litellm/pull/22752) | +| [Bedrock Mantle](../../docs/providers/bedrock) (`bedrock_mantle/`) | `/chat/completions` | Amazon Bedrock via Mantle — alternative auth and routing path for Bedrock models - [PR #22866](https://github.com/BerriAI/litellm/pull/22866) | --- @@ -238,22 +244,92 @@ pip install litellm==1.82.3 - **[Responses API](../../docs/response_api)** - Handle `response.failed`, `response.incomplete`, and `response.cancelled` terminal event types in background streaming — previously only `response.completed` was handled - [PR #23492](https://github.com/BerriAI/litellm/pull/23492) + - WebSocket streaming support for Responses API — real-time streaming via WebSocket for all providers - [PR #22559](https://github.com/BerriAI/litellm/pull/22559), [PR #22771](https://github.com/BerriAI/litellm/pull/22771) + - WebRTC support for real-time audio/video communication - [PR #23446](https://github.com/BerriAI/litellm/pull/23446) + - Responses API support for OpenAI-compatible JSON providers (`openai_like`) - [PR #21398](https://github.com/BerriAI/litellm/pull/21398) + - Route `gpt-5.4+` calls using both tools and reasoning to the Responses API automatically - [PR #23577](https://github.com/BerriAI/litellm/pull/23577) -#### Bug Fixes +- **[Anthropic Files API](../../docs/providers/anthropic)** + - Full Anthropic Files API support — upload, retrieve, list, and delete files; use file references in messages - [PR #16594](https://github.com/BerriAI/litellm/pull/16594) + +- **[Mistral](../../docs/providers/mistral)** + - Voxtral audio transcription support — `mistral/voxtral-mini-*` and `mistral/voxtral-*` for audio transcription via Mistral - [PR #22801](https://github.com/BerriAI/litellm/pull/22801) + +- **[OpenAI](../../docs/providers/openai)** + - `litellm.acount_tokens()` public API — async token counting with full OpenAI provider support - [PR #22809](https://github.com/BerriAI/litellm/pull/22809) + - Normalize `reasoning_effort` dict to string for chat completion API - [PR #22981](https://github.com/BerriAI/litellm/pull/22981) + +- **[OpenRouter](../../docs/providers/openrouter)** + - Image edit support for OpenRouter models - [PR #22403](https://github.com/BerriAI/litellm/pull/22403) + +- **[Google Vertex AI](../../docs/providers/vertex)** + - VIDEO modality token usage tracking in `completion_tokens_details` - [PR #22550](https://github.com/BerriAI/litellm/pull/22550) + +- **Images API** + - `input_fidelity` parameter for image edit API - [PR #23201](https://github.com/BerriAI/litellm/pull/23201) + +- **General** + - Per-request `enable_json_schema_validation` flag for thread-safe JSON schema validation - [PR #21233](https://github.com/BerriAI/litellm/pull/21233) + - Model cost aliases expansion — define aliases in the cost map that inherit pricing from a parent model - [PR #23314](https://github.com/BerriAI/litellm/pull/23314), [PR #23457](https://github.com/BerriAI/litellm/pull/23457) + - Wildcards model support for the Files API - [PR #22740](https://github.com/BerriAI/litellm/pull/22740) + +#### Bugs - **[Anthropic](../../docs/providers/anthropic)** - Preserve native tool format (web_search, bash, tool_search, etc.) when guardrails convert tools for the Anthropic Messages API - [PR #23526](https://github.com/BerriAI/litellm/pull/23526) + - Enforce `type: "object"` on tool input schemas in `_map_tool_helper` — fixes tool call failures for strict-schema providers - [PR #23103](https://github.com/BerriAI/litellm/pull/23103) + - Deduplicate `tool_result` messages by `tool_call_id` — prevents duplicate tool result errors in multi-turn conversations - [PR #23104](https://github.com/BerriAI/litellm/pull/23104) + - Map `reasoning_effort` to `output_config` for Claude 4.6 models - [PR #22220](https://github.com/BerriAI/litellm/pull/22220) + +- **[Google Gemini](../../docs/providers/gemini)** + - Correct streaming `finish_reason` for tool calls — was incorrectly returning `null` instead of `tool_calls` - [PR #21577](https://github.com/BerriAI/litellm/pull/21577) + - Preserve `$ref` in JSON Schema for Gemini 2.0+ — schema references were being stripped, breaking structured output - [PR #21597](https://github.com/BerriAI/litellm/pull/21597) + - Handle `minimal` `reasoning_effort` param for Gemini 3.1 models - [PR #22920](https://github.com/BerriAI/litellm/pull/22920) + +- **[Google Vertex AI](../../docs/providers/vertex)** + - Pass through native Gemini `imageConfig` params for image generation - [PR #21585](https://github.com/BerriAI/litellm/pull/21585) + - Prevent content truncation when `finish_reason` races ahead of content chunks in streaming - [PR #22692](https://github.com/BerriAI/litellm/pull/22692) + - Strip LiteLLM-internal keys from `extra_body` before merging to Gemini request body - [PR #23131](https://github.com/BerriAI/litellm/pull/23131) + - Drop unsupported `output_config` parameter from all Vertex AI requests - [PR #22884](https://github.com/BerriAI/litellm/pull/22884) + - Skip schema transforms for Gemini 2.0+ tool parameters — avoids breaking native Gemini schema handling - [PR #23265](https://github.com/BerriAI/litellm/pull/23265) + +- **[OpenRouter](../../docs/providers/openrouter)** + - Pattern-based fix for native model double-stripping when provider prefix matches model name - [PR #22320](https://github.com/BerriAI/litellm/pull/22320) + - Use provider-reported usage in streaming responses when `stream_options` is not set - [PR #21592](https://github.com/BerriAI/litellm/pull/21592) + +- **[AWS Bedrock](../../docs/providers/bedrock)** + - Extract region and model ID from `bedrock/{region}/{model}` path format - [PR #22546](https://github.com/BerriAI/litellm/pull/22546) + - Strip `scope` from `cache_control` for Anthropic messages on Bedrock and Azure AI - [PR #22867](https://github.com/BerriAI/litellm/pull/22867) + - Populate `completion_tokens_details` in Responses API responses - [PR #23243](https://github.com/BerriAI/litellm/pull/23243) + +- **[Azure AI](../../docs/providers/azure_ai)** + - Resolve `api_base` from environment variable in Document Intelligence OCR - [PR #21581](https://github.com/BerriAI/litellm/pull/21581) - **[Moonshot / Kimi](../../docs/providers/openai_compatible)** - Auto-fill `reasoning_content` for Moonshot Kimi reasoning models - [PR #23580](https://github.com/BerriAI/litellm/pull/23580) + - Preserve `image_url` blocks in multimodal messages for Moonshot - [PR #21595](https://github.com/BerriAI/litellm/pull/21595) - **[HuggingFace](../../docs/providers/huggingface)** - Forward `extra_headers` to HuggingFace embedding API - [PR #23525](https://github.com/BerriAI/litellm/pull/23525) +- **Token Counting / Cost** + - Fix `count_tokens` to include system prompts and tools in token counting API requests - [PR #22301](https://github.com/BerriAI/litellm/pull/22301) + - Pass all custom pricing fields to `register_model` in `completion()` and `embedding()` - [PR #22552](https://github.com/BerriAI/litellm/pull/22552) + +- **Tools / Function Calling** + - Gracefully repair truncated JSON in tool call arguments — prevents crashes on malformed tool responses - [PR #22503](https://github.com/BerriAI/litellm/pull/22503) + - Fix `output_item.done` for function calls not emitting `finish_reason` in streaming - [PR #22553](https://github.com/BerriAI/litellm/pull/22553) + - Preserve thinking block order with multiple web searches - [PR #23093](https://github.com/BerriAI/litellm/pull/23093) + - **General** - Normalize `content_filtered` finish reason across providers - [PR #23564](https://github.com/BerriAI/litellm/pull/23564) + - Unify `finish_reason` mapping to OpenAI-compatible values across all providers - [PR #22138](https://github.com/BerriAI/litellm/pull/22138) - Fix custom cost tracking on deployments for `/v1/messages` and `/v1/responses` - [PR #23647](https://github.com/BerriAI/litellm/pull/23647) - Fix per-request custom pricing when `router_model_id` has no pricing data — now falls back to model name + - Fix batch list showing stale `validating` status after completion - [PR #22982](https://github.com/BerriAI/litellm/pull/22982) + - Fix batch retrieve returning raw `output_file_id` when `model_id` is missing - [PR #23194](https://github.com/BerriAI/litellm/pull/23194) + - Encode batch IDs when `x-litellm-model` header is used - [PR #22653](https://github.com/BerriAI/litellm/pull/22653) + - Map `reasoning` to `reasoning_content` in streaming Delta for gpt-oss providers - [PR #22803](https://github.com/BerriAI/litellm/pull/22803) --- @@ -264,10 +340,31 @@ pip install litellm==1.82.3 - **Virtual Keys** - Add Organization dropdown to Create/Edit Key form — `organization_id` is now a first-class field in Key Ownership - [PR #23595](https://github.com/BerriAI/litellm/pull/23595) - Allow setting `organization_id` on `/key/update` — keys can be assigned or moved to a different organization after creation - [PR #23557](https://github.com/BerriAI/litellm/pull/23557) + - Manual Spend Reset for virtual keys from the UI — admins can reset key spend to zero on demand - [PR #22715](https://github.com/BerriAI/litellm/pull/22715) + - BYOK (Bring Your Own Key) — client-side provider API key takes precedence over proxy key for Anthropic `/v1/messages` - [PR #22964](https://github.com/BerriAI/litellm/pull/22964) + - UI login session duration configurable via `LITELLM_UI_SESSION_DURATION` environment variable - [PR #22182](https://github.com/BerriAI/litellm/pull/22182) + - Auto-redirect UI login to SSO via `auto_redirect_ui_login_to_sso: true` in config.yaml - [PR #23367](https://github.com/BerriAI/litellm/pull/23367) + +- **Access Control (RBAC)** + - Org Admins can now access team management endpoints — `/team/new`, `/team/update`, `/team/delete`, `/team/member_add`, `/team/member_delete` - [PR #23085](https://github.com/BerriAI/litellm/pull/23085), [PR #23095](https://github.com/BerriAI/litellm/pull/23095) + - Org Admins can view and invite internal users — full user management without requiring global admin role - [PR #23080](https://github.com/BerriAI/litellm/pull/23080) + - Allow Admin Viewers to access Audit Logs — view-only admin role now includes audit log access - [PR #23419](https://github.com/BerriAI/litellm/pull/23419) + - RBAC for Vector Stores and Agents — key/team-level access control for vector store and agent resources - [PR #22858](https://github.com/BerriAI/litellm/pull/22858) + - User filter scope (`scope_user_search_to_org`) is now opt-in — previously default-on, causing unintended restriction - [PR #23057](https://github.com/BerriAI/litellm/pull/23057) + +- **Vector Stores** + - Vector Store management endpoints — retrieve, list, update, and delete vector stores via `/v1/vector_stores/*` - [PR #23435](https://github.com/BerriAI/litellm/pull/23435) + +- **Teams** + - Batch expiry setting for teams — configure a default expiry duration for all team keys - [PR #22705](https://github.com/BerriAI/litellm/pull/22705) + - Team Admin can reset key spend - [PR #22725](https://github.com/BerriAI/litellm/pull/22725) - **Internal Users** - Add/Remove Team Membership directly from the Internal Users info page — includes searchable dropdown and role selector; no longer requires navigating to each team - [PR #23638](https://github.com/BerriAI/litellm/pull/23638) +- **Models** + - Attach knowledge base to model via UI - [PR #22656](https://github.com/BerriAI/litellm/pull/22656) + - **Default Team Settings** - Modernize page to antd (consistent with rest of app) - [PR #23614](https://github.com/BerriAI/litellm/pull/23614) - Fix: default team params (budget, duration, tpm, rpm, permissions) now correctly applied on `/team/new` - [PR #23614](https://github.com/BerriAI/litellm/pull/23614) @@ -290,6 +387,13 @@ pip install litellm==1.82.3 - Fix Public Model Hub not showing config-defined models after save - [PR #23501](https://github.com/BerriAI/litellm/pull/23501) - Fix fallback popup model dropdown z-index issue - [PR #23516](https://github.com/BerriAI/litellm/pull/23516) - Fix double-counting bug in org/team key limit checks on `/key/update` +- Fix invite link allowing multiple password resets for the same link - [PR #22462](https://github.com/BerriAI/litellm/pull/22462) +- Fix key expiry default duration not being applied when `duration` is not set - [PR #22956](https://github.com/BerriAI/litellm/pull/22956) +- Fix all proxy models not including model access groups in key creation - [PR #23236](https://github.com/BerriAI/litellm/pull/23236) +- Fix admin viewers unable to see all organizations - [PR #22940](https://github.com/BerriAI/litellm/pull/22940) +- Fix Audit Logs UI: added server-side pagination, filtering, and drawer view - [PR #22476](https://github.com/BerriAI/litellm/pull/22476) +- Fix virtual keys in teams view not applying the team filter correctly - [PR #23065](https://github.com/BerriAI/litellm/pull/23065) +- Fix team expiry enforcement validation - [PR #22728](https://github.com/BerriAI/litellm/pull/22728) --- @@ -297,6 +401,13 @@ pip install litellm==1.82.3 ### Logging +- **[Helicone](../../docs/observability/helicone_integration)** + - Add Gemini and Vertex AI support to HeliconeLogger — routes Gemini and Vertex AI requests through the correct Helicone provider URL - [PR #19288](https://github.com/BerriAI/litellm/pull/19288) + - Fix correct provider URL for Vertex AI Gemini models - [PR #22603](https://github.com/BerriAI/litellm/pull/22603) + +- **[Langfuse](../../docs/proxy/logging#langfuse)** + - Fix failure path kwargs inconsistency causing dropped traces on failed requests - [PR #22390](https://github.com/BerriAI/litellm/pull/22390) + - **[Vantage](https://vantage.sh)** - Add Vantage integration for FOCUS 1.2 CSV export — export LiteLLM proxy spend data as FinOps Open Cost & Usage Specification reports, with time-windowed filenames to prevent overwrites - [PR #23333](https://github.com/BerriAI/litellm/pull/23333) @@ -305,7 +416,10 @@ pip install litellm==1.82.3 ### Guardrails -No major guardrail changes in this release. +- **Guardrail mode default list** — Configure a default list of guardrail modes applied globally when no per-request mode is specified - [PR #22676](https://github.com/BerriAI/litellm/pull/22676) +- **Tag-based guardrail mode lists** — Specify a list of modes in tag-based guardrail configs instead of a single mode - [PR #23020](https://github.com/BerriAI/litellm/pull/23020) +- **Fix presidio PII token leak** — Edge case where Anthropic handle in Presidio caused PII data exposure in token response - [PR #22627](https://github.com/BerriAI/litellm/pull/22627) +- **Fix OTEL orphaned guardrail traces** — Span redundancy and missing response IDs in OpenTelemetry guardrail traces - [PR #23001](https://github.com/BerriAI/litellm/pull/23001) ### Prompt Management @@ -313,7 +427,32 @@ No major prompt management changes in this release. ### Secret Managers -No major secret manager changes in this release. +- **[Hashicorp Vault](../../docs/secret_managers)** — Full Hashicorp Vault integration as a config override backend — secrets defined in Vault are fetched at startup and override `config.yaml` values. UI support for managing vault-sourced credentials included - [PR #22939](https://github.com/BerriAI/litellm/pull/22939), [PR #23036](https://github.com/BerriAI/litellm/pull/23036) + +--- + +## MCP Gateway + +#### Features + +- **Token authentication for MCP servers** — configure `auth_type: "bearer"` per MCP server to require token-based auth on tool calls - [PR #23260](https://github.com/BerriAI/litellm/pull/23260) +- **Team-scoped MCP server filtering** — keys created under a team only see MCP servers available to that team - [PR #23323](https://github.com/BerriAI/litellm/pull/23323) +- **Per-server health recheck in UI** — trigger a health check for individual MCP servers without reloading all servers - [PR #23328](https://github.com/BerriAI/litellm/pull/23328) + +#### Bugs + +- Fix MCP server URL and tools management issues causing tool discovery to fail - [PR #22751](https://github.com/BerriAI/litellm/pull/22751) +- Fix MCP server health checks triggering on server deletion - [PR #23063](https://github.com/BerriAI/litellm/pull/23063) + +--- + +## Spend Tracking, Budgets and Rate Limiting + +- **Fix budget-linked keys never having spend reset** — Keys linked to budget objects were not having their spend reset on the configured reset interval - [PR #20688](https://github.com/BerriAI/litellm/pull/20688) +- **Flex pricing support** — Add `flex_pricing` field to cost map for providers that offer dynamic pricing tiers - [PR #22992](https://github.com/BerriAI/litellm/pull/22992) +- **Fix spend log cleanup** — Resolved lock tracking, integer retention, and skip-log-level issues in spend log cleanup job - [PR #22687](https://github.com/BerriAI/litellm/pull/22687) +- **Fix WebSearch spend log deduplication** — WebSearch interception was failing with thinking enabled; fixed along with spend log dedup - [PR #22679](https://github.com/BerriAI/litellm/pull/22679) +- **Fix TypeError when request has no API key** — Spend tracking was throwing unhandled exception when API key was absent from request - [PR #23363](https://github.com/BerriAI/litellm/pull/23363) --- @@ -323,6 +462,10 @@ No major secret manager changes in this release. - **Fix OOM / Prisma connection loss** on large installs — unbounded managed-object poll was exhausting Prisma connections after ~60–70 minutes on instances with 336K+ queued response rows - [PR #23472](https://github.com/BerriAI/litellm/pull/23472) - **Centralize logging kwarg updates** — root cause fix migrating all logging updates to a single function, eliminating kwarg inconsistencies across logging paths - [PR #23659](https://github.com/BerriAI/litellm/pull/23659) - **Fix tiktoken cache for non-root offline containers** — tiktoken cache now works correctly in offline environments running as non-root users - [PR #23498](https://github.com/BerriAI/litellm/pull/23498) +- **Block proxy startup when Redis transaction buffer has no Redis** — prevents silent data loss when `use_redis_transaction_buffer: true` is set without a Redis connection - [PR #23019](https://github.com/BerriAI/litellm/pull/23019) +- **Fix `InFlightRequestsMiddleware` crash** — undefined kwargs in middleware were causing request failures - [PR #22523](https://github.com/BerriAI/litellm/pull/22523) +- **Fix `BaseModelResponseIterator` crash on non-string stream chunks** — streaming was crashing when providers returned non-string chunk data - [PR #23497](https://github.com/BerriAI/litellm/pull/23497) +- **Fix `SERVER_ROOT_PATH` prefix handling** — strip prefix before checking mapped pass-through routes to prevent double-prefix issues - [PR #23414](https://github.com/BerriAI/litellm/pull/23414) - **Add CodSpeed continuous performance benchmarks** — automated performance regression tracking on CI - [PR #23676](https://github.com/BerriAI/litellm/pull/23676) --- @@ -342,6 +485,16 @@ No major secret manager changes in this release. --- +## Documentation Updates + +- Add Anthropic `/v1/messages` → `/responses` parameter mapping reference - [PR #22893](https://github.com/BerriAI/litellm/pull/22893) +- Update Okta SSO docs and custom SSO handler example - [PR #22786](https://github.com/BerriAI/litellm/pull/22786) +- Add `LITELLM_MAX_BUDGET_PER_SESSION_TTL` to environment variables reference - [PR #23186](https://github.com/BerriAI/litellm/pull/23186) +- Add DB query performance guidelines to `CLAUDE.md` - [PR #23196](https://github.com/BerriAI/litellm/pull/23196) +- Add Gemini Vertex AI PayGo/priority cost tracking docs - [PR #22948](https://github.com/BerriAI/litellm/pull/22948) + +--- + ## New Contributors * @ryanh-ai made their first contribution in [PR #21542](https://github.com/BerriAI/litellm/pull/21542) @@ -359,14 +512,17 @@ No major secret manager changes in this release. ## Diff Summary ## 03/16/2026 -* New Providers: 5 +* New Providers: 7 * New Models / Updated Models: 116 new, 132 removed -* LLM API Endpoints: 5 -* Management Endpoints / UI: 11 -* AI Integrations: 2 -* Performance / Reliability: 5 +* LLM API Endpoints: 37 +* Management Endpoints / UI: 31 +* AI Integrations: 8 +* MCP Gateway: 5 +* Spend Tracking, Budgets and Rate Limiting: 5 +* Performance / Loadbalancing / Reliability improvements: 9 * Security: 3 * Database / Proxy Operations: 2 +* Documentation Updates: 5 --- diff --git a/docs/my-website/sidebars-release-notes.js b/docs/my-website/sidebars-release-notes.js new file mode 100644 index 00000000000..6ed29003ce1 --- /dev/null +++ b/docs/my-website/sidebars-release-notes.js @@ -0,0 +1,14 @@ +// @ts-check + +/** @type {import('@docusaurus/plugin-content-docs').SidebarsConfig} */ +const sidebars = { + releaseNotesSidebar: [ + { type: 'doc', id: 'index', label: 'Release Notes' }, + { + type: 'autogenerated', + dirName: '.', + }, + ], +}; + +module.exports = sidebars; diff --git a/docs/my-website/sidebars.js b/docs/my-website/sidebars.js index 8888a7229da..a1faf231036 100644 --- a/docs/my-website/sidebars.js +++ b/docs/my-website/sidebars.js @@ -20,6 +20,7 @@ const sidebars = { { type: "category", label: "Observability", + link: { type: "doc", id: "integrations/observability_index" }, items: [ { type: "category", @@ -29,61 +30,24 @@ const sidebars = { type: "autogenerated", dirName: "contribute_integration" } - ] + ], }, { type: "autogenerated", dirName: "observability" - } + }, ], }, { type: "category", - label: "Guardrails", + label: "Guardrail Providers", + link: { + type: "generated-index", + title: "Guardrail Providers", + description: "Add safety and content filtering to LLM calls", + slug: "/guardrail_providers" + }, items: [ - "proxy/guardrails/quick_start", - "proxy/guardrails/team_based_guardrails", - "proxy/guardrails/guardrail_load_balancing", - "proxy/guardrails/test_playground", - "proxy/guardrails/litellm_content_filter", - "proxy/guardrails/realtime_guardrails", - { - type: "category", - label: "Providers", - items: [ - ...[ - "proxy/guardrails/qualifire", - "proxy/guardrails/aim_security", - "proxy/guardrails/onyx_security", - "proxy/guardrails/aporia_api", - "proxy/guardrails/azure_content_guardrail", - "proxy/guardrails/bedrock", - "proxy/guardrails/crowdstrike_aidr", - "proxy/guardrails/enkryptai", - "proxy/guardrails/ibm_guardrails", - "proxy/guardrails/grayswan", - "proxy/guardrails/hiddenlayer", - "proxy/guardrails/lasso_security", - "proxy/guardrails/guardrails_ai", - "proxy/guardrails/lakera_ai", - "proxy/guardrails/model_armor", - "proxy/guardrails/noma_security", - "proxy/guardrails/dynamoai", - "proxy/guardrails/openai_moderation", - "proxy/guardrails/pangea", - "proxy/guardrails/pillar_security", - "proxy/guardrails/pii_masking_v2", - "proxy/guardrails/panw_prisma_airs", - "proxy/guardrails/secret_detection", - "proxy/guardrails/custom_guardrail", - "proxy/guardrails/custom_code_guardrail", - "proxy/guardrails/prompt_injection", - "proxy/guardrails/tool_permission", - "proxy/guardrails/zscaler_ai_guard", - "proxy/guardrails/javelin" - ].sort(), - ], - }, { type: "category", label: "Contributing to Guardrails", @@ -93,16 +57,42 @@ const sidebars = { "adding_provider/adding_guardrail_support", ] }, - ], - }, - { - type: "category", - label: "Policies", - items: [ - "proxy/guardrails/guardrail_policies", - "proxy/guardrails/policy_flow_builder", - "proxy/guardrails/policy_templates", - "proxy/guardrails/policy_tags", + { + type: "doc", + id: "proxy/guardrails/team_based_guardrails", + label: "Team Bring-Your-Own Guardrails", + }, + ...[ + "proxy/guardrails/qualifire", + "proxy/guardrails/aim_security", + "proxy/guardrails/onyx_security", + "proxy/guardrails/aporia_api", + "proxy/guardrails/azure_content_guardrail", + "proxy/guardrails/bedrock", + "proxy/guardrails/crowdstrike_aidr", + "proxy/guardrails/enkryptai", + "proxy/guardrails/ibm_guardrails", + "proxy/guardrails/grayswan", + "proxy/guardrails/hiddenlayer", + "proxy/guardrails/lasso_security", + "proxy/guardrails/guardrails_ai", + "proxy/guardrails/lakera_ai", + "proxy/guardrails/model_armor", + "proxy/guardrails/noma_security", + "proxy/guardrails/dynamoai", + "proxy/guardrails/openai_moderation", + "proxy/guardrails/pangea", + "proxy/guardrails/pillar_security", + "proxy/guardrails/pii_masking_v2", + "proxy/guardrails/panw_prisma_airs", + "proxy/guardrails/secret_detection", + "proxy/guardrails/custom_guardrail", + "proxy/guardrails/custom_code_guardrail", + "proxy/guardrails/prompt_injection", + "proxy/guardrails/tool_permission", + "proxy/guardrails/zscaler_ai_guard", + "proxy/guardrails/javelin" + ].sort(), ], }, { @@ -115,11 +105,6 @@ const sidebars = { "proxy/pyroscope_profiling" ] }, - { - type: "doc", - id: "integrations/websearch_interception", - label: "Web Search Integration" - }, { type: "category", label: "[Beta] Prompt Management", @@ -140,7 +125,7 @@ const sidebars = { }, { type: "category", - label: "AI Tools (OpenWebUI, Claude Code, etc.)", + label: "AI Tools", link: { type: "generated-index", title: "AI Tools", @@ -167,14 +152,13 @@ const sidebars = { }, "tutorials/opencode_integration", "tutorials/openclaw_integration", - "tutorials/cost_tracking_coding", "tutorials/cursor_integration", "tutorials/github_copilot_integration", "tutorials/litellm_gemini_cli", - "tutorials/google_genai_sdk", "tutorials/litellm_qwen_code_cli", "tutorials/openai_codex", - "tutorials/retool_assist" + "tutorials/retool_assist", + "tutorials/cost_tracking_coding" ] }, { @@ -191,7 +175,11 @@ const sidebars = { "tutorials/claude_agent_sdk", "tutorials/copilotkit_sdk", "tutorials/google_adk", + "tutorials/google_genai_sdk", "tutorials/livekit_xai_realtime", + "integrations/letta", + { type: "doc", id: "tutorials/instructor", label: "Instructor with LiteLLM" }, + { type: "doc", id: "langchain/langchain", label: "LangChain with LiteLLM" }, "projects/openai-agents" ] }, @@ -212,7 +200,20 @@ const sidebars = { ], // But you can create a sidebar manually tutorialSidebar: [ - { type: "doc", id: "index", label: "Getting Started" }, + // ════════════════════════════════════════════════════════════ + // GET STARTED + // ════════════════════════════════════════════════════════════ + { + type: "category", + label: "Get Started", + collapsible: false, + collapsed: false, + items: [ + { type: "doc", id: "index", label: "Quickstart" }, + { type: "link", label: "Models & Pricing", href: "https://models.litellm.ai" }, + { type: "link", label: "Changelog", href: "/release_notes" }, + ], + }, { type: "category", @@ -280,11 +281,6 @@ const sidebars = { }, "completion/token_usage", "exception_mapping", - { - type: "category", - label: "LangChain, LlamaIndex, Instructor", - items: ["langchain/langchain", "tutorials/instructor"], - } ], }, { @@ -297,16 +293,46 @@ const sidebars = { slug: "/simple_proxy", }, items: [ - "proxy/docker_quick_start", + { type: "doc", id: "proxy/docker_quick_start", label: "Getting Started Tutorial" }, { - type: "link", - label: "A2A Agent Gateway", - href: "https://docs.litellm.ai/docs/a2a", - }, - { - type: "link", - label: "MCP Gateway", - href: "https://docs.litellm.ai/docs/mcp", + type: "category", + label: "Agent & MCP Gateway", + items: [ + { + type: "category", + label: "A2A Agent Gateway", + items: [ + "a2a", + "a2a_invoking_agents", + "a2a_agent_headers", + "a2a_cost_tracking", + "a2a_agent_permissions", + "a2a_iteration_budgets", + ], + }, + { + type: "category", + label: "MCP Gateway", + items: [ + "mcp", + "mcp_usage", + "mcp_openapi", + "mcp_oauth", + "mcp_aws_sigv4", + "mcp_zero_trust", + "mcp_public_internet", + "mcp_semantic_filter", + "mcp_control", + "mcp_cost", + "mcp_guardrail", + { + type: "link", + label: "MCP Troubleshooting Guide", + href: "/docs/mcp_troubleshoot" + }, + ], + }, + ], }, { "type": "category", @@ -365,7 +391,11 @@ const sidebars = { type: "category", label: "Teams & Organizations", items: [ - "proxy/access_control", + { + type: "link", + label: "Role-based Access Controls (RBAC) →", + href: "/docs/proxy/access_control" + }, "proxy/self_serve", "proxy/public_teams", "proxy/ui_project_management", @@ -400,6 +430,7 @@ const sidebars = { "proxy/architecture", "proxy/multi_tenant_architecture", "proxy/control_plane_and_data_plane", + "proxy/high_availability_control_plane", "proxy/db_deadlocks", "proxy/db_info", "proxy/image_handling", @@ -448,14 +479,40 @@ const sidebars = { }, "proxy/caching", { - type: "link", + type: "category", label: "Guardrails", - href: "https://docs.litellm.ai/docs/proxy/guardrails/quick_start", + items: [ + "proxy/guardrails/quick_start", + "proxy/guardrails/team_based_guardrails", + "proxy/guardrails/guardrail_load_balancing", + "proxy/guardrails/test_playground", + "proxy/guardrails/litellm_content_filter", + "proxy/guardrails/realtime_guardrails", + { + type: "link", + label: "Providers →", + href: "/docs/guardrail_providers", + }, + { + type: "category", + label: "Contributing to Guardrails", + items: [ + "adding_provider/generic_guardrail_api", + "adding_provider/simple_guardrail_tutorial", + "adding_provider/adding_guardrail_support", + ] + }, + ], }, { - type: "link", + type: "category", label: "Policies", - href: "https://docs.litellm.ai/docs/proxy/guardrails/guardrail_policies", + items: [ + "proxy/guardrails/guardrail_policies", + "proxy/guardrails/policy_flow_builder", + "proxy/guardrails/policy_templates", + "proxy/guardrails/policy_tags", + ], }, { type: "category", @@ -528,6 +585,7 @@ const sidebars = { label: "Spend Tracking", items: [ "proxy/cost_tracking", + "tutorials/vertex_ai_pay_go", "proxy/request_tags", "proxy/custom_pricing", "proxy/pricing_calculator", @@ -551,16 +609,9 @@ const sidebars = { }, items: [ { - type: "category", + type: "link", label: "/a2a - A2A Agent Gateway", - items: [ - "a2a", - "a2a_invoking_agents", - "a2a_agent_headers", - "a2a_cost_tracking", - "a2a_agent_permissions", - "a2a_iteration_budgets" - ], + href: "/docs/a2a", }, "assistants", "audio_transcription", @@ -637,6 +688,7 @@ const sidebars = { "mcp_control", "mcp_cost", "mcp_guardrail", + "mcp_zero_trust", "mcp_troubleshoot", ] }, @@ -687,6 +739,7 @@ const sidebars = { "proxy/realtime_webrtc", "rerank", "response_api", + "prompt_management", "response_api_compact", { type: "category", @@ -797,6 +850,7 @@ const sidebars = { items: [ "providers/gemini", "providers/gemini/videos", + "providers/gemini/music", "providers/google_ai_studio/files", "providers/google_ai_studio/image_gen", "providers/google_ai_studio/realtime", @@ -974,51 +1028,14 @@ const sidebars = { "providers/zai", ], }, - { - type: "category", - label: "Guides", - items: [ - "budget_manager", - "completion/computer_use", - "completion/web_search", - "completion/web_fetch", - "completion/function_call", - "completion/audio", - "completion/document_understanding", - "completion/drop_params", - "completion/image_generation_chat", - "completion/json_mode", - "completion/knowledgebase", - "providers/anthropic_tool_search", - "guides/code_interpreter", - "completion/message_trimming", - "completion/message_sanitization", - "completion/model_alias", - "completion/mock_requests", - "completion/predict_outputs", - "completion/prefix", - "completion/prompt_caching", - "completion/prompt_formatting", - "completion/reliable_completions", - "completion/stream", - "completion/provider_specific_params", - "completion/vision", - "exception_mapping", - "completion/batching", - "guides/finetuned_models", - "guides/security_settings", - "proxy/veo_video_generation", - "reasoning_content", - "extras/creating_adapters", - ] - }, + { type: "category", - label: "Routing, Loadbalancing & Fallbacks", + label: "Routing & Load Balancing", link: { type: "generated-index", - title: "Routing, Loadbalancing & Fallbacks", + title: "Routing & Load Balancing", description: "Learn how to load balance, route, and set fallbacks for your LLM requests", slug: "/routing-load-balancing", }, @@ -1046,43 +1063,6 @@ const sidebars = { "load_test_rpm", ] }, - { - type: "category", - label: "Tutorials", - items: [ - { - type: "link", - label: "AI Coding Tools (OpenWebUI, Claude Code, Gemini CLI, OpenAI Codex, etc.)", - href: "/docs/ai_tools", - }, - "tutorials/anthropic_file_usage", - "tutorials/default_team_self_serve", - "tutorials/msft_sso", - "tutorials/prompt_caching", - "tutorials/tag_management", - 'tutorials/litellm_proxy_aporia', - "tutorials/presidio_pii_masking", - "tutorials/elasticsearch_logging", - "tutorials/gemini_realtime_with_audio", - "tutorials/claude_code_beta_headers", - { - type: "category", - label: "LiteLLM Python SDK Tutorials", - items: [ - 'tutorials/azure_openai', - 'tutorials/instructor', - "tutorials/gradio_integration", - "tutorials/huggingface_codellama", - "tutorials/huggingface_tutorial", - "tutorials/TogetherAI_liteLLM", - "tutorials/finetuned_chat_gpt", - "tutorials/text_completion", - "tutorials/first_playground", - "tutorials/model_fallbacks", - ], - }, - ] - }, { type: "category", label: "Contributing", @@ -1140,12 +1120,11 @@ const sidebars = { "projects/Codium PR Agent", "projects/Prompt2Model", "projects/SalesGPT", + "projects/Softgen", "projects/Quivr", "projects/Langstream", "projects/Otter", - "projects/GPT Migrate", "projects/YiVal", - "projects/LiteLLM Proxy", "projects/llm_cord", "projects/pgai", "projects/GPTLocalhost", @@ -1183,23 +1162,310 @@ const sidebars = { "troubleshoot", ], }, + ], +}; + +const learnSidebar = { + learnSidebar: [ + // ── Landing page ────────────────────────────────────────────────── + { type: "doc", id: "learn/index", label: "Learn" }, { type: "category", - label: "Blog", + label: "Start Here", + collapsible: true, + collapsed: false, + items: [ + "learn/sdk_quickstart", + "learn/gateway_quickstart", + ], + }, + + // ── Guides ──────────────────────────────────────────────────────── + { + type: "category", + label: "Guides", + collapsible: true, + collapsed: false, + link: { type: "doc", id: "guides/index" }, items: [ { - type: "link", - label: "Day 0 Support: Claude Sonnet 4.6", - href: "/blog/claude_sonnet_4_6", + type: "category", + label: "Core Requests", + collapsible: true, + collapsed: false, + link: { + type: "generated-index", + title: "Core Requests", + description: "Streaming, batching, structured outputs, and reasoning behavior", + slug: "/guides/core_request_response_patterns" + }, + items: [ + "completion/stream", + "completion/batching", + "completion/json_mode", + "reasoning_content", + ], + }, + { + type: "category", + label: "Tool Calling", + collapsible: true, + collapsed: true, + link: { + type: "generated-index", + title: "Tool Calling", + description: "Function calling, web tools, interception patterns, computer use, code interpreter, and tool-call hygiene", + slug: "/guides/tools_integrations" + }, + items: [ + "completion/function_call", + "completion/web_search", + { + type: "doc", + id: "integrations/websearch_interception", + label: "Web Search Interception", + }, + "completion/web_fetch", + "completion/computer_use", + "guides/code_interpreter", + "completion/message_sanitization", + ], + }, + { + type: "category", + label: "Multimodal I/O", + collapsible: true, + collapsed: true, + link: { + type: "generated-index", + title: "Multimodal I/O", + description: "Vision, audio, PDFs, image generation, and video generation", + slug: "/guides/multimodal_io" + }, + items: [ + "completion/vision", + "completion/audio", + "completion/document_understanding", + "completion/image_generation_chat", + "proxy/veo_video_generation", + ], + }, + { + type: "category", + label: "Retrieval & Knowledge", + collapsible: true, + collapsed: true, + link: { + type: "generated-index", + title: "Retrieval & Knowledge", + description: "Vector stores, file search, citations, and knowledge-base routing", + slug: "/guides/retrieval_knowledge" + }, + items: [ + "completion/knowledgebase", + ], + }, + { + type: "category", + label: "Prompts & Context", + collapsible: true, + collapsed: true, + link: { + type: "generated-index", + title: "Prompts & Context", + description: "Prompt caching, trimming, formatting, assistant prefill, and predicted outputs", + slug: "/guides/prompts_context" + }, + items: [ + "completion/prefix", + "completion/predict_outputs", + "completion/message_trimming", + "completion/prompt_caching", + "completion/prompt_formatting", + ], + }, + { + type: "category", + label: "Compatibility & Extensibility", + collapsible: true, + collapsed: true, + link: { + type: "generated-index", + title: "Compatibility & Extensibility", + description: "Provider-specific params, model aliases, fine-tuned models, and adapters", + slug: "/guides/compatibility_extensibility" + }, + items: [ + "completion/provider_specific_params", + "completion/drop_params", + "completion/model_alias", + "guides/finetuned_models", + "extras/creating_adapters", + ], + }, + { + type: "category", + label: "Reliability, Testing & Spend", + collapsible: true, + collapsed: true, + link: { + type: "generated-index", + title: "Reliability, Testing & Spend", + description: "Retries, fallbacks, mock responses, and budget controls", + slug: "/guides/reliability_testing_spend" + }, + items: [ + "completion/mock_requests", + "completion/reliable_completions", + "budget_manager", + ], + }, + { + type: "category", + label: "Security & Network", + collapsible: true, + collapsed: true, + link: { + type: "generated-index", + title: "Security & Network", + description: "SSL, custom CA bundles, HTTP proxy settings, and per-service verification", + slug: "/guides/security_network" + }, + items: [ + "guides/security_settings", + ], + }, + ], + }, + + // ── Tutorials ───────────────────────────────────────────────────── + { + type: "category", + label: "Tutorials", + collapsible: true, + collapsed: false, + link: { type: "doc", id: "tutorials/index" }, + items: [ + { + type: "category", + label: "Getting Started", + collapsed: false, + link: { + type: "generated-index", + title: "Getting Started", + description: "Installation, playground, text completion, and mock completions", + slug: "/tutorials/getting_started" + }, + items: [ + "tutorials/installation", + "tutorials/first_playground", + "tutorials/text_completion", + "tutorials/mock_completion", + ], }, { type: "link", - label: "Incident: Broken Model Cost Map", - href: "/blog/model-cost-map-incident", + label: "Agent SDKs & Frameworks", + href: "/docs/agent_sdks", + }, + { + type: "link", + label: "AI Coding Tools", + href: "/docs/ai_tools", + }, + { + type: "category", + label: "Python SDK", + collapsed: true, + link: { + type: "generated-index", + title: "Python SDK", + description: "Tutorials using only the Python SDK — no proxy server required", + slug: "/tutorials/python_sdk" + }, + items: [ + "tutorials/gradio_integration", + "tutorials/provider_specific_params", + "tutorials/model_fallbacks", + "tutorials/fallbacks", + ], + }, + { + type: "category", + label: "Provider Setup", + collapsed: true, + link: { + type: "generated-index", + title: "Provider Setup", + description: "Connect LiteLLM to Azure OpenAI, HuggingFace, TogetherAI, local models, and more", + slug: "/tutorials/provider_tutorials" + }, + items: [ + "tutorials/azure_openai", + "tutorials/TogetherAI_liteLLM", + "tutorials/huggingface_tutorial", + "tutorials/huggingface_codellama", + "tutorials/finetuned_chat_gpt", + "tutorials/oobabooga", + ], + }, + { + type: "category", + label: "Proxy: Admin & Access", + collapsed: true, + link: { + type: "generated-index", + title: "Proxy: Admin & Access", + description: "User and team management, SSO, SCIM, and routing rules", + slug: "/tutorials/proxy_admin_access" + }, + items: [ + "tutorials/default_team_self_serve", + "tutorials/msft_sso", + "tutorials/scim_litellm", + "tutorials/tag_management", + ], + }, + { + type: "category", + label: "Proxy: Features & Safety", + collapsed: true, + link: { + type: "generated-index", + title: "Proxy: Features & Safety", + description: "Prompt caching, passthrough APIs, realtime, guardrails, and PII masking", + slug: "/tutorials/proxy_features_safety" + }, + items: [ + "tutorials/prompt_caching", + "tutorials/file_search_responses_api", + "tutorials/anthropic_file_usage", + "tutorials/gemini_realtime_with_audio", + "tutorials/litellm_proxy_aporia", + "tutorials/presidio_pii_masking", + ], + }, + { + type: "category", + label: "Observability & Evaluation", + collapsed: true, + link: { + type: "generated-index", + title: "Observability & Evaluation", + description: "Logging, monitoring, benchmarking, and evaluation suites", + slug: "/tutorials/observability_evaluation" + }, + items: [ + "tutorials/elasticsearch_logging", + "tutorials/compare_llms", + "tutorials/litellm_Test_Multiple_Providers", + "tutorials/eval_suites", + "tutorials/lm_evaluation_harness", + ], }, ], }, ], }; -module.exports = sidebars; +module.exports = { ...sidebars, ...learnSidebar }; diff --git a/docs/my-website/src/components/ControlPlaneArchitecture/ControlPlaneArchitecture.tsx b/docs/my-website/src/components/ControlPlaneArchitecture/ControlPlaneArchitecture.tsx new file mode 100644 index 00000000000..5cc41979b02 --- /dev/null +++ b/docs/my-website/src/components/ControlPlaneArchitecture/ControlPlaneArchitecture.tsx @@ -0,0 +1,95 @@ +import React from 'react'; +import styles from './styles.module.css'; + +/* ────────────────────── Shared small pieces ────────────────────── */ + +function InfraChip({ color, label }: { color: string; label: string }) { + const dotClass = + color === 'green' + ? styles.infraDotGreen + : color === 'blue' + ? styles.infraDotBlue + : styles.infraDotOrange; + + return ( + + + {label} + + ); +} + +/* ────────────────────── Architecture tab ────────────────────── */ + +function ArchitectureView() { + return ( +
+ {/* User */} +
+
👤
+ Admin +
+ +
+ + {/* Control Plane */} +
+
+ Control Plane + UI +
+
cp.example.com
+
+ + + +
+
+ + {/* Branch connector */} +
+
+
+
+ + {/* Workers */} +
+
+
+ Worker A + US East +
+
worker-a.example.com
+
+ + + +
+
+ +
+
+ Worker B + EU West +
+
worker-b.example.com
+
+ + + +
+
+
+
+ ); +} + +/* ────────────────────── Main component ────────────────────── */ + +export default function ControlPlaneArchitecture() { + return ( +
+ +
+ ); +} diff --git a/docs/my-website/src/components/ControlPlaneArchitecture/index.tsx b/docs/my-website/src/components/ControlPlaneArchitecture/index.tsx new file mode 100644 index 00000000000..826b4d68818 --- /dev/null +++ b/docs/my-website/src/components/ControlPlaneArchitecture/index.tsx @@ -0,0 +1 @@ +export { default as ControlPlaneArchitecture } from './ControlPlaneArchitecture'; diff --git a/docs/my-website/src/components/ControlPlaneArchitecture/styles.module.css b/docs/my-website/src/components/ControlPlaneArchitecture/styles.module.css new file mode 100644 index 00000000000..8400a8920e0 --- /dev/null +++ b/docs/my-website/src/components/ControlPlaneArchitecture/styles.module.css @@ -0,0 +1,517 @@ +/* ── Custom properties ── */ +:root { + --cp-bg: #ffffff; + --cp-border: #e5e7eb; + --cp-text: #1a1a2e; + --cp-text-secondary: #6b7280; + --cp-text-muted: #9ca3af; + --cp-accent: #3b82f6; + --cp-accent-light: #dbeafe; + --cp-accent-glow: rgba(59, 130, 246, 0.15); + --cp-green: #10b981; + --cp-green-light: #d1fae5; + --cp-green-glow: rgba(16, 185, 129, 0.15); + --cp-orange: #f59e0b; + --cp-orange-light: #fef3c7; + --cp-purple: #8b5cf6; + --cp-purple-light: #ede9fe; + --cp-red: #ef4444; + --cp-red-light: #fee2e2; + --cp-card-bg: #f9fafb; + --cp-infra-bg: #f1f5f9; + --cp-infra-border: #cbd5e1; + --cp-connector: #d1d5db; + --cp-dot-size: 8px; +} + +[data-theme='dark'] { + --cp-bg: #111827; + --cp-border: #374151; + --cp-text: #e5e7eb; + --cp-text-secondary: #9ca3af; + --cp-text-muted: #6b7280; + --cp-accent: #60a5fa; + --cp-accent-light: #1e3a5f; + --cp-accent-glow: rgba(96, 165, 250, 0.2); + --cp-green: #34d399; + --cp-green-light: #064e3b; + --cp-green-glow: rgba(52, 211, 153, 0.2); + --cp-orange: #fbbf24; + --cp-orange-light: #78350f; + --cp-purple: #a78bfa; + --cp-purple-light: #3b0764; + --cp-red: #f87171; + --cp-red-light: #451a1a; + --cp-card-bg: #1f2937; + --cp-infra-bg: #1e293b; + --cp-infra-border: #475569; + --cp-connector: #4b5563; +} + +/* ── Wrapper ── */ +.wrapper { + margin: 1.5rem 0; + font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif; +} + +/* ── Tab bar ── */ +.tabs { + display: flex; + gap: 0; + margin-bottom: 1.5rem; + border-bottom: 2px solid var(--cp-border); +} + +.tab { + padding: 0.6rem 1.25rem; + font-size: 0.85rem; + font-weight: 600; + color: var(--cp-text-secondary); + background: none; + border: none; + border-bottom: 2px solid transparent; + margin-bottom: -2px; + cursor: pointer; + transition: color 0.2s, border-color 0.2s; +} + +.tab:hover { + color: var(--cp-text); +} + +.tabActive { + color: var(--cp-accent); + border-bottom-color: var(--cp-accent); +} + +/* ── Architecture diagram ── */ +.diagram { + display: flex; + flex-direction: column; + align-items: center; + gap: 0; +} + +/* ── User icon ── */ +.userRow { + display: flex; + flex-direction: column; + align-items: center; + margin-bottom: 0.5rem; +} + +.userIcon { + width: 40px; + height: 40px; + border-radius: 50%; + background: var(--cp-accent-light); + border: 2px solid var(--cp-accent); + display: flex; + align-items: center; + justify-content: center; + font-size: 1.1rem; +} + +.userLabel { + font-size: 0.75rem; + color: var(--cp-text-secondary); + margin-top: 0.3rem; + font-weight: 500; +} + +/* ── Connectors ── */ +.connectorDown { + width: 2px; + height: 28px; + background: var(--cp-connector); + position: relative; +} + +.connectorDown::after { + content: ''; + position: absolute; + bottom: -4px; + left: 50%; + transform: translateX(-50%); + width: 0; + height: 0; + border-left: 5px solid transparent; + border-right: 5px solid transparent; + border-top: 5px solid var(--cp-connector); +} + +.connectorBranch { + display: flex; + align-items: flex-start; + justify-content: center; + position: relative; + width: 100%; + max-width: 700px; + height: 36px; +} + +.connectorBranch::before { + content: ''; + position: absolute; + top: 0; + left: 50%; + width: 2px; + height: 12px; + background: var(--cp-connector); + transform: translateX(-50%); +} + +.connectorBranch::after { + content: ''; + position: absolute; + top: 12px; + left: calc(25% + 12px); + right: calc(25% + 12px); + height: 2px; + background: var(--cp-connector); +} + +.branchLeg { + position: absolute; + top: 12px; + width: 2px; + height: 24px; + background: var(--cp-connector); +} + +.branchLeg::after { + content: ''; + position: absolute; + bottom: -4px; + left: 50%; + transform: translateX(-50%); + width: 0; + height: 0; + border-left: 5px solid transparent; + border-right: 5px solid transparent; + border-top: 5px solid var(--cp-connector); +} + +.branchLegLeft { + left: calc(25% + 12px); +} + +.branchLegRight { + right: calc(25% + 12px); +} + +/* ── Node cards ── */ +.node { + border: 2px solid var(--cp-border); + border-radius: 12px; + background: var(--cp-card-bg); + padding: 1rem 1.25rem; + text-align: center; + transition: border-color 0.3s, box-shadow 0.3s; + position: relative; +} + +.nodeControlPlane { + border-color: var(--cp-accent); + box-shadow: 0 0 0 3px var(--cp-accent-glow); + min-width: 280px; +} + +.nodeWorker { + min-width: 220px; +} + +.nodeWorkerA { + border-color: var(--cp-green); + box-shadow: 0 0 0 3px var(--cp-green-glow); +} + +.nodeWorkerB { + border-color: var(--cp-purple); + box-shadow: 0 0 0 3px rgba(139, 92, 246, 0.15); +} + +[data-theme='dark'] .nodeWorkerB { + box-shadow: 0 0 0 3px rgba(167, 139, 250, 0.2); +} + +.nodeHeader { + display: flex; + align-items: center; + justify-content: center; + gap: 0.5rem; + margin-bottom: 0.5rem; +} + +.nodeIcon { + font-size: 1.1rem; +} + +.nodeTitle { + font-size: 0.95rem; + font-weight: 700; + color: var(--cp-text); +} + +.nodeSubtitle { + font-size: 0.75rem; + color: var(--cp-text-secondary); + margin-bottom: 0.75rem; +} + +.badge { + display: inline-block; + font-size: 0.65rem; + font-weight: 600; + padding: 0.15rem 0.5rem; + border-radius: 9999px; + text-transform: uppercase; + letter-spacing: 0.04em; +} + +.badgeBlue { + background: var(--cp-accent-light); + color: var(--cp-accent); +} + +.badgeGreen { + background: var(--cp-green-light); + color: var(--cp-green); +} + +.badgePurple { + background: var(--cp-purple-light); + color: var(--cp-purple); +} + +/* ── Infrastructure chips ── */ +.infraRow { + display: flex; + gap: 0.4rem; + justify-content: center; + flex-wrap: wrap; + margin-top: 0.5rem; +} + +.infraChip { + display: flex; + align-items: center; + gap: 0.3rem; + font-size: 0.7rem; + font-weight: 500; + color: var(--cp-text-secondary); + background: var(--cp-infra-bg); + border: 1px solid var(--cp-infra-border); + border-radius: 6px; + padding: 0.2rem 0.5rem; +} + +.infraDot { + width: 6px; + height: 6px; + border-radius: 50%; + flex-shrink: 0; +} + +.infraDotGreen { + background: var(--cp-green); +} + +.infraDotBlue { + background: var(--cp-accent); +} + +.infraDotOrange { + background: var(--cp-orange); +} + +/* ── Workers row ── */ +.workersRow { + display: flex; + gap: 2rem; + justify-content: center; + flex-wrap: wrap; +} + +/* ── Animated flow ── */ +.flowLabel { + font-size: 0.7rem; + color: var(--cp-accent); + font-weight: 600; + position: absolute; + white-space: nowrap; +} + +/* ── Comparison view ── */ +.comparisonGrid { + display: grid; + grid-template-columns: 1fr 1fr; + gap: 1.5rem; + margin-top: 0.5rem; +} + +.comparisonColumn { + border: 2px solid var(--cp-border); + border-radius: 12px; + padding: 1.25rem; + background: var(--cp-card-bg); +} + +.comparisonColumnOld { + border-color: var(--cp-red); +} + +.comparisonColumnNew { + border-color: var(--cp-green); +} + +.comparisonTitle { + font-size: 0.9rem; + font-weight: 700; + color: var(--cp-text); + text-align: center; + margin-bottom: 1rem; + display: flex; + align-items: center; + justify-content: center; + gap: 0.4rem; +} + +.comparisonTitleOld { + color: var(--cp-red); +} + +.comparisonTitleNew { + color: var(--cp-green); +} + +/* ── Mini diagram inside comparison ── */ +.miniDiagram { + display: flex; + flex-direction: column; + align-items: center; + gap: 0.5rem; +} + +.miniNode { + border: 1.5px solid var(--cp-border); + border-radius: 8px; + background: var(--cp-bg); + padding: 0.5rem 0.75rem; + text-align: center; + font-size: 0.75rem; + font-weight: 600; + color: var(--cp-text); + width: 100%; + max-width: 180px; +} + +.miniNodeHighlight { + border-color: var(--cp-accent); + background: var(--cp-accent-light); +} + +.miniNodeDanger { + border-color: var(--cp-red); + background: var(--cp-red-light); +} + +.miniNodeSuccess { + border-color: var(--cp-green); + background: var(--cp-green-light); +} + +.miniConnector { + width: 1.5px; + height: 16px; + background: var(--cp-connector); +} + +.miniWorkersRow { + display: flex; + gap: 0.5rem; + justify-content: center; + width: 100%; +} + +.miniWorkerStack { + display: flex; + flex-direction: column; + align-items: center; + gap: 0.3rem; + flex: 1; + max-width: 140px; +} + +.miniInfra { + font-size: 0.65rem; + color: var(--cp-text-muted); + font-weight: 500; +} + +.miniInfraShared { + color: var(--cp-red); + font-weight: 600; +} + +.miniInfraOwn { + color: var(--cp-green); + font-weight: 600; +} + +/* ── Callout box ── */ +.callout { + display: flex; + align-items: flex-start; + gap: 0.6rem; + padding: 0.75rem 1rem; + border-radius: 8px; + margin-top: 1rem; + font-size: 0.8rem; + color: var(--cp-text); + line-height: 1.5; +} + +.calloutDanger { + background: var(--cp-red-light); + border: 1px solid var(--cp-red); +} + +.calloutSuccess { + background: var(--cp-green-light); + border: 1px solid var(--cp-green); +} + +.calloutIcon { + font-size: 1rem; + flex-shrink: 0; + margin-top: 0.1rem; +} + +/* ── Responsive ── */ +@media (max-width: 768px) { + .comparisonGrid { + grid-template-columns: 1fr; + } + + .workersRow { + flex-direction: column; + align-items: center; + } + + .nodeControlPlane { + min-width: auto; + width: 100%; + max-width: 300px; + } + + .nodeWorker { + min-width: auto; + width: 100%; + max-width: 260px; + } + + .connectorBranch { + display: none; + } +} diff --git a/docs/my-website/src/components/NavigationCards/index.js b/docs/my-website/src/components/NavigationCards/index.js new file mode 100644 index 00000000000..5efd89ee140 --- /dev/null +++ b/docs/my-website/src/components/NavigationCards/index.js @@ -0,0 +1,44 @@ +import React from 'react'; +import Link from '@docusaurus/Link'; +import styles from './styles.module.css'; + +export default function NavigationCards({ items, columns = 2 }) { + return ( +
+ {items.map((item, i) => { + const isExternal = + item.to && (item.to.startsWith('http://') || item.to.startsWith('https://')); + return ( + + {item.icon && ( +
{item.icon}
+ )} +
{item.title}
+ {item.description && ( +
{item.description}
+ )} + {item.listDescription && ( +
    + {item.listDescription.map((line, j) => ( +
  • {line}
    • + ))} +
+ )} + {isExternal && ( + ↗ + )} + + ); + })} +
+ ); +} diff --git a/docs/my-website/src/components/NavigationCards/styles.module.css b/docs/my-website/src/components/NavigationCards/styles.module.css new file mode 100644 index 00000000000..64f5a42374b --- /dev/null +++ b/docs/my-website/src/components/NavigationCards/styles.module.css @@ -0,0 +1,82 @@ +.grid { + display: grid; + grid-template-columns: repeat(var(--nav-columns, 2), 1fr); + gap: 0.75rem; + margin: 1.25rem 0; +} + +@media (max-width: 768px) { + .grid { + grid-template-columns: 1fr; + } +} + +.card { + position: relative; + display: flex; + flex-direction: column; + padding: 1rem 1.1rem; + border: 1px solid var(--ifm-color-emphasis-200); + border-radius: 6px; + text-decoration: none !important; + color: inherit !important; + background: var(--ifm-background-surface-color); + transition: border-color 0.15s ease, box-shadow 0.15s ease; +} + +.card:hover { + border-color: var(--ifm-color-primary); + box-shadow: 0 0 0 1px var(--ifm-color-primary); + text-decoration: none !important; +} + +[data-theme='dark'] .card { + background: var(--ifm-background-surface-color); + border-color: #2d3748; +} + +[data-theme='dark'] .card:hover { + border-color: var(--ifm-color-primary); + box-shadow: 0 0 0 1px var(--ifm-color-primary); +} + +.icon { + font-size: 1.4rem; + margin-bottom: 0.5rem; + line-height: 1; +} + +.title { + font-size: 14px; + font-weight: 600; + margin-bottom: 0.35rem; + color: var(--ifm-heading-color); +} + +.description { + font-size: 13px; + line-height: 1.5; + color: var(--ifm-color-emphasis-700); + margin-bottom: 0.5rem; +} + +.list { + margin: 0.35rem 0 0 0; + padding-left: 1.1rem; + list-style: disc; +} + +.list li { + font-size: 12.5px; + color: var(--ifm-color-emphasis-700); + line-height: 1.6; + margin-bottom: 0; +} + +.externalIcon { + position: absolute; + top: 0.75rem; + right: 0.75rem; + font-size: 12px; + color: var(--ifm-color-emphasis-500); +} diff --git a/docs/my-website/src/components/VersionVerificationTable/index.tsx b/docs/my-website/src/components/VersionVerificationTable/index.tsx new file mode 100644 index 00000000000..de4caced04b --- /dev/null +++ b/docs/my-website/src/components/VersionVerificationTable/index.tsx @@ -0,0 +1,84 @@ +import React, { useState } from "react"; +import styles from "./styles.module.css"; + +interface VersionEntry { + version: string; + sha256: string; + gitCommit: string; +} + +interface Props { + entries: VersionEntry[]; +} + +function CopyButton({ text }: { text: string }) { + const [copied, setCopied] = useState(false); + + const handleCopy = () => { + navigator.clipboard.writeText(text).then(() => { + setCopied(true); + setTimeout(() => setCopied(false), 1500); + }); + }; + + return ( + + ); +} + +export default function VersionVerificationTable({ entries }: Props) { + return ( +
+ + + + + + + + + + + + + {entries.map((entry) => ( + + + + + + + + + ))} + +
VersionSHA-256Clean of IOCsMatches GitGit CommitStatus
{entry.version} + + {entry.sha256.slice(0, 16)}… + + + + ✔ CLEAN + + ✔ YES + + + {entry.gitCommit} + + + ✔ CLEAN +
+
+ ); +} diff --git a/docs/my-website/src/components/VersionVerificationTable/styles.module.css b/docs/my-website/src/components/VersionVerificationTable/styles.module.css new file mode 100644 index 00000000000..97d2eb17e1c --- /dev/null +++ b/docs/my-website/src/components/VersionVerificationTable/styles.module.css @@ -0,0 +1,106 @@ +.wrapper { + overflow-x: auto; + margin: 1rem 0; +} + +.table { + width: 100%; + border-collapse: separate; + border-spacing: 0; + font-size: 0.9rem; + border: 1px solid var(--ifm-color-emphasis-300); + border-radius: 8px; + overflow: hidden; +} + +.table th, +.table td { + padding: 0.6rem 0.75rem; + text-align: left; + white-space: nowrap; +} + +.table thead th { + background: var(--ifm-color-emphasis-200); + font-weight: 600; + font-size: 0.8rem; + text-transform: uppercase; + letter-spacing: 0.03em; + color: var(--ifm-color-emphasis-700); + border-bottom: 2px solid var(--ifm-color-emphasis-300); +} + +.table tbody tr:nth-child(even) { + background: var(--ifm-color-emphasis-100); +} + +.table tbody tr:hover { + background: var(--ifm-color-emphasis-200); +} + +.table tbody td { + border-bottom: 1px solid var(--ifm-color-emphasis-200); +} + +.table tbody tr:last-child td { + border-bottom: none; +} + +.badge { + display: inline-flex; + align-items: center; + gap: 4px; + padding: 2px 8px; + border-radius: 12px; + font-size: 0.75rem; + font-weight: 600; + line-height: 1.4; +} + +.badgeClean { + composes: badge; + background: #d4edda; + color: #155724; +} + +.badgeYes { + composes: badge; + background: #cce5ff; + color: #004085; +} + +.sha { + display: inline-flex; + align-items: center; + gap: 4px; + font-family: var(--ifm-font-family-monospace); + font-size: 0.8rem; +} + +.copyBtn { + display: inline-flex; + align-items: center; + justify-content: center; + background: none; + border: 1px solid var(--ifm-color-emphasis-300); + border-radius: 4px; + cursor: pointer; + padding: 2px 4px; + font-size: 0.7rem; + color: var(--ifm-color-emphasis-600); + transition: background 0.15s, color 0.15s; +} + +.copyBtn:hover { + background: var(--ifm-color-emphasis-200); + color: var(--ifm-color-emphasis-800); +} + +.commitLink { + font-family: var(--ifm-font-family-monospace); + font-size: 0.8rem; +} + +.version { + font-weight: 600; +} diff --git a/docs/my-website/src/css/custom.css b/docs/my-website/src/css/custom.css index 9fa4443afc9..d0702fcc57c 100644 --- a/docs/my-website/src/css/custom.css +++ b/docs/my-website/src/css/custom.css @@ -1,10 +1,16 @@ /** - * Any CSS included here will be global. The classic template - * bundles Infima by default. Infima is a CSS framework designed to - * work well for content-centric websites. + * Global CSS overrides for LiteLLM docs. + * Infima (Docusaurus CSS framework) variables + custom styling. */ -/* You can override the default Infima variables here. */ +/* ========================================= + FONTS + ========================================= */ +@import url('https://fonts.googleapis.com/css2?family=Inter:wght@300;400;500;600;700&display=swap'); + +/* ========================================= + ROOT — Light Mode Variables + ========================================= */ :root { --ifm-color-primary: #2e8555; --ifm-color-primary-dark: #29784c; @@ -13,11 +19,22 @@ --ifm-color-primary-light: #33925d; --ifm-color-primary-lighter: #359962; --ifm-color-primary-lightest: #3cad6e; - --ifm-code-font-size: 95%; - --docusaurus-highlighted-code-line-bg: rgba(0, 0, 0, 0.1); + --ifm-code-font-size: 85%; + --ifm-menu-color: #6b7280; + --docusaurus-highlighted-code-line-bg: rgba(0, 0, 0, 0.08); + --ifm-font-family-base: 'Inter', -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif; + --ifm-heading-font-weight: 600; + --ifm-font-size-base: 15px; + --ifm-line-height-base: 1.65; + --ifm-border-radius: 6px; + /* Wider reading column — reduces excessive whitespace on large monitors */ + --ifm-container-width: 1380px; + --ifm-container-width-xl: 1560px; } -/* For readability concerns, you should choose a lighter palette in dark mode. */ +/* ========================================= + DARK MODE Variables + ========================================= */ [data-theme='dark'] { --ifm-color-primary: #25c2a0; --ifm-color-primary-dark: #21af90; @@ -26,10 +43,710 @@ --ifm-color-primary-light: #29d5b0; --ifm-color-primary-lighter: #32d8b4; --ifm-color-primary-lightest: #4fddbf; - --docusaurus-highlighted-code-line-bg: rgba(0, 0, 0, 0.3); + --ifm-background-color: #0d1117; + --ifm-background-surface-color: #161b22; + --docusaurus-highlighted-code-line-bg: rgba(255, 255, 255, 0.07); } -/* Levo logo sizing and theme switching */ +/* ========================================= + TYPOGRAPHY + ========================================= */ +.theme-doc-markdown h1 { + font-size: 2.2rem; + letter-spacing: -0.02em; + line-height: 1.2; +} + +.theme-doc-markdown h2 { + font-size: 1.6rem; + letter-spacing: -0.01em; + line-height: 1.3; +} + +.theme-doc-markdown h3 { + font-size: 1.25rem; + line-height: 1.4; +} + +.theme-doc-markdown p, +.theme-doc-markdown ul, +.theme-doc-markdown ol { + font-size: 0.9rem; +} + +.theme-doc-markdown table { + font-size: 0.875rem; +} + +.theme-doc-markdown td { + font-size: 0.85rem; +} + +/* ========================================= + NAVBAR + ========================================= */ +[data-theme='light'] .navbar { + background-color: #ffffff; + box-shadow: 0 1px 0 0 #e5e7eb; +} + +[data-theme='dark'] .navbar { + background-color: var(--ifm-background-color); + border-bottom: 1px solid #21262d; + box-shadow: none; +} + +.navbar__link { + font-weight: 400 !important; + font-size: 14px !important; + border-bottom: 2px solid transparent !important; + padding-bottom: 2px; +} + +.navbar__link--active { + font-weight: 500 !important; + border-bottom: 2px solid var(--ifm-color-primary) !important; +} + +@media (max-width: 1330px) { + .navbar__link { + font-size: 13px !important; + } +} + +/* Three-column navbar: logo | center nav | right icons */ +@media (min-width: 997px) { + .navbar__inner { + display: flex !important; + align-items: center; + justify-content: space-between; + } + + .navbar__brand-col { + display: flex; + align-items: center; + flex: 0 0 auto; + margin-left: 1rem; + } + + .navbar__brand-col .navbar__brand { + font-size: 1.25rem; + } + + .navbar__brand-col .navbar__logo { + height: 2rem; + width: auto; + } + + .navbar__center-col { + display: flex; + align-items: center; + justify-content: center; + flex: 1; + } + + .navbar__right-col { + display: flex; + align-items: center; + flex: 0 0 auto; + gap: 0.25rem; + margin-right: 2rem; + } +} + +/* ========================================= + ALERTS / ADMONITIONS + ========================================= */ +.alert { + padding: 0.75rem 1rem; + font-size: 14px; + border-radius: var(--ifm-border-radius); + border-left-width: 3px; +} + +/* Light mode */ +.alert--info { + --ifm-alert-background-color: #f0f7ff !important; + --ifm-alert-border-color: #2264ab !important; +} + +.alert--success { + --ifm-alert-background-color: #f0fff8 !important; + --ifm-alert-border-color: #09bda8 !important; +} + +.alert--secondary { + --ifm-alert-background-color: #f8fafc !important; + --ifm-alert-border-color: #64748b !important; +} + +.alert--danger { + --ifm-alert-background-color: #fff0f5 !important; + --ifm-alert-border-color: #e11d48 !important; +} + +.alert--warning { + --ifm-alert-background-color: #fffbeb !important; + --ifm-alert-border-color: #d97706 !important; +} + +/* Dark mode */ +[data-theme='dark'] .alert--info { + --ifm-alert-background-color: #0c1e30 !important; + --ifm-alert-border-color: #3b82f6 !important; + color: #bfdbfe !important; +} + +[data-theme='dark'] .alert--success { + --ifm-alert-background-color: #022c22 !important; + --ifm-alert-border-color: #10b981 !important; +} + +[data-theme='dark'] .alert--secondary { + --ifm-alert-background-color: #0f172a !important; + --ifm-alert-border-color: #475569 !important; +} + +[data-theme='dark'] .alert--danger { + --ifm-alert-background-color: #2d0a14 !important; + --ifm-alert-border-color: #f43f5e !important; +} + +[data-theme='dark'] .alert--warning { + --ifm-alert-background-color: #1c1200 !important; + --ifm-alert-border-color: #f59e0b !important; +} + +/* ========================================= + COLLAPSIBLE / DETAILS + ========================================= */ +details { + color: #1d232e; + background-color: #ffffff; + border: 1px solid #e9eef2 !important; + border-radius: var(--ifm-border-radius) !important; + padding: 0.75rem !important; + margin-bottom: 1rem !important; + margin-top: 1.5rem !important; + box-shadow: 0 1px 4px 0 rgba(0, 0, 0, 0.05) !important; +} + +details summary { + font-weight: 600 !important; +} + +details [class*='collapsibleContent'] { + border-top: 1px solid #e2e8f0 !important; +} + +details [class*='collapsibleContent'] p, +details [class*='collapsibleContent'] ul { + font-size: 13px !important; + line-height: 1.75; +} + +[data-theme='dark'] details { + background-color: #1c2130 !important; + color: #e5e7eb !important; + border: 1px solid #2d3748 !important; + box-shadow: 0 1px 6px 0 rgba(0, 0, 0, 0.3) !important; +} + +[data-theme='dark'] details summary { + color: #f3f4f6 !important; +} + +[data-theme='dark'] details [class*='collapsibleContent'] { + border-top: 1px solid #2d3748 !important; +} + +/* ========================================= + TABS + ========================================= */ +.tabs-container > div { + padding: 1rem; + border: 1px solid #e2e8f0; + border-radius: var(--ifm-border-radius); +} + +/* Remove styling from nested tabs containers */ +.tabs-container .tabs-container > div { + padding: 0; + border: none; + border-radius: 0; +} + +[data-theme='dark'] .tabs-container > div { + border-color: #2d3748; +} + +ul.tabs { + border-bottom: 1px solid #e2e8f0 !important; + column-gap: 0.5rem !important; +} + +[data-theme='dark'] ul.tabs { + border-bottom-color: #2d3748 !important; +} + +li.tabs__item { + padding: 0.5rem !important; + font-weight: 500 !important; + font-size: 14px !important; + border-bottom: 2px solid transparent !important; +} + +li.tabs__item--active { + border-bottom: 2px solid var(--ifm-color-primary) !important; +} + +/* ========================================= + CODE BLOCKS + ========================================= */ +.prism-code { + border-radius: var(--ifm-border-radius); + font-size: 12.5px !important; + line-height: 1.6; +} + +[data-theme='dark'] .prism-code { + border: 1px solid #21262d; +} + +[class*='codeLineNumber']::before { + font-size: 12px !important; + display: flex !important; + align-items: center !important; + justify-content: center !important; +} + +[class*='codeBlockTitle'] { + font-size: 13px !important; + font-weight: 500 !important; + padding: 0.5rem 1rem !important; + border-bottom: 1px solid #334155 !important; +} + +.theme-code-block-highlighted-line { + background-color: rgba(0, 0, 0, 0.1) !important; +} + +[data-theme='dark'] .theme-code-block-highlighted-line { + background-color: rgba(255, 255, 255, 0.06) !important; +} + +.theme-code-block-highlighted-line > span { + background-color: transparent !important; +} + +/* ========================================= + SIDEBAR / MENU + ========================================= */ +.menu { + font-weight: 400; + padding: 0.5rem 0.25rem !important; + background-image: radial-gradient(rgba(0, 0, 0, 0.07) 1px, transparent 1px); + background-size: 24px 24px; +} + +[data-theme='dark'] .menu { + background-image: radial-gradient(rgba(255, 255, 255, 0.04) 1px, transparent 1px); + background-size: 24px 24px; +} + +.menu__link { + font-size: 14px; + padding: 0.22rem 0.75rem !important; + border-radius: 4px; +} + +.menu__link--active { + font-weight: 600 !important; + background-color: rgba(46, 133, 85, 0.08) !important; +} + +[data-theme='dark'] .menu__link--active { + background-color: rgba(37, 194, 160, 0.1) !important; +} + +/* ─── Sidebar collapse arrows — uniform size & alignment ────── */ + +/* 1. Categories WITHOUT a link prop: + The button itself holds the text + ::after arrow. + Make it flex so the arrow never wraps to a new line. */ +.menu__link--sublist-caret { + display: flex !important; + align-items: center !important; + justify-content: space-between !important; + gap: 0.5rem; + padding-right: 0.625rem !important; +} + +.menu__link--sublist-caret::after { + content: '' !important; + display: block !important; + flex-shrink: 0 !important; + width: 1.25rem !important; + height: 1.25rem !important; + min-width: 1.25rem !important; + background: var(--ifm-menu-link-sublist-icon) center / 1.25rem 1.25rem no-repeat !important; + margin: 0 !important; +} + +/* 2. Categories WITH a link prop: + A separate + -
- )} Chat diff --git a/ui/litellm-dashboard/src/app/(dashboard)/teams/components/TeamsHeaderTabs.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/teams/components/TeamsHeaderTabs.test.tsx new file mode 100644 index 00000000000..50a7f10f047 --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/teams/components/TeamsHeaderTabs.test.tsx @@ -0,0 +1,54 @@ +import { render, screen } from "@testing-library/react"; +import React from "react"; +import { describe, expect, it, vi } from "vitest"; +import TeamsHeaderTabs from "./TeamsHeaderTabs"; + +vi.mock("@tremor/react", () => ({ + TabGroup: ({ children, ...props }: any) =>
{children}
, + TabList: ({ children, ...props }: any) =>
{children}
, + Tab: ({ children, ...props }: any) => , + TabPanels: ({ children, ...props }: any) =>
{children}
, + Text: ({ children, ...props }: any) => {children}, + Icon: ({ onClick, ...props }: any) => +
+ ) : null, +})); + +const defaultProps = { + dateValue: { from: new Date("2025-01-01"), to: new Date("2025-01-31") }, + entityType: "team" as const, + spendData: { + results: [], + metadata: { + total_spend: 0, + total_api_requests: 0, + total_successful_requests: 0, + total_failed_requests: 0, + total_tokens: 0, + }, + } satisfies EntitySpendData, +}; + +describe("UsageExportHeader", () => { + it("should render", () => { + renderWithProviders(); + expect(screen.getByRole("button", { name: /export data/i })).toBeInTheDocument(); + }); + + it("should open the export modal when the export button is clicked", async () => { + const user = userEvent.setup(); + renderWithProviders(); + await user.click(screen.getByRole("button", { name: /export data/i })); + expect(screen.getByTestId("export-modal")).toBeInTheDocument(); + }); + + it("should close the export modal when onClose is called", async () => { + const user = userEvent.setup(); + renderWithProviders(); + await user.click(screen.getByRole("button", { name: /export data/i })); + await user.click(screen.getByRole("button", { name: /close/i })); + expect(screen.queryByTestId("export-modal")).not.toBeInTheDocument(); + }); + + it("should not show filter dropdown when showFilters is false", () => { + renderWithProviders(); + expect(screen.queryByText(/filter/i)).not.toBeInTheDocument(); + }); + + it("should show filter dropdown when showFilters is true and options provided", () => { + renderWithProviders( + , + ); + expect(screen.getByText("Team")).toBeInTheDocument(); + }); +}); diff --git a/ui/litellm-dashboard/src/components/GuardrailsMonitor/GuardrailConfig.test.tsx b/ui/litellm-dashboard/src/components/GuardrailsMonitor/GuardrailConfig.test.tsx new file mode 100644 index 00000000000..54c7ebabe77 --- /dev/null +++ b/ui/litellm-dashboard/src/components/GuardrailsMonitor/GuardrailConfig.test.tsx @@ -0,0 +1,98 @@ +import { render, screen, act } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import { vi } from "vitest"; +import { GuardrailConfig } from "./GuardrailConfig"; + +describe("GuardrailConfig", () => { + const defaultProps = { + guardrailName: "Content Safety", + guardrailType: "Content Safety", + provider: "bedrock", + }; + + afterEach(() => { + vi.useRealTimers(); + }); + + it("should render", () => { + render(); + expect(screen.getByText("Parameters")).toBeInTheDocument(); + }); + + it("should display the guardrail name in the parameters description", () => { + render(); + expect(screen.getByText(/Configure Content Safety behavior/)).toBeInTheDocument(); + }); + + // Note: Version history entries are hardcoded placeholders in the component. + // These assertions will need updating when wired to real API data. + it("should show version history when 'View history' is clicked", async () => { + const user = userEvent.setup(); + render(); + await user.click(screen.getByRole("button", { name: /view history/i })); + expect(screen.getByText("Initial configuration")).toBeInTheDocument(); + expect(screen.getByText("Added custom categories list")).toBeInTheDocument(); + }); + + it("should toggle version history text between View/Hide", async () => { + const user = userEvent.setup(); + render(); + const button = screen.getByRole("button", { name: /view history/i }); + await user.click(button); + expect(screen.getByRole("button", { name: /hide history/i })).toBeInTheDocument(); + }); + + it("should show custom code textarea when custom code override is toggled on", async () => { + const user = userEvent.setup(); + render(); + // Walk up from "Custom Code Override" heading to find the enclosing section, + // then locate the switch within it + const heading = screen.getByText("Custom Code Override"); + let container = heading.parentElement; + let customCodeSwitch: Element | null = null; + while (container && !customCodeSwitch) { + customCodeSwitch = container.querySelector('[role="switch"]'); + container = container.parentElement; + } + if (!customCodeSwitch) { + throw new Error("Could not find the Custom Code Override switch via DOM traversal"); + } + await user.click(customCodeSwitch); + expect(screen.getByPlaceholderText(/async def evaluate/)).toBeInTheDocument(); + }); + + it("should hide custom code textarea when custom code override is off", () => { + render(); + // There's an input for categories, but no textarea + expect(screen.queryByPlaceholderText(/async def evaluate/)).not.toBeInTheDocument(); + }); + + it("should show the re-run button in idle state", () => { + render(); + expect(screen.getByRole("button", { name: /re-run on failing logs/i })).toBeInTheDocument(); + }); + + it("should show loading state when re-run is clicked", async () => { + vi.useFakeTimers({ shouldAdvanceTime: true }); + const user = userEvent.setup({ advanceTimers: vi.advanceTimersByTime }); + render(); + await user.click(screen.getByRole("button", { name: /re-run on failing logs/i })); + expect(screen.getByText(/Running on 10 samples/)).toBeInTheDocument(); + }); + + it("should show success message after re-run completes", async () => { + vi.useFakeTimers({ shouldAdvanceTime: true }); + const user = userEvent.setup({ advanceTimers: vi.advanceTimersByTime }); + render(); + await user.click(screen.getByRole("button", { name: /re-run on failing logs/i })); + await act(async () => { vi.advanceTimersByTime(2500); }); + expect(screen.getByText(/7\/10 would now pass/)).toBeInTheDocument(); + }); + + it("should display the Revert and Save buttons", () => { + render(); + expect(screen.getByRole("button", { name: /revert/i })).toBeInTheDocument(); + // The component's hardcoded default version is "v3", so Save shows "v4" + expect(screen.getByRole("button", { name: /save as v\d+/i })).toBeInTheDocument(); + }); +}); diff --git a/ui/litellm-dashboard/src/components/GuardrailsMonitor/MetricCard.test.tsx b/ui/litellm-dashboard/src/components/GuardrailsMonitor/MetricCard.test.tsx new file mode 100644 index 00000000000..d30df1a0062 --- /dev/null +++ b/ui/litellm-dashboard/src/components/GuardrailsMonitor/MetricCard.test.tsx @@ -0,0 +1,49 @@ +import { renderWithProviders, screen } from "../../../tests/test-utils"; +import React from "react"; +import { MetricCard } from "./MetricCard"; + +describe("MetricCard", () => { + it("should render", () => { + renderWithProviders(); + expect(screen.getByText("Total Requests")).toBeInTheDocument(); + }); + + it("should display the label and value", () => { + renderWithProviders(); + expect(screen.getByText("Success Rate")).toBeInTheDocument(); + expect(screen.getByText("98.5%")).toBeInTheDocument(); + }); + + it("should display numeric values", () => { + renderWithProviders(); + expect(screen.getByText("42")).toBeInTheDocument(); + }); + + it("should render icon when provided", () => { + renderWithProviders( + icon} + /> + ); + expect(screen.getByTestId("test-icon")).toBeInTheDocument(); + }); + + it("should not render icon container when no icon provided", () => { + renderWithProviders(); + expect(screen.queryByTestId("test-icon")).not.toBeInTheDocument(); + }); + + it("should render subtitle when provided", () => { + renderWithProviders( + + ); + expect(screen.getByText("Last 24 hours")).toBeInTheDocument(); + }); + + it("should not render subtitle when not provided", () => { + renderWithProviders(); + expect(screen.queryByText("Last 24 hours")).not.toBeInTheDocument(); + }); +}); diff --git a/ui/litellm-dashboard/src/components/HelpLink.test.tsx b/ui/litellm-dashboard/src/components/HelpLink.test.tsx index 72033bd93a1..a35a09b9c3f 100644 --- a/ui/litellm-dashboard/src/components/HelpLink.test.tsx +++ b/ui/litellm-dashboard/src/components/HelpLink.test.tsx @@ -23,6 +23,11 @@ describe("HelpLink", () => { expect(screen.getByText("Custom docs link")).toBeInTheDocument(); }); + it("should have the correct href", () => { + renderWithProviders(); + expect(screen.getByRole("link")).toHaveAttribute("href", "https://docs.example.com/test"); + }); + it("should include a screen-reader-only label for accessibility", () => { renderWithProviders(); @@ -46,7 +51,21 @@ describe("HelpIcon", () => { expect(screen.getByText("Tooltip help text")).toBeInTheDocument(); }); + it("should hide tooltip content when not hovered", () => { + renderWithProviders(); + expect(screen.queryByText("Hidden tooltip")).not.toBeInTheDocument(); + }); + it("should show learn more link when learnMoreHref is provided", async () => { + const user = userEvent.setup(); + renderWithProviders( + + ); + await user.hover(screen.getByRole("button", { name: /help information/i })); + expect(screen.getByText("Learn more")).toBeInTheDocument(); + }); + + it("should use custom learn more text when provided", async () => { const user = userEvent.setup(); renderWithProviders( { expect(screen.getByRole("button", { name: /docs/i })).toBeInTheDocument(); }); + it("should hide menu items initially", () => { + renderWithProviders(); + expect(screen.queryByText("Custom pricing")).not.toBeInTheDocument(); + }); + it("should show menu items when button is clicked", async () => { const user = userEvent.setup(); renderWithProviders(); @@ -114,4 +138,18 @@ describe("DocsMenu", () => { await user.click(button); expect(button).toHaveAttribute("aria-expanded", "true"); }); + + it("should close menu when clicking outside", async () => { + const user = userEvent.setup(); + renderWithProviders( +
+ + +
, + ); + await user.click(screen.getByRole("button", { name: /docs/i })); + expect(screen.getByText("Custom pricing")).toBeInTheDocument(); + await user.click(screen.getByRole("button", { name: /outside/i })); + expect(screen.queryByText("Custom pricing")).not.toBeInTheDocument(); + }); }); diff --git a/ui/litellm-dashboard/src/components/Navbar/WorkerDropdown/WorkerDropdown.test.tsx b/ui/litellm-dashboard/src/components/Navbar/WorkerDropdown/WorkerDropdown.test.tsx new file mode 100644 index 00000000000..17b8b393607 --- /dev/null +++ b/ui/litellm-dashboard/src/components/Navbar/WorkerDropdown/WorkerDropdown.test.tsx @@ -0,0 +1,134 @@ +import { render, screen } from "@testing-library/react"; +import { describe, expect, it, vi, beforeEach } from "vitest"; + +// Mock the useWorker hook +const mockUseWorker = vi.fn(); +vi.mock("@/hooks/useWorker", () => ({ + useWorker: () => mockUseWorker(), +})); + +// Mock antd Select +vi.mock("antd", () => ({ + Select: ({ value, options, onChange, style, disabled, ...props }: any) => ( + + ), +})); + +// Mock icon +vi.mock("@ant-design/icons", () => ({ + CloudServerOutlined: () => , +})); + +import WorkerDropdown from "./WorkerDropdown"; + +describe("WorkerDropdown", () => { + const mockOnWorkerSwitch = vi.fn(); + const workers = [ + { worker_id: "w1", name: "Worker 1" }, + { worker_id: "w2", name: "Worker 2" }, + { worker_id: "w3", name: "Worker 3" }, + ]; + + beforeEach(() => { + vi.clearAllMocks(); + }); + + it("renders null when isControlPlane is false", () => { + mockUseWorker.mockReturnValue({ + isControlPlane: false, + selectedWorker: workers[0], + workers, + }); + + const { container } = render(); + expect(container).toBeEmptyDOMElement(); + }); + + it("renders null when selectedWorker is null", () => { + mockUseWorker.mockReturnValue({ + isControlPlane: true, + selectedWorker: null, + workers, + }); + + const { container } = render(); + expect(container).toBeEmptyDOMElement(); + }); + + it("renders the select when isControlPlane and selectedWorker exist", () => { + mockUseWorker.mockReturnValue({ + isControlPlane: true, + selectedWorker: workers[0], + workers, + }); + + render(); + expect(screen.getByTestId("worker-select")).toBeInTheDocument(); + }); + + it("renders all worker options", () => { + mockUseWorker.mockReturnValue({ + isControlPlane: true, + selectedWorker: workers[0], + workers, + }); + + render(); + expect(screen.getByText("Worker 1")).toBeInTheDocument(); + expect(screen.getByText("Worker 2")).toBeInTheDocument(); + expect(screen.getByText("Worker 3")).toBeInTheDocument(); + }); + + it("sets current worker as selected value", () => { + mockUseWorker.mockReturnValue({ + isControlPlane: true, + selectedWorker: workers[1], + workers, + }); + + render(); + const select = screen.getByTestId("worker-select") as HTMLSelectElement; + expect(select.value).toBe("w2"); + }); + + it("disables the currently selected worker in options", () => { + mockUseWorker.mockReturnValue({ + isControlPlane: true, + selectedWorker: workers[0], + workers, + }); + + render(); + const options = screen.getAllByRole("option"); + const selectedOption = options.find((opt) => (opt as HTMLOptionElement).value === "w1"); + expect(selectedOption).toBeDisabled(); + }); + + it("calls onWorkerSwitch when selection changes", async () => { + mockUseWorker.mockReturnValue({ + isControlPlane: true, + selectedWorker: workers[0], + workers, + }); + + render(); + const select = screen.getByTestId("worker-select"); + + const { default: userEvent } = await import("@testing-library/user-event"); + const user = userEvent.setup(); + await user.selectOptions(select, "w2"); + + expect(mockOnWorkerSwitch).toHaveBeenCalledWith("w2"); + }); +}); diff --git a/ui/litellm-dashboard/src/components/Navbar/WorkerDropdown/WorkerDropdown.tsx b/ui/litellm-dashboard/src/components/Navbar/WorkerDropdown/WorkerDropdown.tsx new file mode 100644 index 00000000000..55309a11c18 --- /dev/null +++ b/ui/litellm-dashboard/src/components/Navbar/WorkerDropdown/WorkerDropdown.tsx @@ -0,0 +1,38 @@ +"use client"; + +import React from "react"; +import { Select } from "antd"; +import { CloudServerOutlined } from "@ant-design/icons"; +import { useWorker } from "@/hooks/useWorker"; + +interface WorkerDropdownProps { + onWorkerSwitch: (workerId: string) => void; +} + +const WorkerDropdown: React.FC = ({ onWorkerSwitch }) => { + const { isControlPlane, selectedWorker, workers } = useWorker(); + + if (!isControlPlane || !selectedWorker) return null; + + return ( + } + suffix={isSearching ? : null} + placeholder="Search teams by name..." + onChange={(e) => handleSearchChange(e.target.value)} + allowClear + style={{ maxWidth: 400 }} + /> + handleFilterChange("organization_id", value || "")} + loading={isLoading} + /> + + { + setCurrentPage(page); + setPageSize(size); + fetchTeamsV2({ page, size }); + }} + size="small" + showTotal={(total) => `${total} teams`} + showSizeChanger + pageSizeOptions={["10", "20", "50"]} + /> + + + {renderTeamsContent()} + + + + + ), + }, + { + key: "available-teams", + label: "Available Teams", + children: , + }, + ...(isProxyAdminRole(userRole || "") + ? [ + { + key: "default-settings", + label: "Default Team Settings", + children: , + }, + ] + : []), + ]; + return ( -
- - - {canCreateOrManageTeams(userRole, userID, organizations) && ( - - )} - {selectedTeamId ? ( - { - setTeams((teams) => { - if (teams == null) { - return teams; - } - const updated = teams.map((team) => { - if (data.team_id === team.team_id) { - return updateExistingKeys(team, data); - } - return team; - }); - // Minimal fix: refresh the full team list after an update - if (accessToken) { - fetchTeams(accessToken, userID, userRole, currentOrg, setTeams); - } - return updated; - }); - }} - onClose={() => { - setSelectedTeamId(null); - setEditTeam(false); - }} - accessToken={accessToken} - is_team_admin={is_team_admin(teams?.find((team) => team.team_id === selectedTeamId))} - is_proxy_admin={userRole == "Admin"} - userModels={userModels} - editTeam={editTeam} - premiumUser={premiumUser} - /> - ) : ( - - -
- Your Teams - Available Teams - {isProxyAdminRole(userRole || "") && Default Team Settings} -
-
- {lastRefreshed && Last Refreshed: {lastRefreshed}} - -
- - - - - Click on “Team ID” to view team details and manage team members. - - - - -
-
- {/* Search and Filter Controls */} -
- {/* Team Alias Search */} - handleFilterChange("team_alias", value)} - icon={Search} - /> + + {selectedTeamId ? ( + { + setTeams((teams) => { + if (teams == null) { + return teams; + } + return teams.map((team) => { + if (data.team_id === team.team_id) { + return updateExistingKeys(team, data); + } + return team; + }); + }); + fetchTeamsV2(); + }} + onClose={() => { + setSelectedTeamId(null); + setEditTeam(false); + }} + accessToken={accessToken} + is_team_admin={is_team_admin(teams?.find((team) => team.team_id === selectedTeamId))} + is_proxy_admin={userRole == "Admin"} + userModels={userModels} + editTeam={editTeam} + premiumUser={premiumUser} + /> + ) : ( + <> + + + + <TeamOutlined style={{ marginRight: 8 }} /> + Teams + + + Manage teams, members, and their access to models and budgets + + + {canCreateOrManageTeams(userRole, userID, organizations) && ( + + )} + - {/* Filter Button */} - setShowFilters(!showFilters)} - active={showFilters} - hasActiveFilters={!!(filters.team_id || filters.team_alias || filters.organization_id)} - /> + + + )} - {/* Reset Filters Button */} - -
- - {/* Additional Filters */} - {showFilters && ( -
- {/* Team ID Search */} - handleFilterChange("team_id", value)} - icon={User} - /> - - {/* Organization Dropdown */} -
- -
-
- )} -
-
- - - - Team Name - Team ID - Created - Spend (USD) - Budget (USD) - Models - Organization - Info - Actions - - - - - {teams && teams.length > 0 ? ( - teams - .filter((team) => { - if (!currentOrg) return true; - return team.organization_id === currentOrg.organization_id; - }) - .sort((a, b) => new Date(b.created_at).getTime() - new Date(a.created_at).getTime()) - .map((team: any) => ( - - - {team["team_alias"]} - - -
- - - -
- - - {team.created_at ? new Date(team.created_at).toLocaleDateString() : "N/A"} - - - {formatNumberWithCommas(team["spend"], 4)} - - - {team["max_budget"] !== null && team["max_budget"] !== undefined - ? team["max_budget"] - : "No limit"} - - 3 ? "px-0" : ""} - > -
- {Array.isArray(team.models) ? ( -
- {team.models.length === 0 ? ( - - All Proxy Models - - ) : ( - <> -
- {team.models.length > 3 && ( -
- { - setExpandedAccordions((prev) => ({ - ...prev, - [team.team_id]: !prev[team.team_id], - })); - }} - /> -
- )} -
- {team.models.slice(0, 3).map((model: string, index: number) => - model === "all-proxy-models" ? ( - - All Proxy Models - - ) : ( - - - {model.length > 30 - ? `${getModelDisplayName(model).slice(0, 30)}...` - : getModelDisplayName(model)} - - - ), - )} - {team.models.length > 3 && !expandedAccordions[team.team_id] && ( - - - +{team.models.length - 3}{" "} - {team.models.length - 3 === 1 ? "more model" : "more models"} - - - )} - {expandedAccordions[team.team_id] && ( -
- {team.models.slice(3).map((model: string, index: number) => - model === "all-proxy-models" ? ( - - All Proxy Models - - ) : ( - - - {model.length > 30 - ? `${getModelDisplayName(model).slice(0, 30)}...` - : getModelDisplayName(model)} - - - ), - )} -
- )} -
-
- - )} -
- ) : null} -
- - - - {getOrganizationAlias(team.organization_id, organizationsData || organizations)} - - - - {perTeamInfo && - team.team_id && - perTeamInfo[team.team_id] && - perTeamInfo[team.team_id].keys && - perTeamInfo[team.team_id].keys.length}{" "} - Keys - - - {perTeamInfo && - team.team_id && - perTeamInfo[team.team_id] && - perTeamInfo[team.team_id].team_info && - perTeamInfo[team.team_id].team_info.members_with_roles && - perTeamInfo[team.team_id].team_info.members_with_roles.length}{" "} - Members - - - - {userRole == "Admin" ? ( - <> - { - setSelectedTeamId(team.team_id); - setEditTeam(true); - }} - dataTestId="edit-team-button" - tooltipText="Edit team" - /> - handleDelete(team)} - dataTestId="delete-team-button" - tooltipText="Delete team" - /> - - ) : null} - - - )) - ) : ( - - -
- No teams found - Adjust your filters or create a new team -
- - - )} - -
- - - - - - - - - {isProxyAdminRole(userRole || "") && ( - - - - )} - - - )} - {canCreateOrManageTeams(userRole, userID, organizations) && ( + {canCreateOrManageTeams(userRole, userID, organizations) && ( = ({ : "" } > - = ({ optionFilterProp="children" > {adminOrgs?.map((org) => ( - + {org.organization_alias}{" "} ({org.organization_id}) - + ))} - + {/* Show message when org admin needs to select organization */} {isOrgAdmin && !isSingleOrg && adminOrgs.length > 1 && (
- + Please select an organization to create a team for. You can only create teams within organizations where you are an admin. @@ -1190,11 +1209,11 @@ const Teams: React.FC = ({ - - daily - weekly - monthly - + @@ -1313,7 +1332,7 @@ const Teams: React.FC = ({ className="mt-8" help="Select existing guardrails or enter new ones" > - = ({ className="mt-8" help="Select existing policies or enter new ones" > - = ({
- + Create custom aliases for models that can be used by team members in API calls. This allows you to create shortcuts for specific models. @@ -1548,14 +1567,12 @@ const Teams: React.FC = ({
- Create Team +
)} - - -
+ ); }; diff --git a/ui/litellm-dashboard/src/components/Projects/ProjectModals/CreateProjectModal.tsx b/ui/litellm-dashboard/src/components/Projects/ProjectModals/CreateProjectModal.tsx index e490f89303f..bbf56e4930a 100644 --- a/ui/litellm-dashboard/src/components/Projects/ProjectModals/CreateProjectModal.tsx +++ b/ui/litellm-dashboard/src/components/Projects/ProjectModals/CreateProjectModal.tsx @@ -1,5 +1,6 @@ -import { Modal, Form, Button, Typography, message } from "antd"; +import { Modal, Form, Button, Typography } from "antd"; import { FolderAddOutlined } from "@ant-design/icons"; +import MessageManager from "@/components/molecules/message_manager"; import { useCreateProject, ProjectCreateParams, @@ -32,12 +33,12 @@ export function CreateProjectModal({ createMutation.mutate(params, { onSuccess: () => { - message.success("Project created successfully"); + MessageManager.success("Project created successfully"); form.resetFields(); onClose(); }, onError: (error) => { - message.error(error.message || "Failed to create project"); + MessageManager.error(error.message || "Failed to create project"); }, }); } catch (error) { diff --git a/ui/litellm-dashboard/src/components/Projects/ProjectModals/EditProjectModal.tsx b/ui/litellm-dashboard/src/components/Projects/ProjectModals/EditProjectModal.tsx index 75f56b1373f..dc3b43ef73c 100644 --- a/ui/litellm-dashboard/src/components/Projects/ProjectModals/EditProjectModal.tsx +++ b/ui/litellm-dashboard/src/components/Projects/ProjectModals/EditProjectModal.tsx @@ -1,6 +1,7 @@ import { useEffect } from "react"; -import { Modal, Form, Button, Typography, message } from "antd"; +import { Modal, Form, Button, Typography } from "antd"; import { SaveOutlined } from "@ant-design/icons"; +import MessageManager from "@/components/molecules/message_manager"; import { ProjectResponse } from "@/app/(dashboard)/hooks/projects/useProjects"; import { useUpdateProject, @@ -80,12 +81,12 @@ export function EditProjectModal({ { projectId: project.project_id, params }, { onSuccess: () => { - message.success("Project updated successfully"); + MessageManager.success("Project updated successfully"); onSuccess?.(); onClose(); }, onError: (error) => { - message.error(error.message || "Failed to update project"); + MessageManager.error(error.message || "Failed to update project"); }, }, ); diff --git a/ui/litellm-dashboard/src/components/SearchTools/SearchToolTester.tsx b/ui/litellm-dashboard/src/components/SearchTools/SearchToolTester.tsx index 28b34b06089..d1cb5077b18 100644 --- a/ui/litellm-dashboard/src/components/SearchTools/SearchToolTester.tsx +++ b/ui/litellm-dashboard/src/components/SearchTools/SearchToolTester.tsx @@ -1,5 +1,6 @@ import React, { useState } from "react"; -import { Button, Input, Typography, Spin, message } from "antd"; +import { Button, Input, Typography, Spin } from "antd"; +import MessageManager from "@/components/molecules/message_manager"; import { SearchOutlined, LoadingOutlined } from "@ant-design/icons"; import { searchToolQueryCall } from "../networking"; import NotificationsManager from "../molecules/notifications_manager"; @@ -39,7 +40,7 @@ export const SearchToolTester: React.FC = ({ searchToolNa const handleSearch = async () => { if (!query.trim()) { - message.warning("Please enter a search query"); + MessageManager.warning("Please enter a search query"); return; } diff --git a/ui/litellm-dashboard/src/components/Settings/RouterSettings/Fallbacks/AddFallbacks.tsx b/ui/litellm-dashboard/src/components/Settings/RouterSettings/Fallbacks/AddFallbacks.tsx index 34f059516a6..e55089d27dc 100644 --- a/ui/litellm-dashboard/src/components/Settings/RouterSettings/Fallbacks/AddFallbacks.tsx +++ b/ui/litellm-dashboard/src/components/Settings/RouterSettings/Fallbacks/AddFallbacks.tsx @@ -5,8 +5,9 @@ */ import { Button as TremorButton } from "@tremor/react"; -import { Button, message } from "antd"; +import { Button } from "antd"; import React, { useEffect, useState } from "react"; +import MessageManager from "@/components/molecules/message_manager"; import NotificationManager from "../../../molecules/notifications_manager"; import { fetchAvailableModels, ModelGroup } from "../../../playground/llm_calls/fetch_models"; import { AddFallbacksModal } from "./AddFallbacksModal"; @@ -90,7 +91,7 @@ export default function AddFallbacks({ (g) => !g.primaryModel || g.fallbackModels.length === 0, ); if (invalidGroups.length > 0) { - message.error( + MessageManager.error( `Please complete configuration for all groups. ${invalidGroups.length} group(s) incomplete.`, ); return; diff --git a/ui/litellm-dashboard/src/components/Settings/RouterSettings/Fallbacks/FallbackSelectionForm.tsx b/ui/litellm-dashboard/src/components/Settings/RouterSettings/Fallbacks/FallbackSelectionForm.tsx index 08b031c6835..0482161bc6f 100644 --- a/ui/litellm-dashboard/src/components/Settings/RouterSettings/Fallbacks/FallbackSelectionForm.tsx +++ b/ui/litellm-dashboard/src/components/Settings/RouterSettings/Fallbacks/FallbackSelectionForm.tsx @@ -5,9 +5,10 @@ */ import { Button } from "@tremor/react"; -import { message, Tabs } from "antd"; +import { Tabs } from "antd"; import { Plus } from "lucide-react"; import React, { useEffect, useState } from "react"; +import MessageManager from "@/components/molecules/message_manager"; import { FallbackGroup, FallbackGroupConfig } from "./FallbackGroupConfig"; interface FallbackSelectionFormProps { @@ -60,7 +61,7 @@ export function FallbackSelectionForm({ const handleRemoveGroup = (targetId: string) => { if (groups.length === 1) { - message.warning("At least one group is required"); + MessageManager.warning("At least one group is required"); return; } const newGroups = groups.filter((g) => g.id !== targetId); diff --git a/ui/litellm-dashboard/src/components/TeamSSOSettings.tsx b/ui/litellm-dashboard/src/components/TeamSSOSettings.tsx index a9c07cdbcf8..98745d388a2 100644 --- a/ui/litellm-dashboard/src/components/TeamSSOSettings.tsx +++ b/ui/litellm-dashboard/src/components/TeamSSOSettings.tsx @@ -26,6 +26,10 @@ const PERMISSION_OPTIONS = [ "/key/unblock", "/key/bulk_update", "/key/{key_id}/reset_spend", + "/key/info", + "/key/list", + "/key/aliases", + "/team/daily/activity", ]; interface SettingRowProps { diff --git a/ui/litellm-dashboard/src/components/ToolDetail.tsx b/ui/litellm-dashboard/src/components/ToolDetail.tsx index ed0f866acb8..b2a7d9bfbe4 100644 --- a/ui/litellm-dashboard/src/components/ToolDetail.tsx +++ b/ui/litellm-dashboard/src/components/ToolDetail.tsx @@ -387,7 +387,6 @@ export function ToolDetail({ toolName, onBack, accessToken }: ToolDetailProps) { {blockScope === "team" ? ( setBlockTeamId(id || null)} /> diff --git a/ui/litellm-dashboard/src/components/ToolPolicies/PolicySelect.test.tsx b/ui/litellm-dashboard/src/components/ToolPolicies/PolicySelect.test.tsx new file mode 100644 index 00000000000..cde59fe9c6d --- /dev/null +++ b/ui/litellm-dashboard/src/components/ToolPolicies/PolicySelect.test.tsx @@ -0,0 +1,84 @@ +import { renderWithProviders, screen } from "../../../tests/test-utils"; +import { vi } from "vitest"; +import { + PolicySelect, + policyStyle, + INPUT_POLICY_OPTIONS, + OUTPUT_POLICY_OPTIONS, +} from "./PolicySelect"; + +describe("policyStyle", () => { + it("should return the matching option for a known policy", () => { + expect(policyStyle("trusted")).toEqual(INPUT_POLICY_OPTIONS[1]); + }); + + it("should return the matching option for blocked", () => { + expect(policyStyle("blocked")).toEqual(INPUT_POLICY_OPTIONS[2]); + }); + + it("should return the first option as fallback for unknown policy", () => { + expect(policyStyle("unknown")).toEqual(INPUT_POLICY_OPTIONS[0]); + }); +}); + +describe("PolicySelect", () => { + it("should render", () => { + renderWithProviders( + + ); + expect(screen.getByText("untrusted")).toBeInTheDocument(); + }); + + it("should show the current policy value", () => { + renderWithProviders( + + ); + expect(screen.getByText("trusted")).toBeInTheDocument(); + }); + + it("should be disabled when saving is true", () => { + renderWithProviders( + + ); + expect(screen.getByRole("combobox")).toHaveAttribute("aria-expanded", "false"); + expect(screen.getByRole("combobox").closest(".ant-select")).toHaveClass("ant-select-disabled"); + }); + + it("should not be disabled when saving is false", () => { + renderWithProviders( + + ); + expect(screen.getByRole("combobox").closest(".ant-select")).not.toHaveClass("ant-select-disabled"); + }); +}); + +describe("Policy option constants", () => { + it("should have 3 input policy options", () => { + expect(INPUT_POLICY_OPTIONS).toHaveLength(3); + }); + + it("should have 2 output policy options (no blocked)", () => { + expect(OUTPUT_POLICY_OPTIONS).toHaveLength(2); + expect(OUTPUT_POLICY_OPTIONS.map((o) => o.value)).toEqual(["untrusted", "trusted"]); + }); +}); diff --git a/ui/litellm-dashboard/src/components/add_model/AddModelForm.test.tsx b/ui/litellm-dashboard/src/components/add_model/AddModelForm.test.tsx index 6f63fcfa549..de5e1387396 100644 --- a/ui/litellm-dashboard/src/components/add_model/AddModelForm.test.tsx +++ b/ui/litellm-dashboard/src/components/add_model/AddModelForm.test.tsx @@ -65,6 +65,21 @@ vi.mock("@/app/(dashboard)/hooks/useAuthorized", () => ({ default: vi.fn(), })); +vi.mock("@/app/(dashboard)/hooks/teams/useTeams", () => ({ + useInfiniteTeams: () => ({ + data: { + pages: [{ + teams: [{ team_id: "team-1", team_alias: "Test Team", organization_id: "org-1" }], + total: 1, page: 1, page_size: 20, total_pages: 1, + }], + }, + fetchNextPage: vi.fn(), + hasNextPage: false, + isFetchingNextPage: false, + isLoading: false, + }), +})); + vi.mock("@/app/(dashboard)/hooks/guardrails/useGuardrails", () => ({ useGuardrails: vi.fn().mockReturnValue({ data: [{ guardrail_name: "test-guardrail" }], diff --git a/ui/litellm-dashboard/src/components/add_model/AddModelForm.tsx b/ui/litellm-dashboard/src/components/add_model/AddModelForm.tsx index 2b3f23a35ae..65e239d58ea 100644 --- a/ui/litellm-dashboard/src/components/add_model/AddModelForm.tsx +++ b/ui/litellm-dashboard/src/components/add_model/AddModelForm.tsx @@ -131,7 +131,6 @@ const AddModelForm: React.FC = ({ tooltip="Select the team for which you want to add this model" > { setTeamAdminSelectedTeam(value); }} @@ -325,7 +324,7 @@ const AddModelForm: React.FC = ({ }, ]} > - + )} {isAdmin && ( diff --git a/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.test.tsx b/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.test.tsx new file mode 100644 index 00000000000..59d8721b143 --- /dev/null +++ b/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.test.tsx @@ -0,0 +1,82 @@ +import { renderWithProviders, screen } from "../../../tests/test-utils"; +import { vi } from "vitest"; +import ComplexityRouterConfig from "./ComplexityRouterConfig"; + +const mockModelInfo = [ + { model_group: "gpt-4" }, + { model_group: "gpt-3.5-turbo" }, + { model_group: "claude-3-opus" }, +] as any[]; + +const defaultTiers = { + SIMPLE: "gpt-3.5-turbo", + MEDIUM: "gpt-3.5-turbo", + COMPLEX: "gpt-4", + REASONING: "claude-3-opus", +}; + +describe("ComplexityRouterConfig", () => { + it("should render", () => { + renderWithProviders( + + ); + expect(screen.getByText("Complexity Tier Configuration")).toBeInTheDocument(); + }); + + it("should display all four tier labels", () => { + renderWithProviders( + + ); + expect(screen.getByText("Simple Tier")).toBeInTheDocument(); + expect(screen.getByText("Medium Tier")).toBeInTheDocument(); + expect(screen.getByText("Complex Tier")).toBeInTheDocument(); + expect(screen.getByText("Reasoning Tier")).toBeInTheDocument(); + }); + + it("should show example queries for each tier", () => { + renderWithProviders( + + ); + expect(screen.getByText(/Hello!/)).toBeInTheDocument(); + expect(screen.getByText(/Explain how REST APIs work/)).toBeInTheDocument(); + expect(screen.getByText(/Design a microservices architecture/)).toBeInTheDocument(); + expect(screen.getByText(/Think step by step/)).toBeInTheDocument(); + }); + + it("should display the how classification works section", () => { + renderWithProviders( + + ); + expect(screen.getByText("How Classification Works")).toBeInTheDocument(); + }); + + it("should show score thresholds in the classification section", () => { + renderWithProviders( + + ); + expect(screen.getByText(/Score < 0.15/)).toBeInTheDocument(); + expect(screen.getByText(/Score 0.15 - 0.35/)).toBeInTheDocument(); + expect(screen.getByText(/Score 0.35 - 0.60/)).toBeInTheDocument(); + expect(screen.getByText(/Score > 0.60/)).toBeInTheDocument(); + }); +}); diff --git a/ui/litellm-dashboard/src/components/agent_management/AgentSelector.test.tsx b/ui/litellm-dashboard/src/components/agent_management/AgentSelector.test.tsx new file mode 100644 index 00000000000..26f903cd310 --- /dev/null +++ b/ui/litellm-dashboard/src/components/agent_management/AgentSelector.test.tsx @@ -0,0 +1,144 @@ +import { render, screen, waitFor } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import { describe, expect, it, vi, beforeEach } from "vitest"; + +// Mock networking module +const mockGetAgentsList = vi.fn(); +vi.mock("../networking", () => ({ + getAgentsList: (...args: any[]) => mockGetAgentsList(...args), +})); + +// Mock antd Select +vi.mock("antd", () => { + const SelectComponent = ({ children, onChange, value, mode, placeholder, loading, disabled, ...props }: any) => ( +
+ + {loading && Loading} +
+ ); + + SelectComponent.Option = ({ children, value, ...props }: any) => ( + + ); + + return { Select: SelectComponent }; +}); + +import AgentSelector from "./AgentSelector"; + +describe("AgentSelector", () => { + const defaultProps = { + onChange: vi.fn(), + accessToken: "test-token", + }; + + beforeEach(() => { + vi.clearAllMocks(); + mockGetAgentsList.mockResolvedValue({ + agents: [ + { agent_id: "agent-1", agent_name: "Agent One" }, + { agent_id: "agent-2", agent_name: "Agent Two", agent_access_groups: ["group-a", "group-b"] }, + ], + }); + }); + + it("renders the selector", () => { + render(); + expect(screen.getByTestId("agent-select")).toBeInTheDocument(); + }); + + it("fetches agents on mount with access token", async () => { + render(); + await waitFor(() => { + expect(mockGetAgentsList).toHaveBeenCalledWith("test-token"); + }); + }); + + it("does not fetch when accessToken is empty", () => { + render(); + expect(mockGetAgentsList).not.toHaveBeenCalled(); + }); + + it("shows loading state while fetching", async () => { + // Keep the promise pending + let resolve: any; + mockGetAgentsList.mockReturnValue(new Promise((r) => { resolve = r; })); + + render(); + expect(screen.getByTestId("agent-select")).toHaveAttribute("data-loading", "true"); + + // Resolve to clean up + resolve({ agents: [] }); + await waitFor(() => { + expect(screen.getByTestId("agent-select")).toHaveAttribute("data-loading", "false"); + }); + }); + + it("renders agent options after fetch", async () => { + render(); + await waitFor(() => { + expect(screen.getByText("Agent One")).toBeInTheDocument(); + expect(screen.getByText("Agent Two")).toBeInTheDocument(); + }); + }); + + it("renders access group options with group prefix", async () => { + render(); + await waitFor(() => { + expect(screen.getByText("group-a")).toBeInTheDocument(); + expect(screen.getByText("group-b")).toBeInTheDocument(); + }); + }); + + it("respects disabled prop", () => { + render(); + expect(screen.getByTestId("agent-select")).toHaveAttribute("data-disabled", "true"); + }); + + it("handles API error gracefully", async () => { + const consoleSpy = vi.spyOn(console, "error").mockImplementation(() => {}); + mockGetAgentsList.mockRejectedValue(new Error("API error")); + + render(); + await waitFor(() => { + expect(consoleSpy).toHaveBeenCalledWith("Error fetching agents:", expect.any(Error)); + }); + + consoleSpy.mockRestore(); + }); + + it("passes value as flattened selectedValues", async () => { + render( + + ); + await waitFor(() => { + const select = screen.getByTestId("select-input"); + // The value should contain agent-1 and group:group-a + expect(select).toBeInTheDocument(); + }); + }); + + it("handles null response from API", async () => { + mockGetAgentsList.mockResolvedValue(null); + render(); + await waitFor(() => { + expect(screen.getByTestId("agent-select")).toHaveAttribute("data-loading", "false"); + }); + }); +}); diff --git a/ui/litellm-dashboard/src/components/agents/add_agent_form.tsx b/ui/litellm-dashboard/src/components/agents/add_agent_form.tsx index c5518596b81..046b28640c3 100644 --- a/ui/litellm-dashboard/src/components/agents/add_agent_form.tsx +++ b/ui/litellm-dashboard/src/components/agents/add_agent_form.tsx @@ -1,5 +1,6 @@ import React, { useState, useEffect } from "react"; -import { Modal, Form, message, Select, Input, Steps, Radio, Tag, Divider, Switch, InputNumber, Collapse } from "antd"; +import { Modal, Form, Select, Input, Steps, Radio, Tag, Divider, Switch, InputNumber, Collapse } from "antd"; +import MessageManager from "@/components/molecules/message_manager"; import { Button } from "@tremor/react"; import { CheckCircleFilled, KeyOutlined, RobotOutlined, AppstoreOutlined, InfoCircleOutlined } from "@ant-design/icons"; import CreatedKeyDisplay from "../shared/CreatedKeyDisplay"; @@ -216,7 +217,7 @@ const AddAgentForm: React.FC = ({ const handleCreateAgent = async () => { if (!accessToken) { - message.error("No access token available"); + MessageManager.error("No access token available"); return; } @@ -226,7 +227,7 @@ const AddAgentForm: React.FC = ({ const values = { ...form.getFieldsValue(true) }; const agentData = buildAgentData(values); if (!agentData) { - message.error("Failed to build agent data"); + MessageManager.error("Failed to build agent data"); setIsSubmitting(false); return; } @@ -301,7 +302,7 @@ const AddAgentForm: React.FC = ({ setCreatedKeyValue(keyResponse.key || null); } else if (keyAssignOption === "existing_key") { if (!selectedExistingKey) { - message.error("Please select an existing key to assign"); + MessageManager.error("Please select an existing key to assign"); setIsSubmitting(false); return; } @@ -318,7 +319,7 @@ const AddAgentForm: React.FC = ({ } catch (error) { console.error("Error creating agent:", error); const errorMessage = error instanceof Error ? error.message : String(error); - message.error(errorMessage ? `Failed to create agent: ${errorMessage}` : "Failed to create agent"); + MessageManager.error(errorMessage ? `Failed to create agent: ${errorMessage}` : "Failed to create agent"); } finally { setIsSubmitting(false); } @@ -722,10 +723,7 @@ const AddAgentForm: React.FC = ({ name="team_id" tooltip="Optionally assign this agent to a team. The agent and its key will belong to the selected team." > - + diff --git a/ui/litellm-dashboard/src/components/agents/agent_card_grid.test.tsx b/ui/litellm-dashboard/src/components/agents/agent_card_grid.test.tsx new file mode 100644 index 00000000000..c8a21d1b6e8 --- /dev/null +++ b/ui/litellm-dashboard/src/components/agents/agent_card_grid.test.tsx @@ -0,0 +1,90 @@ +import { renderWithProviders, screen } from "../../../tests/test-utils"; +import { vi } from "vitest"; +import AgentCardGrid from "./agent_card_grid"; +import type { Agent, AgentKeyInfo } from "./types"; + +vi.mock("./agent_card", () => ({ + default: ({ agent, onAgentClick }: any) => ( +
onAgentClick(agent.agent_id)}> + {agent.agent_name} +
+ ), +})); + +const mockAgents: Agent[] = [ + { + agent_id: "agent-1", + agent_name: "Test Agent 1", + litellm_params: { model: "gpt-4" }, + agent_card_params: { description: "First agent" }, + }, + { + agent_id: "agent-2", + agent_name: "Test Agent 2", + litellm_params: { model: "claude-3" }, + agent_card_params: { description: "Second agent" }, + }, +]; + +const mockKeyInfoMap: Record = { + "agent-1": { has_key: true, key_alias: "key-1" }, + "agent-2": { has_key: false }, +}; + +const defaultProps = { + agentsList: mockAgents, + keyInfoMap: mockKeyInfoMap, + isLoading: false, + onDeleteClick: vi.fn(), + accessToken: "test-token", + onAgentUpdated: vi.fn(), + isAdmin: true, + onAgentClick: vi.fn(), +}; + +describe("AgentCardGrid", () => { + it("should render", () => { + renderWithProviders(); + expect(screen.getByText("Test Agent 1")).toBeInTheDocument(); + }); + + it("should render all agent cards", () => { + renderWithProviders(); + expect(screen.getByText("Test Agent 1")).toBeInTheDocument(); + expect(screen.getByText("Test Agent 2")).toBeInTheDocument(); + }); + + it("should show loading skeletons when isLoading is true", () => { + renderWithProviders(); + expect(screen.queryByText("Test Agent 1")).not.toBeInTheDocument(); + }); + + it("should show admin empty state message when no agents and isAdmin", () => { + renderWithProviders( + + ); + expect( + screen.getByText("No agents found. Create one to get started.") + ).toBeInTheDocument(); + }); + + it("should show non-admin empty state message when no agents and not admin", () => { + renderWithProviders( + + ); + expect( + screen.getByText("No agents found. Contact an admin to create agents.") + ).toBeInTheDocument(); + }); + + it("should call onAgentClick when a card is clicked", async () => { + const onAgentClick = vi.fn(); + renderWithProviders( + + ); + const { default: userEvent } = await import("@testing-library/user-event"); + const user = userEvent.setup(); + await user.click(screen.getByTestId("agent-card-agent-1")); + expect(onAgentClick).toHaveBeenCalledWith("agent-1"); + }); +}); diff --git a/ui/litellm-dashboard/src/components/agents/agent_info.tsx b/ui/litellm-dashboard/src/components/agents/agent_info.tsx index b41e318a766..d543be8356a 100644 --- a/ui/litellm-dashboard/src/components/agents/agent_info.tsx +++ b/ui/litellm-dashboard/src/components/agents/agent_info.tsx @@ -1,6 +1,7 @@ import React, { useState, useEffect } from "react"; import { Card, Title, Text, Button as TremorButton, Tab, TabGroup, TabList, TabPanel, TabPanels} from "@tremor/react"; -import { Form, Input, InputNumber, Button as AntButton, message, Spin, Descriptions, Divider } from "antd"; +import { Form, Input, InputNumber, Button as AntButton, Spin, Descriptions, Divider } from "antd"; +import MessageManager from "@/components/molecules/message_manager"; import { ArrowLeftIcon } from "@heroicons/react/outline"; import { getAgentInfo, patchAgentCall, getAgentCreateMetadata, AgentCreateInfo } from "../networking"; import { Agent } from "./types"; @@ -72,7 +73,7 @@ const AgentInfoView: React.FC = ({ } } catch (error) { console.error("Error fetching agent info:", error); - message.error("Failed to load agent information"); + MessageManager.error("Failed to load agent information"); } finally { setIsLoading(false); } @@ -111,12 +112,12 @@ const AgentInfoView: React.FC = ({ } await patchAgentCall(accessToken, agentId, updateData); - message.success("Agent updated successfully"); + MessageManager.success("Agent updated successfully"); setIsEditing(false); fetchAgentInfo(); } catch (error) { console.error("Error updating agent:", error); - message.error("Failed to update agent"); + MessageManager.error("Failed to update agent"); } finally { setIsSaving(false); } diff --git a/ui/litellm-dashboard/src/components/budgets/budget_modal.tsx b/ui/litellm-dashboard/src/components/budgets/budget_modal.tsx index 490613de254..b5ad8aaff34 100644 --- a/ui/litellm-dashboard/src/components/budgets/budget_modal.tsx +++ b/ui/litellm-dashboard/src/components/budgets/budget_modal.tsx @@ -1,17 +1,17 @@ import React from "react"; import { TextInput, Accordion, AccordionHeader, AccordionBody } from "@tremor/react"; import { Button as Button2, Modal, Form, InputNumber, Select } from "antd"; -import { budgetCreateCall } from "../networking"; +import { useCreateBudget } from "@/app/(dashboard)/hooks/budgets/useBudgets"; import NotificationsManager from "../molecules/notifications_manager"; interface BudgetModalProps { isModalVisible: boolean; - accessToken: string | null; setIsModalVisible: React.Dispatch>; - setBudgetList: React.Dispatch>; } -const BudgetModal: React.FC = ({ isModalVisible, accessToken, setIsModalVisible, setBudgetList }) => { +const BudgetModal: React.FC = ({ isModalVisible, setIsModalVisible }) => { const [form] = Form.useForm(); + const createBudget = useCreateBudget(); + const handleOk = () => { setIsModalVisible(false); form.resetFields(); @@ -23,20 +23,15 @@ const BudgetModal: React.FC = ({ isModalVisible, accessToken, }; const handleCreate = async (formValues: Record) => { - if (accessToken == null || accessToken == undefined) { - return; - } try { NotificationsManager.info("Making API Call"); - // setIsModalVisible(true); - const response = await budgetCreateCall(accessToken, formValues); - console.log("key create Response:", response); - setBudgetList((prevData) => (prevData ? [...prevData, response] : [response])); // Check if prevData is null + await createBudget.mutateAsync(formValues); NotificationsManager.success("Budget Created"); form.resetFields(); + setIsModalVisible(false); } catch (error) { - console.error("Error creating the key:", error); - NotificationsManager.fromBackend(`Error creating the key: ${error}`); + console.error("Error creating the budget:", error); + NotificationsManager.fromBackend(`Error creating the budget: ${error}`); } }; diff --git a/ui/litellm-dashboard/src/components/budgets/budget_panel.test.tsx b/ui/litellm-dashboard/src/components/budgets/budget_panel.test.tsx index 534693d3984..ecae379c9f1 100644 --- a/ui/litellm-dashboard/src/components/budgets/budget_panel.test.tsx +++ b/ui/litellm-dashboard/src/components/budgets/budget_panel.test.tsx @@ -1,31 +1,50 @@ -import * as networking from "../networking"; import { fireEvent, render, waitFor, screen } from "@testing-library/react"; import { act } from "@testing-library/react"; +import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; import { afterEach, describe, expect, it, vi } from "vitest"; import BudgetPanel from "./budget_panel"; -vi.mock("../networking", () => ({ - getBudgetList: vi.fn(), - budgetDeleteCall: vi.fn(), +const mockBudgets = [ + { + budget_id: "budget-1", + max_budget: 100, + rpm_limit: 10, + tpm_limit: 1000, + updated_at: "2024-01-01T00:00:00Z", + }, +]; + +vi.mock("@/app/(dashboard)/hooks/budgets/useBudgets", () => ({ + useBudgets: vi.fn().mockReturnValue({ data: [], isLoading: false }), + useDeleteBudget: vi.fn().mockReturnValue({ mutateAsync: vi.fn(), isPending: false }), + useCreateBudget: vi.fn().mockReturnValue({ mutateAsync: vi.fn() }), + useUpdateBudget: vi.fn().mockReturnValue({ mutateAsync: vi.fn() }), })); +import { useBudgets, useDeleteBudget, useCreateBudget, useUpdateBudget } from "@/app/(dashboard)/hooks/budgets/useBudgets"; + +const createQueryClient = () => + new QueryClient({ + defaultOptions: { queries: { retry: false, gcTime: 0 } }, + }); + +function renderWithProviders(ui: React.ReactElement) { + const qc = createQueryClient(); + return render({ui}); +} + describe("Budget Panel", () => { afterEach(() => { vi.clearAllMocks(); }); it("should render the budget panel and load budgets", async () => { - vi.mocked(networking.getBudgetList).mockResolvedValue([ - { - budget_id: "budget-1", - max_budget: "100", - rpm_limit: 10, - tpm_limit: 1000, - updated_at: "2024-01-01T00:00:00Z", - }, - ]); + vi.mocked(useBudgets).mockReturnValue({ + data: mockBudgets, + isLoading: false, + } as any); - render(); + renderWithProviders(); await waitFor(() => { expect(screen.getByText("Create a budget to assign to customers.")).toBeInTheDocument(); @@ -34,17 +53,20 @@ describe("Budget Panel", () => { }); it("should open delete modal when clicking delete icon", async () => { - vi.mocked(networking.getBudgetList).mockResolvedValue([ - { - budget_id: "budget-to-delete", - max_budget: "200", - rpm_limit: 20, - tpm_limit: 2000, - updated_at: "2024-01-02T00:00:00Z", - }, - ]); + vi.mocked(useBudgets).mockReturnValue({ + data: [ + { + budget_id: "budget-to-delete", + max_budget: 200, + rpm_limit: 20, + tpm_limit: 2000, + updated_at: "2024-01-02T00:00:00Z", + }, + ], + isLoading: false, + } as any); - render(); + renderWithProviders(); await waitFor(() => { expect(screen.getByText("budget-to-delete")).toBeInTheDocument(); @@ -62,18 +84,25 @@ describe("Budget Panel", () => { }); it("should successfully delete a budget", async () => { - vi.mocked(networking.getBudgetList).mockResolvedValue([ - { - budget_id: "budget-to-delete", - max_budget: "200", - rpm_limit: 20, - tpm_limit: 2000, - updated_at: "2024-01-02T00:00:00Z", - }, - ]); - vi.mocked(networking.budgetDeleteCall).mockResolvedValue(undefined); + const deleteMutateAsync = vi.fn().mockResolvedValue(undefined); + vi.mocked(useBudgets).mockReturnValue({ + data: [ + { + budget_id: "budget-to-delete", + max_budget: 200, + rpm_limit: 20, + tpm_limit: 2000, + updated_at: "2024-01-02T00:00:00Z", + }, + ], + isLoading: false, + } as any); + vi.mocked(useDeleteBudget).mockReturnValue({ + mutateAsync: deleteMutateAsync, + isPending: false, + } as any); - render(); + renderWithProviders(); await waitFor(() => { expect(screen.getByText("budget-to-delete")).toBeInTheDocument(); @@ -96,24 +125,43 @@ describe("Budget Panel", () => { }); await waitFor(() => { - expect(networking.budgetDeleteCall).toHaveBeenCalledWith("token-123", "budget-to-delete"); - expect(networking.getBudgetList).toHaveBeenCalledTimes(2); // Initial load + refresh after delete + expect(deleteMutateAsync).toHaveBeenCalledWith("budget-to-delete"); + }); + }); + + it("should render empty state without crashing", async () => { + vi.mocked(useBudgets).mockReturnValue({ + data: [], + isLoading: false, + } as any); + + renderWithProviders(); + + await waitFor(() => { + expect(screen.getByText("Create a budget to assign to customers.")).toBeInTheDocument(); }); }); it("should handle delete error", async () => { - vi.mocked(networking.getBudgetList).mockResolvedValue([ - { - budget_id: "budget-to-delete", - max_budget: "200", - rpm_limit: 20, - tpm_limit: 2000, - updated_at: "2024-01-02T00:00:00Z", - }, - ]); - vi.mocked(networking.budgetDeleteCall).mockRejectedValue(new Error("Delete failed")); + const deleteMutateAsync = vi.fn().mockRejectedValue(new Error("Delete failed")); + vi.mocked(useBudgets).mockReturnValue({ + data: [ + { + budget_id: "budget-to-delete", + max_budget: 200, + rpm_limit: 20, + tpm_limit: 2000, + updated_at: "2024-01-02T00:00:00Z", + }, + ], + isLoading: false, + } as any); + vi.mocked(useDeleteBudget).mockReturnValue({ + mutateAsync: deleteMutateAsync, + isPending: false, + } as any); - render(); + renderWithProviders(); await waitFor(() => { expect(screen.getByText("budget-to-delete")).toBeInTheDocument(); @@ -136,10 +184,38 @@ describe("Budget Panel", () => { }); await waitFor(() => { - expect(networking.budgetDeleteCall).toHaveBeenCalledWith("token-123", "budget-to-delete"); + expect(deleteMutateAsync).toHaveBeenCalledWith("budget-to-delete"); + }); + }); + + it("should open edit modal when clicking edit icon", async () => { + vi.mocked(useBudgets).mockReturnValue({ + data: [ + { + budget_id: "budget-to-edit", + max_budget: 300, + rpm_limit: 30, + tpm_limit: 3000, + updated_at: "2024-01-03T00:00:00Z", + }, + ], + isLoading: false, + } as any); + + renderWithProviders(); + + await waitFor(() => { + expect(screen.getByText("budget-to-edit")).toBeInTheDocument(); }); - // Modal should still be open (error handling) - expect(screen.getByText("Delete Budget?")).toBeInTheDocument(); + const editButton = screen.getByTestId("edit-budget-button"); + + act(() => { + fireEvent.click(editButton); + }); + + await waitFor(() => { + expect(screen.getByText("Edit Budget")).toBeInTheDocument(); + }); }); }); diff --git a/ui/litellm-dashboard/src/components/budgets/budget_panel.tsx b/ui/litellm-dashboard/src/components/budgets/budget_panel.tsx index b52ef5ab947..e42d0569652 100644 --- a/ui/litellm-dashboard/src/components/budgets/budget_panel.tsx +++ b/ui/litellm-dashboard/src/components/budgets/budget_panel.tsx @@ -19,12 +19,12 @@ import { TabPanels, Text, } from "@tremor/react"; -import React, { useEffect, useState } from "react"; +import React, { useState } from "react"; import { Prism as SyntaxHighlighter } from "react-syntax-highlighter"; import DeleteResourceModal from "../common_components/DeleteResourceModal"; import TableIconActionButton from "../common_components/IconActionButton/TableIconActionButtons/TableIconActionButton"; import NotificationsManager from "../molecules/notifications_manager"; -import { budgetDeleteCall, getBudgetList } from "../networking"; +import { useBudgets, useDeleteBudget } from "@/app/(dashboard)/hooks/budgets/useBudgets"; import BudgetModal from "./budget_modal"; import EditBudgetModal from "./edit_budget_modal"; import { CREATE_END_USER_CURL_COMMAND, CHAT_COMPLETIONS_CURL_COMMAND, OPENAI_SDK_PYTHON_CODE } from "./constants"; @@ -35,7 +35,7 @@ interface BudgetSettingsPageProps { export interface budgetItem { budget_id: string; - max_budget: string | null; + max_budget: number | null; rpm_limit: number | null; tpm_limit: number | null; updated_at: string; @@ -45,17 +45,10 @@ const BudgetPanel: React.FC = ({ accessToken }) => { const [isCreateModelVisible, setIsCreateModelVisible] = useState(false); const [isEditModalVisible, setIsEditModalVisible] = useState(false); const [selectedBudget, setSelectedBudget] = useState(null); - const [budgetList, setBudgetList] = useState([]); - const [isDeleting, setIsDeleting] = useState(false); const [isDeleteModalVisible, setIsDeleteModalVisible] = useState(false); - useEffect(() => { - if (!accessToken) { - return; - } - getBudgetList(accessToken).then((data) => { - setBudgetList(data); - }); - }, [accessToken]); + + const { data: budgetList = [] } = useBudgets(); + const deleteBudget = useDeleteBudget(); const handleEditCall = async (budget: budgetItem) => { if (accessToken == null) { @@ -74,11 +67,9 @@ const BudgetPanel: React.FC = ({ accessToken }) => { if (!selectedBudget || accessToken == null) { return; } - setIsDeleting(true); try { - await budgetDeleteCall(accessToken, selectedBudget.budget_id); + await deleteBudget.mutateAsync(selectedBudget.budget_id); NotificationsManager.success("Budget deleted."); - await handleUpdateCall(); } catch (error) { console.error("Error deleting budget:", error); if (typeof NotificationsManager.fromBackend === "function") { @@ -87,7 +78,6 @@ const BudgetPanel: React.FC = ({ accessToken }) => { NotificationsManager.info("Failed to delete budget"); } } finally { - setIsDeleting(false); setIsDeleteModalVisible(false); setSelectedBudget(null); } @@ -97,15 +87,6 @@ const BudgetPanel: React.FC = ({ accessToken }) => { setIsDeleteModalVisible(false); }; - const handleUpdateCall = async () => { - if (accessToken == null) { - return; - } - getBudgetList(accessToken).then((data) => { - setBudgetList(data); - }); - }; - return (