diff --git a/.circleci/config.yml b/.circleci/config.yml index 790efc79862..d4943b59c9a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -406,119 +406,6 @@ jobs: # Store test results - store_test_results: path: test-results - caching_unit_tests: - docker: - - image: cimg/python:3.11 - auth: - username: ${DOCKERHUB_USERNAME} - password: ${DOCKERHUB_PASSWORD} - resource_class: large - working_directory: ~/project - parallelism: 2 - - steps: - - checkout - - setup_google_dns - - run: - name: DNS lookup for Redis host - command: | - sudo apt-get update - sudo apt-get install -y dnsutils - dig redis-19899.c239.us-east-1-2.ec2.redns.redis-cloud.com +short - - run: - name: Show git commit hash - command: | - echo "Git commit hash: $CIRCLE_SHA1" - - - restore_cache: - keys: - - v2-caching-deps-{{ checksum ".circleci/requirements.txt" }} - - v2-caching-deps- - - run: - name: Install Dependencies - command: | - python -m pip install --upgrade pip - python -m pip install -r .circleci/requirements.txt - pip install "pytest==7.3.1" - pip install "pytest-retry==1.6.3" - pip install "pytest-asyncio==0.21.1" - pip install "pytest-cov==5.0.0" - pip install "mypy==1.18.2" - pip install "google-generativeai==0.3.2" - pip install "google-cloud-aiplatform==1.43.0" - pip install pyarrow - pip install "boto3==1.36.0" - pip install "aioboto3==13.4.0" - pip install langchain - pip install lunary==0.2.5 - pip install "azure-identity==1.16.1" - pip install "langfuse==2.59.7" - pip install "logfire==0.29.0" - pip install numpydoc - pip install traceloop-sdk==0.21.1 - pip install opentelemetry-api==1.25.0 - pip install opentelemetry-sdk==1.25.0 - pip install opentelemetry-exporter-otlp==1.25.0 - pip install openai==1.100.1 - pip install prisma==0.11.0 - pip install "detect_secrets==1.5.0" - pip install "httpx==0.24.1" - pip install "respx==0.22.0" - pip install fastapi - pip install "gunicorn==21.2.0" - pip install "anyio==4.2.0" - pip install "aiodynamo==23.10.1" - pip install "asyncio==3.4.3" - pip install "apscheduler==3.10.4" - pip install "PyGithub==1.59.1" - pip install argon2-cffi - pip install "pytest-mock==3.12.0" - pip install python-multipart - pip install google-cloud-aiplatform - pip install prometheus-client==0.20.0 - pip install "pydantic==2.10.2" - pip install "diskcache==5.6.1" - pip install "Pillow==10.3.0" - pip install "jsonschema==4.22.0" - pip install "websockets==13.1.0" - pip install "pytest-xdist==3.6.1" - - setup_litellm_enterprise_pip - - save_cache: - paths: - - /home/circleci/.pyenv/versions - - /home/circleci/.local - key: v2-caching-deps-{{ checksum ".circleci/requirements.txt" }} - - run: - name: Run prisma ./docker/entrypoint.sh - command: | - set +e - chmod +x docker/entrypoint.sh - ./docker/entrypoint.sh - set -e - - # Run pytest and generate JUnit XML report - - run: - name: Run tests - command: | - pwd - ls - mkdir -p test-results - - TEST_FILES=$(circleci tests glob "tests/local_testing/**/test_*.py") - - echo "$TEST_FILES" | circleci tests run \ - --split-by=timings \ - --verbose \ - --command="xargs python -m pytest \ - -v \ - --junitxml=test-results/junit.xml \ - --durations=5 \ - -k 'caching or cache'" - no_output_timeout: 15m - - # Store test results - - store_test_results: - path: test-results auth_ui_unit_tests: docker: - image: cimg/python:3.11 @@ -664,376 +551,6 @@ jobs: # Store test results - store_test_results: path: test-results - litellm_security_tests: - docker: - - image: cimg/python:3.13 - auth: - username: ${DOCKERHUB_USERNAME} - password: ${DOCKERHUB_PASSWORD} - - image: cimg/postgres:14.0 - environment: - POSTGRES_USER: postgres - POSTGRES_PASSWORD: postgres - POSTGRES_DB: circle_test - resource_class: xlarge - working_directory: ~/project - environment: - DATABASE_URL: "postgresql://postgres:postgres@localhost:5432/circle_test" - steps: - - checkout - - setup_google_dns - - run: - name: Show git commit hash - command: | - echo "Git commit hash: $CIRCLE_SHA1" - - setup_remote_docker: - docker_layer_caching: true - - restore_cache: - keys: - - v3-litellm-uv-deps-{{ checksum "requirements.txt" }}-{{ checksum ".circleci/config.yml" }} - - run: - name: Install Dependencies - command: | - python -m pip install --upgrade pip uv - uv pip install --system -r requirements.txt - pip install "pytest==7.3.1" "pytest-retry==1.6.3" "pytest-mock==3.12.0" \ - "pytest-asyncio==0.21.1" "pytest-cov==5.0.0" - - save_cache: - paths: - - ~/.local/lib - - ~/.local/bin - - ~/.cache/uv - key: v3-litellm-uv-deps-{{ checksum "requirements.txt" }}-{{ checksum ".circleci/config.yml" }} - - run: - name: Install dockerize - command: | - wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz - sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz - rm dockerize-linux-amd64-v0.6.1.tar.gz - - run: - name: Wait for PostgreSQL to be ready - command: dockerize -wait tcp://localhost:5432 -timeout 1m - - run: - name: Run Security Scans - command: | - chmod +x ci_cd/security_scans.sh - ./ci_cd/security_scans.sh - - run: - name: Run prisma ./docker/entrypoint.sh - command: | - set +e - chmod +x docker/entrypoint.sh - ./docker/entrypoint.sh - set -e - # Run pytest and generate JUnit XML report - - run: - name: Run tests - command: | - python -m pytest tests/proxy_security_tests -v -x --junitxml=test-results/junit.xml --durations=5 - no_output_timeout: 15m - # Store test results - - store_test_results: - path: test-results - # Split proxy unit tests into 3 jobs for faster execution and better debugging - # test_key_generate_prisma runs separately without parallel execution to avoid event loop issues with logging worker - litellm_proxy_unit_testing_key_generation: - docker: - - image: cimg/python:3.11 - auth: - username: ${DOCKERHUB_USERNAME} - password: ${DOCKERHUB_PASSWORD} - working_directory: ~/project - resource_class: medium - steps: - - checkout - - setup_google_dns - - run: - name: Show git commit hash - command: | - echo "Git commit hash: $CIRCLE_SHA1" - - run: - name: Install PostgreSQL - command: | - sudo apt-get update - sudo apt-get install -y postgresql-14 postgresql-contrib-14 - - restore_cache: - keys: - - v1-dependencies-{{ checksum ".circleci/requirements.txt" }} - - run: - name: Install Dependencies - command: | - python -m pip install --upgrade pip - python -m pip install -r .circleci/requirements.txt - pip install "pytest==7.3.1" - pip install "pytest-retry==1.6.3" - pip install "pytest-asyncio==0.21.1" - pip install "pytest-cov==5.0.0" - pip install "pytest-timeout==2.2.0" - pip install "pytest-forked==1.6.0" - pip install "mypy==1.18.2" - pip install "google-generativeai==0.3.2" - pip install "google-cloud-aiplatform==1.43.0" - pip install "google-genai==1.22.0" - pip install pyarrow - pip install "boto3==1.36.0" - pip install "aioboto3==13.4.0" - pip install langchain - pip install lunary==0.2.5 - pip install "azure-identity==1.16.1" - pip install "langfuse==2.59.7" - pip install "logfire==0.29.0" - pip install numpydoc - pip install traceloop-sdk==0.21.1 - pip install opentelemetry-api==1.25.0 - pip install opentelemetry-sdk==1.25.0 - pip install opentelemetry-exporter-otlp==1.25.0 - pip install openai==1.100.1 - pip install prisma==0.11.0 - pip install "detect_secrets==1.5.0" - pip install "httpx==0.24.1" - pip install "respx==0.22.0" - pip install fastapi - pip install "gunicorn==21.2.0" - pip install "anyio==4.2.0" - pip install "aiodynamo==23.10.1" - pip install "asyncio==3.4.3" - pip install "apscheduler==3.10.4" - pip install "PyGithub==1.59.1" - pip install argon2-cffi - pip install "pytest-mock==3.12.0" - pip install python-multipart - pip install google-cloud-aiplatform - pip install prometheus-client==0.20.0 - pip install "pydantic==2.10.2" - pip install "diskcache==5.6.1" - pip install "Pillow==10.3.0" - pip install "jsonschema==4.22.0" - pip install "pytest-postgresql==7.0.1" - pip install "fakeredis==2.28.1" - - setup_litellm_enterprise_pip - - save_cache: - paths: - - ./venv - key: v1-dependencies-{{ checksum ".circleci/requirements.txt" }} - - run: - name: Run prisma ./docker/entrypoint.sh - command: | - set +e - chmod +x docker/entrypoint.sh - ./docker/entrypoint.sh - set -e - - run: - name: Run key generation tests (no parallel execution to avoid event loop issues) - command: | - pwd - ls - # Run without -n flag to avoid pytest-xdist event loop conflicts with logging worker - python -m pytest tests/proxy_unit_tests/test_key_generate_prisma.py --cov=litellm --cov-report=xml --junitxml=test-results/junit-key-generation.xml --durations=10 --timeout=300 -vv --log-cli-level=INFO - no_output_timeout: 15m - - run: - name: Rename the coverage files - command: | - mv coverage.xml litellm_proxy_unit_tests_key_generation_coverage.xml - mv .coverage litellm_proxy_unit_tests_key_generation_coverage - - store_test_results: - path: test-results - - persist_to_workspace: - root: . - paths: - - litellm_proxy_unit_tests_key_generation_coverage.xml - - litellm_proxy_unit_tests_key_generation_coverage - litellm_proxy_unit_testing_part1: - docker: - - image: cimg/python:3.11 - auth: - username: ${DOCKERHUB_USERNAME} - password: ${DOCKERHUB_PASSWORD} - working_directory: ~/project - resource_class: xlarge - steps: - - checkout - - setup_google_dns - - run: - name: Show git commit hash - command: | - echo "Git commit hash: $CIRCLE_SHA1" - - run: - name: Install PostgreSQL - command: | - sudo apt-get update - sudo apt-get install -y postgresql-14 postgresql-contrib-14 - - restore_cache: - keys: - - v1-dependencies-{{ checksum ".circleci/requirements.txt" }} - - run: - name: Install Dependencies - command: | - python -m pip install --upgrade pip - python -m pip install -r .circleci/requirements.txt - pip install "pytest==7.3.1" - pip install "pytest-retry==1.6.3" - pip install "pytest-asyncio==0.21.1" - pip install "pytest-cov==5.0.0" - pip install "pytest-timeout==2.2.0" - pip install "pytest-forked==1.6.0" - pip install "mypy==1.18.2" - pip install "google-generativeai==0.3.2" - pip install "google-cloud-aiplatform==1.43.0" - pip install "google-genai==1.22.0" - pip install pyarrow - pip install "boto3==1.36.0" - pip install "aioboto3==13.4.0" - pip install langchain - pip install lunary==0.2.5 - pip install "azure-identity==1.16.1" - pip install "langfuse==2.59.7" - pip install "logfire==0.29.0" - pip install numpydoc - pip install traceloop-sdk==0.21.1 - pip install opentelemetry-api==1.25.0 - pip install opentelemetry-sdk==1.25.0 - pip install opentelemetry-exporter-otlp==1.25.0 - pip install openai==1.100.1 - pip install prisma==0.11.0 - pip install "detect_secrets==1.5.0" - pip install "httpx==0.24.1" - pip install "respx==0.22.0" - pip install fastapi - pip install "gunicorn==21.2.0" - pip install "anyio==4.2.0" - pip install "aiodynamo==23.10.1" - pip install "asyncio==3.4.3" - pip install "apscheduler==3.10.4" - pip install "PyGithub==1.59.1" - pip install argon2-cffi - pip install "pytest-mock==3.12.0" - pip install python-multipart - pip install google-cloud-aiplatform - pip install prometheus-client==0.20.0 - pip install "pydantic==2.10.2" - pip install "diskcache==5.6.1" - pip install "Pillow==10.3.0" - pip install "jsonschema==4.22.0" - pip install "pytest-postgresql==7.0.1" - pip install "fakeredis==2.28.1" - pip install "pytest-xdist==3.6.1" - - setup_litellm_enterprise_pip - - save_cache: - paths: - - ./venv - key: v1-dependencies-{{ checksum ".circleci/requirements.txt" }} - - run: - name: Run prisma ./docker/entrypoint.sh - command: | - set +e - chmod +x docker/entrypoint.sh - ./docker/entrypoint.sh - set -e - - run: - name: Run proxy unit tests (part 1 - auth checks) - command: | - pwd - ls - python -m pytest tests/proxy_unit_tests/test_auth_checks.py tests/proxy_unit_tests/test_user_api_key_auth.py --junitxml=test-results/junit-part1.xml --durations=10 -n 8 --timeout=300 -v - no_output_timeout: 15m - - store_test_results: - path: test-results - litellm_proxy_unit_testing_part2: - docker: - - image: cimg/python:3.11 - auth: - username: ${DOCKERHUB_USERNAME} - password: ${DOCKERHUB_PASSWORD} - working_directory: ~/project - resource_class: xlarge - steps: - - checkout - - setup_google_dns - - run: - name: Show git commit hash - command: | - echo "Git commit hash: $CIRCLE_SHA1" - - run: - name: Install PostgreSQL - command: | - sudo apt-get update - sudo apt-get install -y postgresql-14 postgresql-contrib-14 - - restore_cache: - keys: - - v1-dependencies-{{ checksum ".circleci/requirements.txt" }} - - run: - name: Install Dependencies - command: | - python -m pip install --upgrade pip - python -m pip install -r .circleci/requirements.txt - pip install "pytest==7.3.1" - pip install "pytest-retry==1.6.3" - pip install "pytest-asyncio==0.21.1" - pip install "pytest-cov==5.0.0" - pip install "pytest-timeout==2.2.0" - pip install "pytest-forked==1.6.0" - pip install "mypy==1.18.2" - pip install "google-generativeai==0.3.2" - pip install "google-cloud-aiplatform==1.43.0" - pip install "google-genai==1.22.0" - pip install pyarrow - pip install "boto3==1.36.0" - pip install "aioboto3==13.4.0" - pip install langchain - pip install lunary==0.2.5 - pip install "azure-identity==1.16.1" - pip install "langfuse==2.59.7" - pip install "logfire==0.29.0" - pip install numpydoc - pip install traceloop-sdk==0.21.1 - pip install opentelemetry-api==1.25.0 - pip install opentelemetry-sdk==1.25.0 - pip install opentelemetry-exporter-otlp==1.25.0 - pip install openai==1.100.1 - pip install prisma==0.11.0 - pip install "detect_secrets==1.5.0" - pip install "httpx==0.24.1" - pip install "respx==0.22.0" - pip install fastapi - pip install "gunicorn==21.2.0" - pip install "anyio==4.2.0" - pip install "aiodynamo==23.10.1" - pip install "asyncio==3.4.3" - pip install "apscheduler==3.10.4" - pip install "PyGithub==1.59.1" - pip install argon2-cffi - pip install "pytest-mock==3.12.0" - pip install python-multipart - pip install google-cloud-aiplatform - pip install prometheus-client==0.20.0 - pip install "pydantic==2.10.2" - pip install "diskcache==5.6.1" - pip install "Pillow==10.3.0" - pip install "jsonschema==4.22.0" - pip install "pytest-postgresql==7.0.1" - pip install "fakeredis==2.28.1" - pip install "pytest-xdist==3.6.1" - - setup_litellm_enterprise_pip - - save_cache: - paths: - - ./venv - key: v1-dependencies-{{ checksum ".circleci/requirements.txt" }} - - run: - name: Run prisma ./docker/entrypoint.sh - command: | - set +e - chmod +x docker/entrypoint.sh - ./docker/entrypoint.sh - set -e - - run: - name: Run proxy unit tests (part 2 - remaining tests) - command: | - pwd - ls - python -m pytest tests/proxy_unit_tests --ignore=tests/proxy_unit_tests/test_key_generate_prisma.py --ignore=tests/proxy_unit_tests/test_auth_checks.py --ignore=tests/proxy_unit_tests/test_user_api_key_auth.py --junitxml=test-results/junit-part2.xml --durations=10 -n 8 --timeout=300 -v - no_output_timeout: 15m - - store_test_results: - path: test-results litellm_assistants_api_testing: # Runs all tests with the "assistants" keyword docker: - image: cimg/python:3.13.1 @@ -1507,101 +1024,6 @@ jobs: no_output_timeout: 15m - store_test_results: path: test-results - litellm_mapped_tests_llms: - docker: - - image: cimg/python:3.11 - auth: - username: ${DOCKERHUB_USERNAME} - password: ${DOCKERHUB_PASSWORD} - working_directory: ~/project - resource_class: large - steps: - - setup_litellm_test_deps - - run: - name: Run LLM provider tests - command: | - python -m pytest tests/test_litellm/llms --junitxml=test-results/junit-llms.xml --durations=10 -n 4 --maxfail=5 --timeout=300 -vv --log-cli-level=WARNING - no_output_timeout: 15m - - store_test_results: - path: test-results - litellm_mapped_tests_core: - docker: - - image: cimg/python:3.11 - auth: - username: ${DOCKERHUB_USERNAME} - password: ${DOCKERHUB_PASSWORD} - working_directory: ~/project - resource_class: large - steps: - - setup_litellm_test_deps - - run: - name: Run core tests - command: | - python -m pytest tests/test_litellm --ignore=tests/test_litellm/proxy --ignore=tests/test_litellm/llms --ignore=tests/test_litellm/integrations --ignore=tests/test_litellm/litellm_core_utils --ignore=tests/test_litellm/experimental_mcp_client --junitxml=test-results/junit-core.xml --durations=10 -n 4 --maxfail=5 --timeout=300 -vv --log-cli-level=WARNING - no_output_timeout: 15m - - store_test_results: - path: test-results - litellm_mapped_tests_litellm_core_utils: - docker: - - image: cimg/python:3.11 - auth: - username: ${DOCKERHUB_USERNAME} - password: ${DOCKERHUB_PASSWORD} - working_directory: ~/project - resource_class: large - steps: - - setup_litellm_test_deps - - run: - name: Run litellm_core_utils tests - command: | - python -m pytest tests/test_litellm/litellm_core_utils --junitxml=test-results/junit-litellm-core-utils.xml --durations=10 -n 4 --maxfail=5 --timeout=300 -vv --log-cli-level=WARNING - no_output_timeout: 15m - - store_test_results: - path: test-results - litellm_mapped_tests_mcps: - docker: - - image: cimg/python:3.11 - auth: - username: ${DOCKERHUB_USERNAME} - password: ${DOCKERHUB_PASSWORD} - working_directory: ~/project - resource_class: medium - steps: - - setup_litellm_test_deps - - run: - name: Run MCP client tests - command: | - python -m pytest tests/test_litellm/experimental_mcp_client --cov=litellm --cov-report=xml --junitxml=test-results/junit-mcps.xml --durations=10 -n 2 --maxfail=5 --timeout=300 -vv --log-cli-level=WARNING - no_output_timeout: 15m - - run: - name: Rename the coverage files - command: | - mv coverage.xml litellm_mcps_tests_coverage.xml - mv .coverage litellm_mcps_tests_coverage - - store_test_results: - path: test-results - - persist_to_workspace: - root: . - paths: - - litellm_mcps_tests_coverage.xml - - litellm_mcps_tests_coverage - litellm_mapped_tests_integrations: - docker: - - image: cimg/python:3.11 - auth: - username: ${DOCKERHUB_USERNAME} - password: ${DOCKERHUB_PASSWORD} - working_directory: ~/project - resource_class: large - steps: - - setup_litellm_test_deps - - run: - name: Run integrations tests - command: | - python -m pytest tests/test_litellm/integrations --junitxml=test-results/junit-integrations.xml --durations=10 -n 4 --maxfail=5 --timeout=300 -vv --log-cli-level=WARNING - no_output_timeout: 15m - - store_test_results: - path: test-results litellm_mapped_enterprise_tests: docker: - image: cimg/python:3.11 @@ -2137,6 +1559,25 @@ jobs: pip install "pytest-asyncio==0.21.1" pip install aiohttp pip install apscheduler + - run: + name: Install dockerize + command: | + sudo wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz + sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz + sudo rm dockerize-linux-amd64-v0.6.1.tar.gz + - run: + name: Start PostgreSQL Database + command: | + docker run -d \ + --name postgres-db \ + -e POSTGRES_USER=postgres \ + -e POSTGRES_PASSWORD=postgres \ + -e POSTGRES_DB=litellm_test \ + -p 5432:5432 \ + postgres:14 + - run: + name: Wait for PostgreSQL to be ready + command: dockerize -wait tcp://localhost:5432 -timeout 1m - attach_workspace: at: ~/project - run: @@ -2145,29 +1586,41 @@ jobs: zstd -d litellm-docker-database.tar.zst --stdout | docker load docker images | grep litellm-docker-database - run: - name: Run Docker container + name: Seed database with real schema + command: | + docker run -d \ + -p 4001:4000 \ + -e DATABASE_URL="postgresql://postgres:postgres@host.docker.internal:5432/litellm_test" \ + -e LITELLM_MASTER_KEY="sk-1234" \ + --name schema-seed \ + --add-host=host.docker.internal:host-gateway \ + -v $(pwd)/litellm/proxy/example_config_yaml/simple_config.yaml:/app/config.yaml \ + litellm-docker-database:ci \ + --config /app/config.yaml \ + --port 4000 \ + --use_prisma_db_push + - run: + name: Wait for schema seed to complete + command: dockerize -wait http://localhost:4001 -timeout 5m + - run: + name: Stop schema seed container + command: docker stop schema-seed && docker rm schema-seed + - run: + name: Run Docker container with bad schema and disabled updates command: | docker run -d \ -p 4000:4000 \ - -e DATABASE_URL=$PROXY_DATABASE_URL \ + -e DATABASE_URL="postgresql://postgres:postgres@host.docker.internal:5432/litellm_test" \ -e DEFAULT_NUM_WORKERS_LITELLM_PROXY=1 \ -e DISABLE_SCHEMA_UPDATE="True" \ + --name my-app \ + --add-host=host.docker.internal:host-gateway \ -v $(pwd)/litellm/proxy/example_config_yaml/bad_schema.prisma:/app/schema.prisma \ -v $(pwd)/litellm/proxy/example_config_yaml/bad_schema.prisma:/app/litellm/proxy/schema.prisma \ -v $(pwd)/litellm/proxy/example_config_yaml/disable_schema_update.yaml:/app/config.yaml \ - --name my-app \ litellm-docker-database:ci \ --config /app/config.yaml \ --port 4000 - - run: - name: Install curl and dockerize - command: | - sudo apt-get update - sudo apt-get install -y curl - sudo wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz - sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz - sudo rm dockerize-linux-amd64-v0.6.1.tar.gz - - run: name: Wait for container to be ready command: dockerize -wait http://localhost:4000 -timeout 1m @@ -2575,9 +2028,6 @@ jobs: -e OPENAI_API_KEY=$OPENAI_API_KEY \ -e LITELLM_LICENSE=$LITELLM_LICENSE \ -e OTEL_EXPORTER="in_memory" \ - -e APORIA_API_BASE_2=$APORIA_API_BASE_2 \ - -e APORIA_API_KEY_2=$APORIA_API_KEY_2 \ - -e APORIA_API_BASE_1=$APORIA_API_BASE_1 \ -e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID \ -e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY \ -e DEFAULT_NUM_WORKERS_LITELLM_PROXY=1 \ @@ -2585,7 +2035,6 @@ jobs: -e DD_API_KEY=$DD_API_KEY \ -e DD_SITE=$DD_SITE \ -e AWS_REGION_NAME=$AWS_REGION_NAME \ - -e APORIA_API_KEY_1=$APORIA_API_KEY_1 \ -e COHERE_API_KEY=$COHERE_API_KEY \ -e GCS_FLUSH_INTERVAL="1" \ --add-host host.docker.internal:host-gateway \ @@ -3061,6 +2510,20 @@ jobs: name: Build Docker image command: | docker build -t my-app:latest -f docker/build_from_pip/Dockerfile.build_from_pip . + - run: + name: Start PostgreSQL Database + command: | + docker run -d \ + --name postgres-db \ + -e POSTGRES_USER=postgres \ + -e POSTGRES_PASSWORD=postgres \ + -e POSTGRES_DB=circle_test \ + -p 5432:5432 \ + postgres:14 + - run: + name: Wait for PostgreSQL to be ready + command: | + timeout 60s bash -c 'until docker exec postgres-db pg_isready -U postgres -d circle_test; do sleep 2; done' - run: name: Run Docker container # intentionally give bad redis credentials here @@ -3068,7 +2531,7 @@ jobs: command: | docker run -d \ -p 4000:4000 \ - -e DATABASE_URL=$PROXY_DATABASE_URL \ + -e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \ -e REDIS_HOST=$REDIS_HOST \ -e REDIS_PASSWORD=$REDIS_PASSWORD \ -e REDIS_PORT=$REDIS_PORT \ @@ -3076,18 +2539,15 @@ jobs: -e OPENAI_API_KEY=$OPENAI_API_KEY \ -e LITELLM_LICENSE=$LITELLM_LICENSE \ -e OTEL_EXPORTER="in_memory" \ - -e APORIA_API_BASE_2=$APORIA_API_BASE_2 \ - -e APORIA_API_KEY_2=$APORIA_API_KEY_2 \ - -e APORIA_API_BASE_1=$APORIA_API_BASE_1 \ -e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID \ -e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY \ -e AWS_REGION_NAME=$AWS_REGION_NAME \ - -e APORIA_API_KEY_1=$APORIA_API_KEY_1 \ -e COHERE_API_KEY=$COHERE_API_KEY \ -e USE_DDTRACE=True \ -e DD_API_KEY=$DD_API_KEY \ -e DD_SITE=$DD_SITE \ -e GCS_FLUSH_INTERVAL="1" \ + --add-host host.docker.internal:host-gateway \ --name my-app \ -v $(pwd)/docker/build_from_pip/litellm_config.yaml:/app/config.yaml \ my-app:latest \ @@ -3118,8 +2578,11 @@ jobs: - run: name: Stop and remove first container command: | - docker stop my-app - docker rm my-app + docker stop my-app || true + docker rm my-app || true + docker stop postgres-db || true + docker rm postgres-db || true + when: always proxy_pass_through_endpoint_tests: machine: image: ubuntu-2204:2023.10.1 @@ -3543,93 +3006,6 @@ jobs: - codecov/upload: file: ./coverage.xml - publish_to_pypi: - docker: - - image: cimg/python:3.8 - working_directory: ~/project - - environment: - TWINE_USERNAME: __token__ - - steps: - - checkout - - - run: - name: Copy model_prices_and_context_window File to model_prices_and_context_window_backup - command: | - cp model_prices_and_context_window.json litellm/model_prices_and_context_window_backup.json - - - run: - name: Checkout code - command: git checkout $CIRCLE_SHA1 - - # Check if setup.py is modified and publish to PyPI - - run: - name: PyPI publish - command: | - echo "Install TOML package." - python -m pip install toml - VERSION=$(python -c "import toml; print(toml.load('pyproject.toml')['tool']['poetry']['version'])") - PACKAGE_NAME=$(python -c "import toml; print(toml.load('pyproject.toml')['tool']['poetry']['name'])") - if ! pip show -v $PACKAGE_NAME | grep -q "Version: ${VERSION}"; then - echo "pyproject.toml modified" - echo -e "[pypi]\nusername = $PYPI_PUBLISH_USERNAME\npassword = $PYPI_PUBLISH_PASSWORD" > ~/.pypirc - python -m pip install --upgrade pip - pip install build - pip install wheel - pip install --upgrade twine setuptools - rm -rf build dist - - echo "Building package" - python -m build - - echo "Twine upload to dist" - echo "Contents of dist directory:" - ls dist/ - twine upload --verbose dist/* - else - echo "Version ${VERSION} of package is already published on PyPI." - - # Check if corresponding Docker nightly image exists - NIGHTLY_TAG="v${VERSION}-nightly" - echo "Checking for Docker nightly image: litellm/litellm:${NIGHTLY_TAG}" - - # Check Docker Hub for the nightly image - if curl -s "https://hub.docker.com/v2/repositories/litellm/litellm/tags/${NIGHTLY_TAG}" | grep -q "name"; then - echo "Docker nightly image ${NIGHTLY_TAG} exists. This release was already completed successfully." - echo "Skipping PyPI publish and continuing to ensure Docker images are up to date." - circleci step halt - else - echo "ERROR: PyPI package ${VERSION} exists but Docker nightly image ${NIGHTLY_TAG} does not exist!" - echo "This indicates an incomplete release. Please investigate." - exit 1 - fi - fi - - run: - name: Trigger Github Action for new Docker Container + Trigger Load Testing - command: | - echo "Install TOML package." - python3 -m pip install toml - VERSION=$(python3 -c "import toml; print(toml.load('pyproject.toml')['tool']['poetry']['version'])") - echo "LiteLLM Version ${VERSION}" - - # Determine which branch to use for Docker build - if [[ "$CIRCLE_BRANCH" =~ ^litellm_release_day_.* ]]; then - BUILD_BRANCH="$CIRCLE_BRANCH" - echo "Using release branch: $BUILD_BRANCH" - else - BUILD_BRANCH="main" - echo "Using default branch: $BUILD_BRANCH" - fi - - curl -X POST \ - -H "Accept: application/vnd.github.v3+json" \ - -H "Authorization: Bearer $GITHUB_TOKEN" \ - "https://api.github.com/repos/BerriAI/litellm/actions/workflows/ghcr_deploy.yml/dispatches" \ - -d "{\"ref\":\"${BUILD_BRANCH}\", \"inputs\":{\"tag\":\"v${VERSION}-nightly\", \"commit_hash\":\"$CIRCLE_SHA1\"}}" - echo "triggering load testing server for version ${VERSION} and commit ${CIRCLE_SHA1}" - curl -X POST "https://proxyloadtester-production.up.railway.app/start/load/test?version=${VERSION}&commit_hash=${CIRCLE_SHA1}&release_type=nightly" - publish_proxy_extras: docker: - image: cimg/python:3.8 @@ -3942,37 +3318,39 @@ jobs: - setup_google_dns - attach_workspace: at: ~/project + - run: + name: Install dockerize + command: | + sudo wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz + sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz + sudo rm dockerize-linux-amd64-v0.6.1.tar.gz + - run: + name: Start PostgreSQL Database + command: | + docker run -d \ + --name postgres-db \ + -e POSTGRES_USER=postgres \ + -e POSTGRES_PASSWORD=postgres \ + -e POSTGRES_DB=litellm_schema_sync \ + -p 5432:5432 \ + postgres:14 + - run: + name: Wait for PostgreSQL to be ready + command: dockerize -wait tcp://localhost:5432 -timeout 1m - run: name: Load Docker Database Image command: | zstd -d litellm-docker-database.tar.zst --stdout | docker load docker images | grep litellm-docker-database - run: - name: Install Neon CLI + name: Run schema sync via prisma db push command: | - npm i -g neonctl - - run: - name: Install curl and dockerize - command: | - sudo apt-get update - sudo apt-get install -y curl - sudo wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz - sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz - sudo rm dockerize-linux-amd64-v0.6.1.tar.gz - - run: - name: Sync schema on base e2e database - command: | - BASE_DATABASE_URL=$(neon connection-string \ - --project-id $NEON_PROJECT_ID \ - --api-key $NEON_API_KEY \ - --branch br-fancy-paper-ad1olsb3 \ - --database-name yuneng-trial-db \ - --role neondb_owner) docker run -d \ -p 4000:4000 \ - -e DATABASE_URL=$BASE_DATABASE_URL \ + -e DATABASE_URL="postgresql://postgres:postgres@host.docker.internal:5432/litellm_schema_sync" \ -e LITELLM_MASTER_KEY="sk-1234" \ --name schema-sync \ + --add-host=host.docker.internal:host-gateway \ -v $(pwd)/litellm/proxy/example_config_yaml/simple_config.yaml:/app/config.yaml \ litellm-docker-database:ci \ --config /app/config.yaml \ @@ -4089,34 +3467,14 @@ workflows: only: - main - /litellm_.*/ - - caching_unit_tests: - filters: - branches: - only: - main - /litellm_.*/ - - litellm_proxy_unit_testing_key_generation: - filters: - branches: - only: - main - /litellm_.*/ - - litellm_proxy_unit_testing_part1: - filters: - branches: - only: - main - /litellm_.*/ - - litellm_proxy_unit_testing_part2: - filters: - branches: - only: - main - /litellm_.*/ - - litellm_security_tests: - filters: - branches: - only: - main - /litellm_.*/ - litellm_assistants_api_testing: @@ -4170,7 +3528,6 @@ workflows: - main - /litellm_.*/ - prisma_schema_sync: - context: e2e_ui_tests requires: - build_docker_database_image filters: @@ -4178,32 +3535,32 @@ workflows: only: - main - /litellm_.*/ - - e2e_ui_testing: - name: e2e_ui_testing_chromium - browser: chromium - context: e2e_ui_tests - requires: - - ui_build - - build_docker_database_image - - prisma_schema_sync - filters: - branches: - only: - - main - - /litellm_.*/ - - e2e_ui_testing: - name: e2e_ui_testing_firefox - browser: firefox - context: e2e_ui_tests - requires: - - ui_build - - build_docker_database_image - - prisma_schema_sync - filters: - branches: - only: - - main - - /litellm_.*/ + # - e2e_ui_testing: # migrate to dynamic db - currently requires neon cli + # name: e2e_ui_testing_chromium + # browser: chromium + # context: e2e_ui_tests + # requires: + # - ui_build + # - build_docker_database_image + # - prisma_schema_sync + # filters: + # branches: + # only: + # - main + # - /litellm_.*/ + # - e2e_ui_testing: + # name: e2e_ui_testing_firefox + # browser: firefox + # context: e2e_ui_tests + # requires: + # - ui_build + # - build_docker_database_image + # - prisma_schema_sync + # filters: + # branches: + # only: + # - main + # - /litellm_.*/ - build_and_test: requires: - build_docker_database_image @@ -4352,34 +3709,14 @@ workflows: only: - main - /litellm_.*/ - - litellm_mapped_tests_llms: - filters: - branches: - only: - main - /litellm_.*/ - - litellm_mapped_tests_core: - filters: - branches: - only: - main - /litellm_.*/ - - litellm_mapped_tests_mcps: - filters: - branches: - only: - main - /litellm_.*/ - - litellm_mapped_tests_integrations: - filters: - branches: - only: - main - /litellm_.*/ - - litellm_mapped_tests_litellm_core_utils: - filters: - branches: - only: - main - /litellm_.*/ - batches_testing: @@ -4429,11 +3766,6 @@ workflows: - search_testing - litellm_mapped_tests_proxy_part1 - litellm_mapped_tests_proxy_part2 - - litellm_mapped_tests_llms - - litellm_mapped_tests_core - - litellm_mapped_tests_mcps - - litellm_mapped_tests_integrations - - litellm_mapped_tests_litellm_core_utils - litellm_mapped_enterprise_tests - batches_testing - litellm_utils_testing @@ -4441,8 +3773,6 @@ workflows: - image_gen_testing - logging_testing - audio_testing - - caching_unit_tests - - litellm_proxy_unit_testing_key_generation - langfuse_logging_unit_tests - local_testing_part1 - local_testing_part2 @@ -4489,59 +3819,3 @@ workflows: only: - main - /litellm_release_day_.*/ - - publish_to_pypi: - requires: - - mypy_linting - - semgrep - - local_testing_part1 - - local_testing_part2 - - build_and_test - - e2e_openai_endpoints - - test_bad_database_url - - llm_translation_testing - - realtime_translation_testing - - mcp_testing - - agent_testing - - google_generate_content_endpoint_testing - - llm_responses_api_testing - - ocr_testing - - search_testing - - litellm_mapped_tests_proxy_part1 - - litellm_mapped_tests_proxy_part2 - - litellm_mapped_tests_llms - - litellm_mapped_tests_core - - litellm_mapped_tests_mcps - - litellm_mapped_tests_integrations - - litellm_mapped_tests_litellm_core_utils - - litellm_mapped_enterprise_tests - - batches_testing - - litellm_utils_testing - - pass_through_unit_testing - - image_gen_testing - - logging_testing - - audio_testing - - litellm_router_testing - - litellm_router_unit_testing - - caching_unit_tests - - langfuse_logging_unit_tests - - litellm_assistants_api_testing - - auth_ui_unit_tests - - ui_unit_tests - - db_migration_disable_update_check - - e2e_ui_testing_chromium - - e2e_ui_testing_firefox - - litellm_proxy_unit_testing_key_generation - - litellm_proxy_unit_testing_part1 - - litellm_proxy_unit_testing_part2 - - litellm_security_tests - - installing_litellm_on_python - - installing_litellm_on_python_3_13 - - proxy_logging_guardrails_model_info_tests - - proxy_spend_accuracy_tests - - proxy_multi_instance_tests - - proxy_store_model_in_db_tests - - proxy_build_from_pip_tests - - proxy_pass_through_endpoint_tests - - check_code_and_doc_quality - - publish_proxy_extras - - guardrails_testing diff --git a/.claude/settings.json b/.claude/settings.json deleted file mode 100644 index 8c1d85f96e0..00000000000 --- a/.claude/settings.json +++ /dev/null @@ -1,36 +0,0 @@ -{ - "permissions": { - "allow": [ - "Bash(git show:*)", - "Bash(git worktree add:*)", - "Read(//Users/krrishdholakia/Documents/litellm/**)", - "Read(//Users/krrishdholakia/Documents/litellm-claude-code-guardrails/litellm/types/**)", - "Read(//Users/krrishdholakia/Documents/litellm-claude-code-guardrails/**)", - "Read(//Users/krrishdholakia/Documents/litellm-claude-code-guardrails/litellm/**)", - "Bash(python:*)", - "Bash(python -c \"\nimport sys; sys.path.insert\\(0, ''.''\\)\nfrom litellm.proxy.guardrails.guardrail_hooks.claude_code.guardrail import ClaudeCodeGuardrail, HOSTED_TOOL_PREFIXES\nprint\\(''HOSTED_TOOL_PREFIXES:'', HOSTED_TOOL_PREFIXES\\)\nprint\\(''ClaudeCodeGuardrail imported OK''\\)\n\")", - "Read(//Users/krrishdholakia/Documents/litellm-mcp-jwt-groups/litellm/proxy/**)", - "Read(//Users/krrishdholakia/Documents/litellm-mcp-jwt-groups/**)", - "Bash(poetry run pytest:*)", - "Bash(git add:*)", - "Bash(git commit:*)", - "Bash(poetry run python:*)", - "Bash(poetry run pip:*)", - "Bash(git reset:*)", - "Bash(git cherry-pick:*)", - "Bash(git checkout:*)", - "Read(//Users/krrishdholakia/Documents/litellm/litellm/proxy/guardrails/guardrail_hooks/**)", - "Read(//Users/krrishdholakia/Documents/**)", - "Bash(git -C /Users/krrishdholakia/Documents/litellm-mcp-user-permissions worktree list)", - "Bash(ls:*)" - ], - "additionalDirectories": [ - "/Users/krrishdholakia/Documents/litellm-mcp-group-plan/plan", - "/Users/krrishdholakia/Documents/litellm-claude-code-guardrails/litellm/proxy/guardrails/guardrail_hooks/claude_code", - "/Users/krrishdholakia/Documents/litellm-claude-code-guardrails/litellm/types", - "/Users/krrishdholakia/Documents/litellm-claude-code-guardrails", - "/Users/krrishdholakia/Documents/litellm-mcp-jwt-groups/litellm/proxy", - "/Users/krrishdholakia/Documents/litellm-mcp-jwt-groups/tests/test_litellm/proxy/auth" - ] - } -} diff --git a/.devcontainer/post-create.sh b/.devcontainer/post-create.sh index bd72e91a20f..484baa9041d 100644 --- a/.devcontainer/post-create.sh +++ b/.devcontainer/post-create.sh @@ -12,6 +12,6 @@ echo "[post-create] Generating Prisma client" poetry run prisma generate echo "[post-create] Installing npm dependencies" -cd ui/litellm-dashboard && npm install --no-audit --no-fund +cd ui/litellm-dashboard && npm ci echo "[post-create] Done" \ No newline at end of file diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml index 4744ab048c7..cbf380bac01 100644 --- a/.github/ISSUE_TEMPLATE/config.yml +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -1,7 +1,7 @@ blank_issues_enabled: true contact_links: - name: Schedule Demo - url: https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions + url: https://enterprise.litellm.ai/demo about: Speak directly with Krrish and Ishaan, the founders, to discuss issues, share feedback, or explore improvements for LiteLLM - name: Discord url: https://discord.com/invite/wuPM9dRgDw diff --git a/.github/actions/helm-oci-chart-releaser/action.yml b/.github/actions/helm-oci-chart-releaser/action.yml index 1823e262832..454c591d436 100644 --- a/.github/actions/helm-oci-chart-releaser/action.yml +++ b/.github/actions/helm-oci-chart-releaser/action.yml @@ -41,32 +41,54 @@ runs: using: composite steps: - name: Helm | Setup - uses: azure/setup-helm@v4 + uses: azure/setup-helm@1a275c3b69536ee54be43f2070a358922e12c8d4 # v4.3.1 with: version: v3.20.0 - name: Helm | Login shell: bash - run: echo ${{ inputs.registry_password }} | helm registry login -u ${{ inputs.registry_username }} --password-stdin ${{ inputs.registry }} + env: + REGISTRY_PASSWORD: ${{ inputs.registry_password }} + REGISTRY_USERNAME: ${{ inputs.registry_username }} + REGISTRY: ${{ inputs.registry }} + run: echo "$REGISTRY_PASSWORD" | helm registry login -u "$REGISTRY_USERNAME" --password-stdin "$REGISTRY" - name: Helm | Dependency if: inputs.update_dependencies == 'true' shell: bash - run: helm dependency update ${{ inputs.path == null && format('{0}/{1}', 'charts', inputs.name) || inputs.path }} + env: + CHART_PATH: ${{ inputs.path == null && format('{0}/{1}', 'charts', inputs.name) || inputs.path }} + run: helm dependency update "$CHART_PATH" - name: Helm | Package shell: bash - run: helm package ${{ inputs.path == null && format('{0}/{1}', 'charts', inputs.name) || inputs.path }} --version ${{ inputs.tag }} --app-version ${{ inputs.app_version }} + env: + CHART_PATH: ${{ inputs.path == null && format('{0}/{1}', 'charts', inputs.name) || inputs.path }} + TAG: ${{ inputs.tag }} + APP_VERSION: ${{ inputs.app_version }} + run: helm package "$CHART_PATH" --version "$TAG" --app-version "$APP_VERSION" - name: Helm | Push shell: bash - run: helm push ${{ inputs.name }}-${{ inputs.tag }}.tgz oci://${{ inputs.registry }}/${{ inputs.repository }} + env: + NAME: ${{ inputs.name }} + TAG: ${{ inputs.tag }} + REGISTRY: ${{ inputs.registry }} + REPOSITORY: ${{ inputs.repository }} + run: helm push "${NAME}-${TAG}.tgz" "oci://${REGISTRY}/${REPOSITORY}" - name: Helm | Logout shell: bash - run: helm registry logout ${{ inputs.registry }} + env: + REGISTRY: ${{ inputs.registry }} + run: helm registry logout "$REGISTRY" - name: Helm | Output id: output shell: bash - run: echo "image=${{ inputs.registry }}/${{ inputs.repository }}/${{ inputs.name }}:${{ inputs.tag }}" >> $GITHUB_OUTPUT + env: + REGISTRY: ${{ inputs.registry }} + REPOSITORY: ${{ inputs.repository }} + NAME: ${{ inputs.name }} + TAG: ${{ inputs.tag }} + run: echo "image=${REGISTRY}/${REPOSITORY}/${NAME}:${TAG}" >> $GITHUB_OUTPUT diff --git a/.github/codeql/codeql-config.yml b/.github/codeql/codeql-config.yml index 20807685e12..36d70c1d746 100644 --- a/.github/codeql/codeql-config.yml +++ b/.github/codeql/codeql-config.yml @@ -1,22 +1,21 @@ name: "LiteLLM CodeQL config" -# Use security-extended suite instead of security-and-quality to avoid -# result sets > 2 GiB on this codebase that cause fatal OOM failures. queries: - - uses: security-extended + - uses: security-and-quality -# These two queries are security queries included in security-extended that -# individually produce result sets > 2 GiB on this codebase, causing fatal -# OOM failures. Exclude them as a safety net until CI confirms they no longer -# OOM; drop these exclusions in a follow-up once verified. +# Known OOM queries on large Python codebases: +# CodeQL builds a full data flow graph in memory. These two queries trace +# sensitive data through every log call / regex pattern, causing combinatorial +# path explosion on codebases with extensive logging like LiteLLM (>2 GiB +# result sets). This is a known CodeQL scaling limitation, not a code issue. +# Re-test periodically as CodeQL improves or the codebase refactors logging. query-filters: - exclude: - id: py/clear-text-logging-sensitive-data # CWE-312 — > 2 GiB result set + id: py/clear-text-logging-sensitive-data # CWE-312 - exclude: - id: py/polynomial-redos # CWE-730 — > 2 GiB result set + id: py/polynomial-redos # CWE-730 paths-ignore: - tests - docs - "**/*.md" - - litellm/proxy/_experimental/out diff --git a/.github/dependabot.yaml b/.github/dependabot.yaml index 58e7cfe10da..c49882a8d62 100644 --- a/.github/dependabot.yaml +++ b/.github/dependabot.yaml @@ -4,6 +4,9 @@ updates: directory: "/" schedule: interval: "daily" + cooldown: + default-days: 7 + semver-major-days: 14 groups: github-actions: patterns: diff --git a/.github/workflows/_test-unit-base.yml b/.github/workflows/_test-unit-base.yml new file mode 100644 index 00000000000..f1ae30e67d7 --- /dev/null +++ b/.github/workflows/_test-unit-base.yml @@ -0,0 +1,96 @@ +name: _Unit Test Base (Reusable) + +on: + workflow_call: + inputs: + test-path: + description: "Pytest path(s) to run" + required: true + type: string + workers: + description: "Number of pytest-xdist workers" + required: false + type: number + default: 2 + reruns: + description: "Number of reruns for flaky tests" + required: false + type: number + default: 2 + timeout-minutes: + description: "Job timeout in minutes" + required: false + type: number + default: 20 + max-failures: + description: "Stop after this many failures" + required: false + type: number + default: 10 + +permissions: + contents: read + +jobs: + run: + name: Run tests + runs-on: ubuntu-latest + timeout-minutes: ${{ inputs.timeout-minutes }} + + steps: + - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + with: + persist-credentials: false + + - name: Set up Python + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: "3.12" + + - name: Install Poetry + run: pip install 'poetry==2.3.2' + + - name: Cache Poetry dependencies + uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 + with: + path: | + ~/.cache/pypoetry + ~/.cache/pip + .venv + key: ${{ runner.os }}-poetry-${{ hashFiles('poetry.lock') }} + restore-keys: | + ${{ runner.os }}-poetry- + + - name: Install dependencies + run: | + poetry config virtualenvs.in-project true + poetry install --with dev,proxy-dev --extras "proxy semantic-router" + poetry run pip install google-genai==1.22.0 \ + google-cloud-aiplatform==1.115.0 fastapi-offline==1.7.3 python-multipart==0.0.22 openapi-core==0.23.0 + + - name: Setup litellm-enterprise + run: | + poetry run pip install --force-reinstall --no-deps -e enterprise/ + + - name: Generate Prisma client + env: + PRISMA_BINARY_CACHE_DIR: ${{ runner.temp }}/prisma-cache + run: | + poetry run pip install nodejs-wheel-binaries==24.13.1 + poetry run prisma generate --schema litellm/proxy/schema.prisma + + - name: Run tests + env: + TEST_PATH: ${{ inputs.test-path }} + MAX_FAILURES: ${{ inputs.max-failures }} + WORKERS: ${{ inputs.workers }} + RERUNS: ${{ inputs.reruns }} + run: | + poetry run pytest ${TEST_PATH:?} \ + --tb=short -vv \ + --maxfail="${MAX_FAILURES}" \ + -n "${WORKERS}" \ + --reruns "${RERUNS}" \ + --reruns-delay 1 \ + --dist=loadscope \ + --durations=20 diff --git a/.github/workflows/_test-unit-services-base.yml b/.github/workflows/_test-unit-services-base.yml new file mode 100644 index 00000000000..d53a9e8822a --- /dev/null +++ b/.github/workflows/_test-unit-services-base.yml @@ -0,0 +1,164 @@ +name: _Unit Test Services Base (Reusable) + +on: + workflow_call: + inputs: + test-path: + description: "Pytest path(s) to run" + required: true + type: string + workers: + description: "Number of pytest-xdist workers (0 = no parallelism)" + required: false + type: number + default: 2 + reruns: + description: "Number of reruns for flaky tests" + required: false + type: number + default: 2 + timeout-minutes: + description: "Job timeout in minutes" + required: false + type: number + default: 20 + max-failures: + description: "Stop after this many failures" + required: false + type: number + default: 10 + enable-redis: + description: "Pass Redis Cloud credentials to tests via REDIS_HOST/PORT/PASSWORD env vars" + required: false + type: boolean + default: false + enable-postgres: + description: "Start a local Postgres service container and run Prisma migrations" + required: false + type: boolean + default: false + secrets: + REDIS_HOST: + required: false + REDIS_PORT: + required: false + REDIS_PASSWORD: + required: false + DATABASE_URL: + required: false + POSTGRES_USER: + required: false + POSTGRES_PASSWORD: + required: false + +permissions: + contents: read + +jobs: + run: + name: Run tests + runs-on: ubuntu-latest + timeout-minutes: ${{ inputs.timeout-minutes }} + # Environment is derived from the enable-* flags, not caller-controllable. + # This prevents callers from passing arbitrary environment names to bypass secret scoping. + # Note: Postgres service container always starts (GHA limitation), so any Redis job + # also needs Postgres secrets → uses integration-redis-postgres, not integration-redis. + environment: >- + ${{ + inputs.enable-redis && 'integration-redis-postgres' || + inputs.enable-postgres && 'integration-postgres' || + '' + }} + + services: + postgres: + image: postgres@sha256:705a5d5b5836f3fcba0d02c4d281e6a7dd9ed2dd4078640f08a1e1e9896e097d # postgres:14 + env: + POSTGRES_USER: ${{ secrets.POSTGRES_USER }} + POSTGRES_PASSWORD: ${{ secrets.POSTGRES_PASSWORD }} + POSTGRES_DB: litellm_test + ports: + - 5432:5432 + options: >- + --health-cmd "pg_isready" + --health-interval 10s + --health-timeout 5s + --health-retries 5 + + steps: + - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + with: + persist-credentials: false + + - name: Set up Python + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: "3.12" + + - name: Install Poetry + run: pip install 'poetry==2.3.2' + + - name: Cache Poetry dependencies + uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 + with: + path: | + ~/.cache/pypoetry + ~/.cache/pip + .venv + key: ${{ runner.os }}-poetry-services-${{ hashFiles('poetry.lock') }} + restore-keys: | + ${{ runner.os }}-poetry-services- + + - name: Install dependencies + run: | + poetry config virtualenvs.in-project true + poetry install --with dev,proxy-dev --extras "proxy semantic-router" + poetry run pip install google-genai==1.22.0 \ + google-cloud-aiplatform==1.115.0 fastapi-offline==1.7.3 python-multipart==0.0.22 openapi-core==0.23.0 + + - name: Setup litellm-enterprise + run: | + poetry run pip install --force-reinstall --no-deps -e enterprise/ + + - name: Generate Prisma client + env: + PRISMA_BINARY_CACHE_DIR: ${{ runner.temp }}/prisma-cache + run: | + poetry run pip install nodejs-wheel-binaries==24.13.1 + poetry run prisma generate --schema litellm/proxy/schema.prisma + + - name: Run Prisma migrations + if: ${{ inputs.enable-postgres }} + env: + DATABASE_URL: ${{ secrets.DATABASE_URL }} + run: | + poetry run prisma db push --schema litellm/proxy/schema.prisma --accept-data-loss + + - name: Run tests + env: + TEST_PATH: ${{ inputs.test-path }} + MAX_FAILURES: ${{ inputs.max-failures }} + WORKERS: ${{ inputs.workers }} + RERUNS: ${{ inputs.reruns }} + DATABASE_URL: ${{ inputs.enable-postgres && secrets.DATABASE_URL || '' }} + REDIS_HOST: ${{ inputs.enable-redis && secrets.REDIS_HOST || '' }} + REDIS_PORT: ${{ inputs.enable-redis && secrets.REDIS_PORT || '' }} + REDIS_PASSWORD: ${{ inputs.enable-redis && secrets.REDIS_PASSWORD || '' }} + run: | + if [ "${WORKERS}" = "0" ]; then + poetry run pytest ${TEST_PATH:?} \ + --tb=short -vv \ + --maxfail="${MAX_FAILURES}" \ + --reruns "${RERUNS}" \ + --reruns-delay 1 \ + --durations=20 + else + poetry run pytest ${TEST_PATH:?} \ + --tb=short -vv \ + --maxfail="${MAX_FAILURES}" \ + -n "${WORKERS}" \ + --reruns "${RERUNS}" \ + --reruns-delay 1 \ + --dist=loadscope \ + --durations=20 + fi diff --git a/.github/workflows/auto_update_price_and_context_window.yml b/.github/workflows/auto_update_price_and_context_window.yml index 98b9d868e68..60e89936219 100644 --- a/.github/workflows/auto_update_price_and_context_window.yml +++ b/.github/workflows/auto_update_price_and_context_window.yml @@ -2,18 +2,24 @@ name: Updates model_prices_and_context_window.json and Create Pull Request on: schedule: - - cron: "0 0 * * 0" # Run every Sundays at midnight + - cron: "0 0 * * 0" # Run every Sundays at midnight #- cron: "0 0 * * *" # Run daily at midnight +permissions: + contents: write + pull-requests: write + jobs: auto_update_price_and_context_window: if: github.repository == 'BerriAI/litellm' runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + with: + persist-credentials: false - name: Install Dependencies run: | - pip install aiohttp + pip install 'aiohttp==3.13.3' - name: Update JSON Data run: | python ".github/workflows/auto_update_price_and_context_window_file.py" @@ -26,4 +32,4 @@ jobs: --head auto-update-price-and-context-window-$(date +'%Y-%m-%d') \ --base main env: - GH_TOKEN: ${{ secrets.GH_TOKEN }} \ No newline at end of file + GH_TOKEN: ${{ secrets.GH_TOKEN }} diff --git a/.github/workflows/check-schema-sync.yml b/.github/workflows/check-schema-sync.yml new file mode 100644 index 00000000000..0e5e2804e60 --- /dev/null +++ b/.github/workflows/check-schema-sync.yml @@ -0,0 +1,58 @@ +name: Check Schema Sync + +on: + pull_request: + paths: + - 'schema.prisma' + - 'litellm/proxy/schema.prisma' + - 'litellm-proxy-extras/litellm_proxy_extras/schema.prisma' + +permissions: + contents: read + +jobs: + check-sync: + name: Verify schema.prisma copies match root + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - name: Checkout PR + uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + with: + persist-credentials: false + + - name: Reject symlinked schema files + run: | + for f in schema.prisma litellm/proxy/schema.prisma litellm-proxy-extras/litellm_proxy_extras/schema.prisma; do + if [ -L "$f" ]; then + echo "::error file=$f::$f is a symlink, which is not allowed" + exit 1 + fi + done + + - name: Check all schemas match root + run: | + EXIT=0 + + diff schema.prisma litellm/proxy/schema.prisma || { + echo "::error file=litellm/proxy/schema.prisma::litellm/proxy/schema.prisma differs from root schema.prisma" + EXIT=1 + } + + diff schema.prisma litellm-proxy-extras/litellm_proxy_extras/schema.prisma || { + echo "::error file=litellm-proxy-extras/litellm_proxy_extras/schema.prisma::litellm-proxy-extras/litellm_proxy_extras/schema.prisma differs from root schema.prisma" + EXIT=1 + } + + if [ "$EXIT" -ne 0 ]; then + echo "" + echo "Schema files are out of sync." + echo "The root schema.prisma is the source of truth." + echo "" + echo "To fix, run from the repo root:" + echo " cp schema.prisma litellm/proxy/schema.prisma" + echo " cp schema.prisma litellm-proxy-extras/litellm_proxy_extras/schema.prisma" + exit 1 + fi + + echo "All schema copies are in sync with root." diff --git a/.github/workflows/check_duplicate_issues.yml b/.github/workflows/check_duplicate_issues.yml index 6d11ce573eb..289d78880ad 100644 --- a/.github/workflows/check_duplicate_issues.yml +++ b/.github/workflows/check_duplicate_issues.yml @@ -12,7 +12,7 @@ jobs: contents: read steps: - name: Check for potential duplicates - uses: wow-actions/potential-duplicates@v1 + uses: wow-actions/potential-duplicates@4d4ea0352e0383859279938e255179dd1dbb67b5 # v1.1.0 with: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} label: potential-duplicate @@ -30,13 +30,14 @@ jobs: - name: Checkout close script if: github.event.action == 'opened' - uses: actions/checkout@v4 + uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 with: sparse-checkout: .github/scripts + persist-credentials: false - name: Set up Python if: github.event.action == 'opened' - uses: actions/setup-python@v5 + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version: "3.11" diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 91b88b66a1f..e86fca17c7a 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -6,8 +6,8 @@ on: pull_request: branches: [main] schedule: - # Run weekly on Sundays at 04:00 UTC - - cron: "0 4 * * 0" + # Run daily at 04:00 UTC + - cron: "0 4 * * *" concurrency: group: ${{ github.workflow }}-${{ github.ref }} @@ -38,16 +38,18 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v4 + uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + with: + persist-credentials: false - name: Initialize CodeQL - uses: github/codeql-action/init@v3 + uses: github/codeql-action/init@ebcb5b36ded6beda4ceefea6a8bc4cc885255bb3 # v3 with: languages: ${{ matrix.language }} build-mode: ${{ matrix.build-mode }} config-file: ./.github/codeql/codeql-config.yml - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v3 + uses: github/codeql-action/analyze@ebcb5b36ded6beda4ceefea6a8bc4cc885255bb3 # v3 with: category: "/language:${{ matrix.language }}" diff --git a/.github/workflows/codspeed.yml b/.github/workflows/codspeed.yml index 385b95fdaf5..52d64addea9 100644 --- a/.github/workflows/codspeed.yml +++ b/.github/workflows/codspeed.yml @@ -25,10 +25,12 @@ jobs: timeout-minutes: 15 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + with: + persist-credentials: false - name: Set up Python - uses: actions/setup-python@v5 + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version: "3.12" @@ -38,7 +40,7 @@ jobs: pip install pytest pytest-codspeed==4.3.0 - name: Run benchmarks - uses: CodSpeedHQ/action@v4 + uses: CodSpeedHQ/action@1c8ae4843586d3ba879736b7f6b7b0c990757fab # v4.12.1 with: mode: simulation run: pytest tests/benchmarks/ --codspeed diff --git a/.github/workflows/create-release.yml b/.github/workflows/create-release.yml new file mode 100644 index 00000000000..2ae01823a96 --- /dev/null +++ b/.github/workflows/create-release.yml @@ -0,0 +1,93 @@ +name: Create Release + +on: + workflow_dispatch: + inputs: + tag: + description: "Release tag (e.g. v1.83.0-stable)" + required: true + type: string + commit_hash: + description: "Full 40-char commit SHA to target" + required: true + type: string + +permissions: {} + +jobs: + release: + name: Create Release + runs-on: ubuntu-latest + permissions: + contents: write + steps: + - name: Validate inputs + env: + TAG: ${{ inputs.tag }} + COMMIT_HASH: ${{ inputs.commit_hash }} + run: | + if ! echo "${COMMIT_HASH}" | grep -qE '^[0-9a-f]{40}$'; then + echo "::error::commit_hash must be a full 40-character commit SHA" + exit 1 + fi + if ! echo "${TAG}" | grep -qE '^v[0-9]+\.[0-9]+\.[0-9]+'; then + echo "::error::tag must start with vX.Y.Z" + exit 1 + fi + + - name: Create release + env: + TAG: ${{ inputs.tag }} + COMMIT_HASH: ${{ inputs.commit_hash }} + uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1 + with: + script: | + const tag = process.env.TAG; + const commitHash = process.env.COMMIT_HASH; + + const cosignSection = [ + `## Verify Docker Image Signature`, + ``, + `All LiteLLM Docker images are signed with [cosign](https://docs.sigstore.dev/cosign/overview/). To verify the integrity of an image before deploying:`, + ``, + '```bash', + `cosign verify \\`, + ` --key https://raw.githubusercontent.com/BerriAI/litellm/${tag}/cosign.pub \\`, + ` ghcr.io/berriai/litellm:${tag}`, + '```', + ``, + `Expected output:`, + ``, + '```', + `The following checks were performed on each of these signatures:`, + ` - The cosign claims were validated`, + ` - The signatures were verified against the specified public key`, + '```', + ``, + `---`, + ``, + ].join('\n'); + + try { + const response = await github.rest.repos.createRelease({ + draft: true, + generate_release_notes: true, + target_commitish: commitHash, + name: tag, + owner: context.repo.owner, + prerelease: false, + repo: context.repo.repo, + tag_name: tag, + }); + + const updatedBody = cosignSection + (response.data.body ?? ''); + await github.rest.repos.updateRelease({ + owner: context.repo.owner, + repo: context.repo.repo, + release_id: response.data.id, + body: updatedBody, + draft: false, + }); + } catch (error) { + core.setFailed(error.message); + } diff --git a/.github/workflows/create_daily_staging_branch.yml b/.github/workflows/create_daily_staging_branch.yml index ec27aaad8ac..424d8de0a41 100644 --- a/.github/workflows/create_daily_staging_branch.yml +++ b/.github/workflows/create_daily_staging_branch.yml @@ -2,19 +2,22 @@ name: Create Daily Staging Branch on: schedule: - - cron: '0 0,12 * * *' # Runs every 12 hours at midnight and noon UTC - workflow_dispatch: # Allow manual trigger + - cron: "0 0,12 * * *" # Runs every 12 hours at midnight and noon UTC + workflow_dispatch: # Allow manual trigger jobs: create-staging-branch: if: github.repository == 'BerriAI/litellm' runs-on: ubuntu-latest + permissions: + contents: write steps: - name: Checkout repository - uses: actions/checkout@v3 + uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 with: fetch-depth: 0 + persist-credentials: false - name: Create daily staging branch env: @@ -46,12 +49,15 @@ jobs: create-internal-dev-branch: if: github.repository == 'BerriAI/litellm' runs-on: ubuntu-latest + permissions: + contents: write steps: - name: Checkout repository - uses: actions/checkout@v3 + uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 with: fetch-depth: 0 + persist-credentials: false - name: Create internal dev branch env: diff --git a/.github/workflows/helm_unit_test.yml b/.github/workflows/helm_unit_test.yml index c4b83af70a1..06836b1d1cd 100644 --- a/.github/workflows/helm_unit_test.yml +++ b/.github/workflows/helm_unit_test.yml @@ -6,22 +6,36 @@ on: branches: - main +permissions: + contents: read + jobs: unit-test: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + with: + persist-credentials: false - name: Set up Helm 3.11.1 - uses: azure/setup-helm@v1 + uses: azure/setup-helm@1a275c3b69536ee54be43f2070a358922e12c8d4 # v4.3.1 with: - version: '3.11.1' + version: "3.11.1" - name: Install Helm Unit Test Plugin run: | helm plugin install https://github.com/helm-unittest/helm-unittest --version v0.4.4 + - name: Verify Helm Unit Test Plugin integrity + run: | + EXPECTED_SHA="e251ba198448629678ff2168e1a469249d998155" + PLUGIN_DIR="$(helm env HELM_PLUGINS)/helm-unittest" + ACTUAL_SHA="$(git -C "$PLUGIN_DIR" rev-parse HEAD)" + if [ "$ACTUAL_SHA" != "$EXPECTED_SHA" ]; then + echo "::error::Helm unittest plugin checksum mismatch! Expected $EXPECTED_SHA but got $ACTUAL_SHA" + exit 1 + fi + echo "Helm unittest plugin integrity verified: $ACTUAL_SHA" - name: Run unit tests - run: - helm unittest -f 'tests/*.yaml' deploy/charts/litellm-helm \ No newline at end of file + run: helm unittest -f 'tests/*.yaml' deploy/charts/litellm-helm diff --git a/.github/workflows/interpret_load_test.py b/.github/workflows/interpret_load_test.py deleted file mode 100644 index 348ff300fff..00000000000 --- a/.github/workflows/interpret_load_test.py +++ /dev/null @@ -1,139 +0,0 @@ -import csv -import os -from github import Github - - -def interpret_results(csv_file): - with open(csv_file, newline="") as csvfile: - csvreader = csv.DictReader(csvfile) - rows = list(csvreader) - """ - in this csv reader - - Create 1 new column "Status" - - if a row has a median response time < 300 and an average response time < 300, Status = "Passed ✅" - - if a row has a median response time >= 300 or an average response time >= 300, Status = "Failed ❌" - - Order the table in this order Name, Status, Median Response Time, Average Response Time, Requests/s,Failures/s, Min Response Time, Max Response Time, all other columns - """ - - # Add a new column "Status" - for row in rows: - median_response_time = float( - row["Median Response Time"].strip().rstrip("ms") - ) - average_response_time = float( - row["Average Response Time"].strip().rstrip("s") - ) - - request_count = int(row["Request Count"]) - failure_count = int(row["Failure Count"]) - - failure_percent = round((failure_count / request_count) * 100, 2) - - # Determine status based on conditions - if ( - median_response_time < 300 - and average_response_time < 300 - and failure_percent < 5 - ): - row["Status"] = "Passed ✅" - else: - row["Status"] = "Failed ❌" - - # Construct Markdown table header - markdown_table = "| Name | Status | Median Response Time (ms) | Average Response Time (ms) | Requests/s | Failures/s | Request Count | Failure Count | Min Response Time (ms) | Max Response Time (ms) |" - markdown_table += ( - "\n| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |" - ) - - # Construct Markdown table rows - for row in rows: - markdown_table += f"\n| {row['Name']} | {row['Status']} | {row['Median Response Time']} | {row['Average Response Time']} | {row['Requests/s']} | {row['Failures/s']} | {row['Request Count']} | {row['Failure Count']} | {row['Min Response Time']} | {row['Max Response Time']} |" - print("markdown table: ", markdown_table) - return markdown_table - - -def _get_docker_run_command_stable_release(release_version): - return f""" -\n\n -## Docker Run LiteLLM Proxy - -``` -docker run \\ --e STORE_MODEL_IN_DB=True \\ --p 4000:4000 \\ -ghcr.io/berriai/litellm:litellm_stable_release_branch-{release_version} -``` - """ - - -def _get_docker_run_command(release_version): - return f""" -\n\n -## Docker Run LiteLLM Proxy - -``` -docker run \\ --e STORE_MODEL_IN_DB=True \\ --p 4000:4000 \\ -ghcr.io/berriai/litellm:main-{release_version} -``` - """ - - -def get_docker_run_command(release_version): - if "stable" in release_version: - return _get_docker_run_command_stable_release(release_version) - else: - return _get_docker_run_command(release_version) - - -if __name__ == "__main__": - return - csv_file = "load_test_stats.csv" # Change this to the path of your CSV file - markdown_table = interpret_results(csv_file) - - # Update release body with interpreted results - github_token = os.getenv("GITHUB_TOKEN") - g = Github(github_token) - repo = g.get_repo( - "BerriAI/litellm" - ) # Replace with your repository's username and name - latest_release = repo.get_latest_release() - print("got latest release: ", latest_release) - print(latest_release.title) - print(latest_release.tag_name) - - release_version = latest_release.title - - print("latest release body: ", latest_release.body) - print("markdown table: ", markdown_table) - - # check if "Load Test LiteLLM Proxy Results" exists - existing_release_body = latest_release.body - if "Load Test LiteLLM Proxy Results" in latest_release.body: - # find the "Load Test LiteLLM Proxy Results" section and delete it - start_index = latest_release.body.find("Load Test LiteLLM Proxy Results") - existing_release_body = latest_release.body[:start_index] - - docker_run_command = get_docker_run_command(release_version) - print("docker run command: ", docker_run_command) - - new_release_body = ( - existing_release_body - + docker_run_command - + "\n\n" - + "### Don't want to maintain your internal proxy? get in touch 🎉" - + "\nHosted Proxy Alpha: https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions" - + "\n\n" - + "## Load Test LiteLLM Proxy Results" - + "\n\n" - + markdown_table - ) - print("new release body: ", new_release_body) - try: - latest_release.update_release( - name=latest_release.tag_name, - message=new_release_body, - ) - except Exception as e: - print(e) diff --git a/.github/workflows/issue-keyword-labeler.yml b/.github/workflows/issue-keyword-labeler.yml index 936f90f747f..7e2693209b6 100644 --- a/.github/workflows/issue-keyword-labeler.yml +++ b/.github/workflows/issue-keyword-labeler.yml @@ -2,8 +2,8 @@ name: Issue Keyword Labeler on: issues: - types: - - opened + types: + - opened jobs: scan-and-label: @@ -13,7 +13,9 @@ jobs: contents: read steps: - name: Checkout code - uses: actions/checkout@v4 + uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + with: + persist-credentials: false - name: Scan for provider keywords id: scan @@ -24,7 +26,7 @@ jobs: - name: Ensure label exists if: steps.scan.outputs.found == 'true' - uses: actions/github-script@v7 + uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7.1.0 with: github-token: ${{ secrets.GITHUB_TOKEN }} script: | @@ -51,7 +53,7 @@ jobs: - name: Add label to the issue if: steps.scan.outputs.found == 'true' - uses: actions/github-script@v7 + uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7.1.0 with: github-token: ${{ secrets.GITHUB_TOKEN }} script: | @@ -61,4 +63,3 @@ jobs: issue_number: context.issue.number, labels: ['llm translation'] }); - diff --git a/.github/workflows/label-component.yml b/.github/workflows/label-component.yml index fd079fce6c1..e0c2fa94d8c 100644 --- a/.github/workflows/label-component.yml +++ b/.github/workflows/label-component.yml @@ -12,7 +12,7 @@ jobs: issues: write steps: - name: Add component labels - uses: actions/github-script@v7 + uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7.1.0 with: github-token: ${{ secrets.GITHUB_TOKEN }} script: | diff --git a/.github/workflows/llm-translation-testing.yml b/.github/workflows/llm-translation-testing.yml index 7fda37a66dc..922013c4b54 100644 --- a/.github/workflows/llm-translation-testing.yml +++ b/.github/workflows/llm-translation-testing.yml @@ -4,38 +4,41 @@ on: workflow_dispatch: inputs: release_candidate_tag: - description: 'Release candidate tag/version' + description: "Release candidate tag/version" required: true type: string push: tags: - - 'v*-rc*' # Triggers on release candidate tags like v1.0.0-rc1 - + - "v*-rc*" # Triggers on release candidate tags like v1.0.0-rc1 + +permissions: + contents: read + jobs: run-llm-translation-tests: runs-on: ubuntu-latest timeout-minutes: 90 - + steps: - name: Checkout code - uses: actions/checkout@v4 + uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 with: + persist-credentials: false ref: ${{ github.event.inputs.release_candidate_tag || github.ref }} - + - name: Set up Python - uses: actions/setup-python@v5 + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: - python-version: '3.11' - + python-version: "3.11" + - name: Install Poetry - uses: snok/install-poetry@v1 - with: - version: latest - virtualenvs-create: true - virtualenvs-in-project: true - - - name: Cache Poetry dependencies - uses: actions/cache@v3 + run: | + pip install 'poetry==2.3.2' + poetry config virtualenvs.create true + poetry config virtualenvs.in-project true + + - name: Restore Poetry dependencies cache + uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.0.0 with: path: | ~/.cache/pypoetry @@ -43,15 +46,15 @@ jobs: key: ${{ runner.os }}-poetry-${{ hashFiles('**/poetry.lock') }} restore-keys: | ${{ runner.os }}-poetry- - + - name: Install dependencies run: | poetry install --with dev - poetry run pip install pytest-xdist pytest-timeout - + poetry run pip install 'pytest-xdist==3.8.0' 'pytest-timeout==2.4.0' + - name: Create test results directory run: mkdir -p test-results - + - name: Run LLM Translation Tests env: OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} @@ -61,13 +64,14 @@ jobs: AZURE_API_KEY: ${{ secrets.AZURE_API_KEY }} AZURE_API_BASE: ${{ secrets.AZURE_API_BASE }} AZURE_API_VERSION: ${{ secrets.AZURE_API_VERSION }} - # Add other API keys as needed + RC_TAG: ${{ github.event.inputs.release_candidate_tag || github.ref_name }} + COMMIT_SHA: ${{ github.sha }} run: | python .github/workflows/run_llm_translation_tests.py \ - --tag "${{ github.event.inputs.release_candidate_tag || github.ref_name }}" \ - --commit "${{ github.sha }}" \ + --tag "$RC_TAG" \ + --commit "$COMMIT_SHA" \ || true # Continue even if tests fail - + - name: Display test summary if: always() run: | @@ -79,9 +83,9 @@ jobs: else echo "Warning: Test report was not generated" fi - + - name: Upload test artifacts - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 if: always() with: name: LLM-Translation-Artifact-${{ github.event.inputs.release_candidate_tag || github.ref_name }} diff --git a/.github/workflows/load_test.yml b/.github/workflows/load_test.yml deleted file mode 100644 index cdaffa328c9..00000000000 --- a/.github/workflows/load_test.yml +++ /dev/null @@ -1,59 +0,0 @@ -name: Test Locust Load Test - -on: - workflow_run: - workflows: ["Build, Publish LiteLLM Docker Image. New Release"] - types: - - completed - workflow_dispatch: - -jobs: - build: - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v1 - - name: Setup Python - uses: actions/setup-python@v2 - with: - python-version: '3.x' - - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install PyGithub - - name: re-deploy proxy - run: | - echo "Current working directory: $PWD" - ls - python ".github/workflows/redeploy_proxy.py" - env: - LOAD_TEST_REDEPLOY_URL1: ${{ secrets.LOAD_TEST_REDEPLOY_URL1 }} - LOAD_TEST_REDEPLOY_URL2: ${{ secrets.LOAD_TEST_REDEPLOY_URL2 }} - working-directory: ${{ github.workspace }} - - name: Run Load Test - id: locust_run - uses: BerriAI/locust-github-action@master - with: - LOCUSTFILE: ".github/workflows/locustfile.py" - URL: "https://post-release-load-test-proxy.onrender.com/" - USERS: "20" - RATE: "20" - RUNTIME: "300s" - - name: Process Load Test Stats - run: | - echo "Current working directory: $PWD" - ls - python ".github/workflows/interpret_load_test.py" - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - working-directory: ${{ github.workspace }} - - name: Upload CSV as Asset to Latest Release - uses: xresloader/upload-to-github-release@v1 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - file: "load_test_stats.csv;load_test.html" - update_latest_release: true - tag_name: "load-test" - overwrite: true \ No newline at end of file diff --git a/.github/workflows/locustfile.py b/.github/workflows/locustfile.py deleted file mode 100644 index 36dbeee9c48..00000000000 --- a/.github/workflows/locustfile.py +++ /dev/null @@ -1,28 +0,0 @@ -from locust import HttpUser, task, between - - -class MyUser(HttpUser): - wait_time = between(1, 5) - - @task - def chat_completion(self): - headers = { - "Content-Type": "application/json", - "Authorization": "Bearer sk-8N1tLOOyH8TIxwOLahhIVg", - # Include any additional headers you may need for authentication, etc. - } - - # Customize the payload with "model" and "messages" keys - payload = { - "model": "fake-openai-endpoint", - "messages": [ - {"role": "system", "content": "You are a chat bot."}, - {"role": "user", "content": "Hello, how are you?"}, - ], - # Add more data as necessary - } - - # Make a POST request to the "chat/completions" endpoint - response = self.client.post("chat/completions", json=payload, headers=headers) - - # Print or log the response if needed diff --git a/.github/workflows/publish_to_pypi.yml b/.github/workflows/publish_to_pypi.yml new file mode 100644 index 00000000000..8f675bb3075 --- /dev/null +++ b/.github/workflows/publish_to_pypi.yml @@ -0,0 +1,136 @@ +name: Publish to PyPI + +on: + workflow_dispatch: + +jobs: + preflight-checks: + name: Preflight Checks + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + contents: read + # No environment — read-only checks, no approval needed + outputs: + needs_publish: ${{ steps.check-litellm.outputs.needs_publish }} + version: ${{ steps.check-litellm.outputs.version }} + + steps: + - name: Checkout repo + uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + + - name: Set up Python + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: "3.12" + + - name: Check litellm version on PyPI + id: check-litellm + run: | + VERSION=$(grep -m1 '^version' pyproject.toml | sed 's/version = "\(.*\)"/\1/') + echo "version=$VERSION" >> "$GITHUB_OUTPUT" + echo "Checking if litellm $VERSION exists on PyPI..." + + HTTP_STATUS=$(curl -s -o /dev/null -w "%{http_code}" "https://pypi.org/pypi/litellm/$VERSION/json") + if [ "$HTTP_STATUS" = "200" ]; then + echo "litellm $VERSION already exists on PyPI. Skipping publish." + echo "needs_publish=false" >> "$GITHUB_OUTPUT" + else + echo "litellm $VERSION not found on PyPI. Publish needed." + echo "needs_publish=true" >> "$GITHUB_OUTPUT" + fi + + - name: Sanity check proxy-extras version + run: | + # Read pinned version from requirements.txt + REQ_VERSION=$(grep -oP 'litellm-proxy-extras==\K[0-9.]+' requirements.txt) + if [ -z "$REQ_VERSION" ]; then + echo "::error::Could not find litellm-proxy-extras version in requirements.txt" + exit 1 + fi + echo "requirements.txt pins litellm-proxy-extras==$REQ_VERSION" + + # Read pinned version from pyproject.toml dependency + PYPROJECT_VERSION=$(python3 -c " + import re + with open('pyproject.toml') as f: + content = f.read() + match = re.search(r'litellm-proxy-extras\s*=\s*\{version\s*=\s*\"([^\"]+)\"', content) + if match: + print(match.group(1).lstrip('^~>=')) + else: + import sys + print('::error::Could not find litellm-proxy-extras dependency in pyproject.toml', file=sys.stderr) + sys.exit(1) + ") + echo "pyproject.toml pins litellm-proxy-extras version: $PYPROJECT_VERSION" + + # Check that both pinned versions match + if [ "$REQ_VERSION" != "$PYPROJECT_VERSION" ]; then + echo "::error::Version mismatch: requirements.txt has $REQ_VERSION but pyproject.toml has $PYPROJECT_VERSION" + exit 1 + fi + + # Check that the pinned version exists on PyPI + echo "Checking if litellm-proxy-extras $REQ_VERSION exists on PyPI..." + HTTP_STATUS=$(curl -s -o /dev/null -w "%{http_code}" "https://pypi.org/pypi/litellm-proxy-extras/$REQ_VERSION/json") + if [ "$HTTP_STATUS" != "200" ]; then + echo "::error::litellm-proxy-extras $REQ_VERSION is not published on PyPI yet. Publish it before releasing litellm." + exit 1 + fi + echo "litellm-proxy-extras $REQ_VERSION exists on PyPI. Sanity check passed." + + publish-litellm: + name: Publish litellm to PyPI + needs: preflight-checks + if: needs.preflight-checks.outputs.needs_publish == 'true' + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + id-token: write + contents: read + environment: pypi-publish + + steps: + - name: Checkout repo + uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + + - name: Set up Python + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: "3.12" + + - name: Copy model prices backup + run: cp model_prices_and_context_window.json litellm/model_prices_and_context_window_backup.json + + - name: Install build tools + run: python -m pip install --upgrade pip build==1.4.2 + + - name: Build package + run: | + rm -rf build dist + python -m build + + - name: Verify build artifacts + env: + EXPECTED_VERSION: ${{ needs.preflight-checks.outputs.version }} + run: | + echo "Contents of dist/:" + ls -la dist/ + # Ensure we have both sdist and wheel + ls dist/*.tar.gz + ls dist/*.whl + # Verify built version matches expected + ls dist/ | grep -q "litellm-${EXPECTED_VERSION}" || { + echo "::error::Built artifacts do not match expected version $EXPECTED_VERSION" + ls dist/ + exit 1 + } + + - name: Validate package metadata + run: | + pip install twine==6.2.0 + twine check dist/* + + - name: Publish to PyPI + uses: pypa/gh-action-pypi-publish@ed0c53931b1dc9bd32cbe73a98c7f6766f8a527e # v1.13.0 diff --git a/.github/workflows/read_pyproject_version.yml b/.github/workflows/read_pyproject_version.yml index 8f6310f935b..04b4a38ce19 100644 --- a/.github/workflows/read_pyproject_version.yml +++ b/.github/workflows/read_pyproject_version.yml @@ -3,7 +3,10 @@ name: Read Version from pyproject.toml on: push: branches: - - main # Change this to the default branch of your repository + - main # Change this to the default branch of your repository + +permissions: + contents: read jobs: read-version: @@ -11,20 +14,14 @@ jobs: steps: - name: Checkout code - uses: actions/checkout@v2 - - - name: Set up Python - uses: actions/setup-python@v2 + uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 with: - python-version: 3.8 # Adjust the Python version as needed - - - name: Install dependencies - run: pip install toml + persist-credentials: false - name: Read version from pyproject.toml id: read-version run: | - version=$(python -c 'import toml; print(toml.load("pyproject.toml")["tool"]["commitizen"]["version"])') + version=$(grep -m1 '^version' pyproject.toml | sed 's/version = "\(.*\)"/\1/') printf "LITELLM_VERSION=%s" "$version" >> $GITHUB_ENV - name: Display version diff --git a/.github/workflows/redeploy_proxy.py b/.github/workflows/redeploy_proxy.py deleted file mode 100644 index ed46bef73a2..00000000000 --- a/.github/workflows/redeploy_proxy.py +++ /dev/null @@ -1,20 +0,0 @@ -""" - -redeploy_proxy.py -""" - -import os -import requests -import time - -# send a get request to this endpoint -deploy_hook1 = os.getenv("LOAD_TEST_REDEPLOY_URL1") -response = requests.get(deploy_hook1, timeout=20) - - -deploy_hook2 = os.getenv("LOAD_TEST_REDEPLOY_URL2") -response = requests.get(deploy_hook2, timeout=20) - -print("SENT GET REQUESTS to re-deploy proxy") -print("sleeeping.... for 60s") -time.sleep(60) diff --git a/.github/workflows/regenerate-poetry-lock.yml b/.github/workflows/regenerate-poetry-lock.yml deleted file mode 100644 index c0844f1c705..00000000000 --- a/.github/workflows/regenerate-poetry-lock.yml +++ /dev/null @@ -1,80 +0,0 @@ -name: Regenerate poetry.lock - -# Runs whenever pyproject.toml is merged into main (the most common cause of -# the "pyproject.toml changed significantly since poetry.lock was last generated" -# CI failure). Can also be triggered manually. -on: - push: - branches: - - main - paths: - - pyproject.toml - workflow_dispatch: - -permissions: - contents: write # needed to push the auto/regenerate-poetry-lock-* branch - pull-requests: write # needed to open the PR and enable auto-merge - -jobs: - regenerate-lock: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: "3.11" - - - name: Install Poetry - run: pip install poetry - - - name: Regenerate poetry.lock - run: poetry lock - - - name: Check whether poetry.lock actually changed - id: diff - run: | - if git diff --quiet poetry.lock; then - echo "changed=false" >> "$GITHUB_OUTPUT" - else - echo "changed=true" >> "$GITHUB_OUTPUT" - fi - - - name: Open PR with the refreshed lock file - if: steps.diff.outputs.changed == 'true' - id: open-pr - run: | - BRANCH="auto/regenerate-poetry-lock-$(date +'%Y%m%d%H%M%S')" - git config user.name "github-actions[bot]" - git config user.email "github-actions[bot]@users.noreply.github.com" - git checkout -b "$BRANCH" - git add poetry.lock - git commit -m "chore: regenerate poetry.lock to match pyproject.toml" - git push -f origin "$BRANCH" - - cat > /tmp/pr-body.md << 'BODY' - Automated regeneration of `poetry.lock` after `pyproject.toml` was updated on `main`. - - Fixes the recurring CI failure: - ``` - pyproject.toml changed significantly since poetry.lock was last generated. - Run `poetry lock` to fix the lock file. - ``` - BODY - - PR_URL=$(gh pr create \ - --title "chore: regenerate poetry.lock to match pyproject.toml" \ - --body-file /tmp/pr-body.md \ - --head "$BRANCH" \ - --base main) - echo "pr_url=$PR_URL" >> "$GITHUB_OUTPUT" - env: - GH_TOKEN: ${{ github.token }} - - - name: Enable auto-merge - if: steps.diff.outputs.changed == 'true' - run: | - gh pr merge "${{ steps.open-pr.outputs.pr_url }}" --auto --squash - env: - GH_TOKEN: ${{ github.token }} diff --git a/.github/workflows/run_observatory_tests.yml b/.github/workflows/run_observatory_tests.yml index d343098ed32..a25b96766d7 100644 --- a/.github/workflows/run_observatory_tests.yml +++ b/.github/workflows/run_observatory_tests.yml @@ -33,7 +33,9 @@ jobs: timeout-minutes: 30 steps: - name: Checkout repository - uses: actions/checkout@v4 + uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + with: + persist-credentials: false - name: Validate tag input env: @@ -49,11 +51,12 @@ jobs: TAG: ${{ inputs.tag }} AZURE_API_KEY: ${{ secrets.AZURE_API_KEY }} AZURE_API_BASE: ${{ secrets.AZURE_API_BASE }} + WORKSPACE: ${{ github.workspace }} run: | docker run -d \ --name litellm-rc \ -p 4000:4000 \ - -v "${{ github.workspace }}/.github/observatory/litellm_config.yaml:/app/config.yaml" \ + -v "${WORKSPACE}/.github/observatory/litellm_config.yaml:/app/config.yaml" \ -e LITELLM_MASTER_KEY="${LITELLM_MASTER_KEY}" \ -e AZURE_API_KEY="${AZURE_API_KEY}" \ -e AZURE_API_BASE="${AZURE_API_BASE}" \ @@ -77,8 +80,9 @@ jobs: - name: Start cloudflared tunnel run: | - # Install cloudflared + # Install cloudflared (pinned version + checksum) curl -sL https://github.com/cloudflare/cloudflared/releases/download/2025.2.1/cloudflared-linux-amd64 -o /usr/local/bin/cloudflared + echo "afdfadd1ef552e66bffc35246fe30a9bd578356d2d386de95585ccfc432472b8 /usr/local/bin/cloudflared" | sha256sum -c - chmod +x /usr/local/bin/cloudflared # Start a quick tunnel (no account needed) and capture the URL @@ -103,11 +107,11 @@ jobs: - name: Verify tunnel connectivity run: | - echo "Testing tunnel at ${{ env.TUNNEL_URL }}..." + echo "Testing tunnel at ${TUNNEL_URL}..." # Quick tunnels need time for DNS propagation; retry to avoid # transient NXDOMAIN (curl exit code 6) on first attempt. for i in $(seq 1 10); do - if curl -sf "${{ env.TUNNEL_URL }}/health/liveliness" > /dev/null 2>&1; then + if curl -sf "${TUNNEL_URL}/health/liveliness" > /dev/null 2>&1; then echo "Tunnel is working (attempt $i)" exit 0 fi @@ -221,5 +225,5 @@ jobs: - name: Cleanup if: always() run: | - kill "${{ env.CLOUDFLARED_PID }}" 2>/dev/null || true + kill "$CLOUDFLARED_PID" 2>/dev/null || true docker rm -f litellm-rc 2>/dev/null || true diff --git a/.github/workflows/scan_duplicate_issues.yml b/.github/workflows/scan_duplicate_issues.yml index 06e8f453a8c..222ff11f304 100644 --- a/.github/workflows/scan_duplicate_issues.yml +++ b/.github/workflows/scan_duplicate_issues.yml @@ -21,14 +21,15 @@ jobs: contents: read steps: - name: Checkout scripts - uses: actions/checkout@v4 + uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 with: sparse-checkout: .github/scripts + persist-credentials: false - name: Set up Python - uses: actions/setup-python@v5 + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: - python-version: "3.11" + python-version: "3.13" - name: Scan for duplicate issues env: diff --git a/.github/workflows/scorecard.yml b/.github/workflows/scorecard.yml new file mode 100644 index 00000000000..7cd12bb219c --- /dev/null +++ b/.github/workflows/scorecard.yml @@ -0,0 +1,47 @@ +name: Scorecard supply-chain security + +on: + branch_protection_rule: + schedule: + - cron: '27 12 * * 4' + push: + branches: ["main"] + +permissions: read-all + +jobs: + analysis: + name: Scorecard analysis + runs-on: ubuntu-latest + if: github.event.repository.default_branch == github.ref_name + permissions: + security-events: write + id-token: write + # Uncomment for private repos if needed: + # contents: read + # actions: read + + steps: + - name: Checkout code + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + persist-credentials: false + + - name: Run analysis + uses: ossf/scorecard-action@f49aabe0b5af0936a0987cfb85d86b75731b0186 # v2.4.1 + with: + results_file: results.sarif + results_format: sarif + publish_results: true + + - name: Upload artifact + uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1 + with: + name: SARIF file + path: results.sarif + retention-days: 5 + + - name: Upload to code scanning + uses: github/codeql-action/upload-sarif@c10b806170c8ee63ea24152429041b5624f0baf5 # v4.35.1 + with: + sarif_file: results.sarif diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index 4945655b614..c905bb12312 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -2,20 +2,24 @@ name: "Stale Issue Management" on: schedule: - - cron: '0 0 * * *' # Runs daily at midnight UTC + - cron: "0 0 * * *" # Runs daily at midnight UTC workflow_dispatch: +permissions: + issues: write + pull-requests: write + jobs: stale: if: github.repository == 'BerriAI/litellm' runs-on: ubuntu-latest steps: - - uses: actions/stale@v8 + - uses: actions/stale@1160a2240286f5da8ec72b1c0816ce2481aabf84 # v8 with: repo-token: "${{ secrets.GITHUB_TOKEN }}" stale-issue-message: "This issue has been automatically marked as stale because it has not had recent activity. It will be closed if no further activity occurs." stale-pr-message: "This pull request has been automatically marked as stale because it has not had recent activity. It will be closed if no further activity occurs." - days-before-stale: 90 # Revert to 60 days - days-before-close: 7 # Revert to 7 days + days-before-stale: 90 # Revert to 60 days + days-before-close: 7 # Revert to 7 days stale-issue-label: "stale" - operations-per-run: 1000 \ No newline at end of file + operations-per-run: 1000 diff --git a/.github/workflows/sync-schema.yml b/.github/workflows/sync-schema.yml new file mode 100644 index 00000000000..72a5c56293e --- /dev/null +++ b/.github/workflows/sync-schema.yml @@ -0,0 +1,73 @@ +name: Sync schema.prisma copies + +on: + pull_request: + paths: + - 'schema.prisma' + +# Scoped to ONLY the permissions needed: +# - contents:write to push the sync commit to the PR branch +# - pull-requests:read is implicit (needed to check out the PR) +permissions: + contents: write + +jobs: + sync: + name: Copy root schema to proxy and proxy-extras + runs-on: ubuntu-latest + timeout-minutes: 5 + # Only run on PRs from branches in THIS repo (not forks). + # Fork PRs cannot push back to the head branch with GITHUB_TOKEN, + # and pull_request events from forks have read-only tokens anyway. + # Also reject PRs from branches named after protected branches to + # prevent pushing directly to main/master. + if: >- + github.event.pull_request.head.repo.full_name == github.repository + && github.head_ref != 'main' + && github.head_ref != 'master' + steps: + - name: Checkout PR branch by SHA + uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + with: + # Use the merge commit SHA for safety — github.head_ref is an + # attacker-controlled string (the branch name) and could contain + # unusual characters that cause unexpected git behavior. + ref: ${{ github.event.pull_request.head.sha }} + persist-credentials: true # needed for git push + + - name: Reject symlinked schema files + run: | + for f in schema.prisma litellm/proxy/schema.prisma litellm-proxy-extras/litellm_proxy_extras/schema.prisma; do + if [ -L "$f" ]; then + echo "::error file=$f::$f is a symlink, which is not allowed" + exit 1 + fi + done + + - name: Copy root schema to other locations + run: | + cp schema.prisma litellm/proxy/schema.prisma + cp schema.prisma litellm-proxy-extras/litellm_proxy_extras/schema.prisma + + - name: Check for changes + id: diff + run: | + if git diff --quiet -- litellm/proxy/schema.prisma litellm-proxy-extras/litellm_proxy_extras/schema.prisma; then + echo "changed=false" >> "$GITHUB_OUTPUT" + echo "Schemas already in sync. Nothing to do." + else + echo "changed=true" >> "$GITHUB_OUTPUT" + echo "Schema copies need updating." + fi + + - name: Commit synced schemas + if: steps.diff.outputs.changed == 'true' + run: | + # Push to the PR's head branch (need the branch name for git push). + # We checked out by SHA above for safety, so configure the push target explicitly. + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git checkout -B "$GITHUB_HEAD_REF" + git add -- litellm/proxy/schema.prisma litellm-proxy-extras/litellm_proxy_extras/schema.prisma + git commit -m "chore: sync schema.prisma copies from root" + git push origin "HEAD:$GITHUB_HEAD_REF" diff --git a/.github/workflows/test-linting.yml b/.github/workflows/test-linting.yml index 4cedb8b5bae..5bb85716a17 100644 --- a/.github/workflows/test-linting.yml +++ b/.github/workflows/test-linting.yml @@ -2,7 +2,10 @@ name: LiteLLM Linting on: pull_request: - branches: [ main ] + branches: [main] + +permissions: + contents: read jobs: lint: @@ -10,72 +13,73 @@ jobs: timeout-minutes: 5 steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - clean: true + - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + with: + fetch-depth: 0 + clean: true + persist-credentials: false - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: '3.12' + - name: Set up Python + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: "3.12" - - name: Install Poetry - uses: snok/install-poetry@v1 + - name: Install Poetry + run: pip install 'poetry==2.3.2' - - name: Clean Python cache - run: | - find . -type d -name "__pycache__" -exec rm -rf {} + || true - find . -name "*.pyc" -delete || true + - name: Clean Python cache + run: | + find . -type d -name "__pycache__" -exec rm -rf {} + || true + find . -name "*.pyc" -delete || true - - name: Check poetry.lock is up to date - run: | - poetry check --lock || (echo "❌ poetry.lock is out of sync with pyproject.toml. Run 'poetry lock' locally and commit the result." && exit 1) + - name: Check poetry.lock is up to date + run: | + poetry check --lock || (echo "❌ poetry.lock is out of sync with pyproject.toml. Run 'poetry lock' locally and commit the result." && exit 1) - - name: Install dependencies - run: | - poetry install --with dev + - name: Install dependencies + run: | + poetry install --with dev - - name: Check Black formatting - run: | - cd litellm - poetry run black --check --exclude '/enterprise/' . - cd .. + - name: Check Black formatting + run: | + cd litellm + poetry run black --check --exclude '/enterprise/' . + cd .. - - name: Debug - Check file state - run: | - echo "Current branch:" - git branch --show-current - echo "Last 3 commits:" - git log --oneline -3 - echo "File content around line 43:" - head -50 litellm/litellm_core_utils/custom_logger_registry.py | tail -10 - - - name: Run Ruff linting - run: | - cd litellm - poetry run ruff check . - cd .. + - name: Debug - Check file state + run: | + echo "Current branch:" + git branch --show-current + echo "Last 3 commits:" + git log --oneline -3 + echo "File content around line 43:" + head -50 litellm/litellm_core_utils/custom_logger_registry.py | tail -10 - - name: Print OpenAI version - run: | - poetry run python -c "import openai; print(f'OpenAI version: {openai.__version__}')" + - name: Run Ruff linting + run: | + cd litellm + poetry run ruff check . + cd .. - - name: Run MyPy type checking - run: | - cd litellm - poetry run mypy . - cd .. + - name: Print OpenAI version + run: | + poetry run python -c "import openai; print(f'OpenAI version: {openai.__version__}')" - - name: Check for circular imports - run: | - cd litellm - poetry run python ../tests/documentation_tests/test_circular_imports.py - cd .. + - name: Run MyPy type checking + run: | + cd litellm + poetry run mypy . + cd .. - - name: Check import safety - run: | - poetry run python -c "from litellm import *" || (echo '🚨 import failed, this means you introduced unprotected imports! 🚨'; exit 1) + - name: Check for circular imports + run: | + cd litellm + poetry run python ../tests/documentation_tests/test_circular_imports.py + cd .. + + - name: Check import safety + run: | + poetry run python -c "from litellm import *" || (echo '🚨 import failed, this means you introduced unprotected imports! 🚨'; exit 1) secret-scan: runs-on: ubuntu-latest @@ -84,27 +88,28 @@ jobs: contents: read steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 + - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + with: + fetch-depth: 0 + persist-credentials: false - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: '3.12' + - name: Set up Python + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: "3.12" - - name: Run secret scan test - run: | - pip install pytest - pytest tests/litellm/test_no_hardcoded_secrets.py -v + - name: Run secret scan test + run: | + pip install 'pytest==9.0.2' + pytest tests/litellm/test_no_hardcoded_secrets.py -v - - name: Run ggshield secret scan - env: - GITGUARDIAN_API_KEY: ${{ secrets.GITGUARDIAN_API_KEY }} - run: | - if [ -n "$GITGUARDIAN_API_KEY" ]; then - pip install ggshield - ggshield secret scan repo . - else - echo "GITGUARDIAN_API_KEY not set, skipping ggshield scan" - fi + - name: Run ggshield secret scan + env: + GITGUARDIAN_API_KEY: ${{ secrets.GITGUARDIAN_API_KEY }} + run: | + if [ -n "$GITGUARDIAN_API_KEY" ]; then + pip install 'ggshield==1.48.0' + ggshield secret scan repo . + else + echo "GITGUARDIAN_API_KEY not set, skipping ggshield scan" + fi diff --git a/.github/workflows/test-litellm-matrix.yml b/.github/workflows/test-litellm-matrix.yml index d0ac28ab41a..860d25636c5 100644 --- a/.github/workflows/test-litellm-matrix.yml +++ b/.github/workflows/test-litellm-matrix.yml @@ -4,6 +4,9 @@ on: pull_request: branches: [main] +permissions: + contents: read + # Cancel in-progress runs for the same PR concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} @@ -12,7 +15,7 @@ concurrency: jobs: test: runs-on: ubuntu-latest - timeout-minutes: 20 # Increased from 15 to 20 + timeout-minutes: 20 # Increased from 15 to 20 strategy: fail-fast: false matrix: @@ -43,7 +46,7 @@ jobs: - name: "integrations" path: "tests/test_litellm/integrations" workers: 2 - reruns: 3 # Integration tests tend to be flakier + reruns: 3 # Integration tests tend to be flakier - name: "core-utils" path: "tests/test_litellm/litellm_core_utils" workers: 2 @@ -117,18 +120,20 @@ jobs: name: test (${{ matrix.test-group.name }}) steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + with: + persist-credentials: false - name: Set up Python - uses: actions/setup-python@v5 + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version: "3.12" - name: Install Poetry - uses: snok/install-poetry@v1 + run: pip install 'poetry==2.3.2' - name: Cache Poetry dependencies - uses: actions/cache@v4 + uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.0.0 with: path: | ~/.cache/pypoetry @@ -144,14 +149,17 @@ jobs: poetry install --with dev,proxy-dev --extras "proxy semantic-router" # pytest-rerunfailures and pytest-xdist are in pyproject.toml dev dependencies poetry run pip install google-genai==1.22.0 \ - google-cloud-aiplatform>=1.38 fastapi-offline==1.7.3 python-multipart==0.0.22 openapi-core + google-cloud-aiplatform==1.115.0 fastapi-offline==1.7.3 python-multipart==0.0.22 openapi-core==0.23.0 - name: Setup litellm-enterprise run: | poetry run pip install --force-reinstall --no-deps -e enterprise/ - name: Generate Prisma client + env: + PRISMA_BINARY_CACHE_DIR: ${{ runner.temp }}/prisma-cache run: | + poetry run pip install nodejs-wheel-binaries==24.13.1 poetry run prisma generate --schema litellm/proxy/schema.prisma - name: Run tests - ${{ matrix.test-group.name }} diff --git a/.github/workflows/test-litellm-ui-build.yml b/.github/workflows/test-litellm-ui-build.yml index b0a8b648a44..bef568298e0 100644 --- a/.github/workflows/test-litellm-ui-build.yml +++ b/.github/workflows/test-litellm-ui-build.yml @@ -16,17 +16,19 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v4 + uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + with: + persist-credentials: false - name: Setup Node.js - uses: actions/setup-node@v4 + uses: actions/setup-node@a0853c24544627f65ddf259abe73b1d18a591444 # v5.0 with: node-version: "20" cache: "npm" cache-dependency-path: ui/litellm-dashboard/package-lock.json - name: Install dependencies - run: npm install + run: npm ci - name: Build run: npm run build diff --git a/.github/workflows/test-litellm.yml b/.github/workflows/test-litellm.yml index 3f8369df926..0c040b3ebe7 100644 --- a/.github/workflows/test-litellm.yml +++ b/.github/workflows/test-litellm.yml @@ -4,45 +4,50 @@ name: LiteLLM Mock Tests (folder - tests/test_litellm) # the same tests in parallel across 10 jobs for faster CI times. # Kept for manual debugging only. on: - workflow_dispatch: # Manual trigger only + workflow_dispatch: # Manual trigger only # pull_request: # branches: [ main ] +permissions: + contents: read + jobs: test: runs-on: ubuntu-latest timeout-minutes: 25 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + with: + persist-credentials: false - - name: Thank You Message - run: | - echo "### 🙏 Thank you for contributing to LiteLLM!" >> $GITHUB_STEP_SUMMARY - echo "Your PR is being tested now. We appreciate your help in making LiteLLM better!" >> $GITHUB_STEP_SUMMARY + - name: Thank You Message + run: | + echo "### 🙏 Thank you for contributing to LiteLLM!" >> $GITHUB_STEP_SUMMARY + echo "Your PR is being tested now. We appreciate your help in making LiteLLM better!" >> $GITHUB_STEP_SUMMARY - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: '3.12' + - name: Set up Python + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: "3.12" - - name: Install Poetry - uses: snok/install-poetry@v1 + - name: Install Poetry + run: pip install 'poetry==2.3.2' - - name: Install dependencies - run: | - poetry lock - poetry install --with dev,proxy-dev --extras "proxy semantic-router" - poetry run pip install "pytest-retry==1.6.3" - poetry run pip install pytest-xdist - poetry run pip install "google-genai==1.22.0" - poetry run pip install "google-cloud-aiplatform>=1.38" - poetry run pip install "fastapi-offline==1.7.3" - poetry run pip install "python-multipart>=0.0.20" - poetry run pip install "openapi-core" - - name: Setup litellm-enterprise as local package - run: | - poetry run pip install --force-reinstall --no-deps -e enterprise/ - - name: Run tests - run: | - poetry run pytest tests/test_litellm --tb=short -vv --maxfail=10 -n 4 --durations=50 + - name: Install dependencies + run: | + poetry lock + poetry install --with dev,proxy-dev --extras "proxy semantic-router" + poetry run pip install "pytest-retry==1.6.3" + poetry run pip install 'pytest-xdist==3.8.0' + poetry run pip install "google-genai==1.22.0" + poetry run pip install "google-cloud-aiplatform==1.115.0" + poetry run pip install "fastapi-offline==1.7.3" + poetry run pip install "python-multipart==0.0.22" + poetry run pip install "openapi-core==0.23.0" + - name: Setup litellm-enterprise as local package + run: | + poetry run pip install --force-reinstall --no-deps -e enterprise/ + - name: Run tests + run: | + poetry run pytest tests/test_litellm --tb=short -vv --maxfail=10 -n 4 --durations=50 diff --git a/.github/workflows/test-mcp.yml b/.github/workflows/test-mcp.yml index 2e32aae7680..1b228ab76bb 100644 --- a/.github/workflows/test-mcp.yml +++ b/.github/workflows/test-mcp.yml @@ -2,7 +2,10 @@ name: LiteLLM MCP Tests (folder - tests/mcp_tests) on: pull_request: - branches: [ main ] + branches: [main] + +permissions: + contents: read jobs: test: @@ -10,38 +13,40 @@ jobs: timeout-minutes: 25 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + with: + persist-credentials: false - - name: Thank You Message - run: | - echo "### 🙏 Thank you for contributing to LiteLLM!" >> $GITHUB_STEP_SUMMARY - echo "Your PR is being tested now. We appreciate your help in making LiteLLM better!" >> $GITHUB_STEP_SUMMARY + - name: Thank You Message + run: | + echo "### 🙏 Thank you for contributing to LiteLLM!" >> $GITHUB_STEP_SUMMARY + echo "Your PR is being tested now. We appreciate your help in making LiteLLM better!" >> $GITHUB_STEP_SUMMARY - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: '3.12' + - name: Set up Python + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: "3.12" - - name: Install Poetry - uses: snok/install-poetry@v1 + - name: Install Poetry + run: pip install 'poetry==2.3.2' - - name: Install dependencies - run: | - poetry lock - poetry install --with dev,proxy-dev --extras "proxy semantic-router" - poetry run pip install "pytest==7.3.1" - poetry run pip install "pytest-retry==1.6.3" - poetry run pip install "pytest-cov==5.0.0" - poetry run pip install "pytest-asyncio==0.21.1" - poetry run pip install "respx==0.22.0" - poetry run pip install "pydantic==2.11.0" - poetry run pip install "mcp==1.25.0" - poetry run pip install pytest-xdist + - name: Install dependencies + run: | + poetry lock + poetry install --with dev,proxy-dev --extras "proxy semantic-router" + poetry run pip install "pytest==7.3.1" + poetry run pip install "pytest-retry==1.6.3" + poetry run pip install "pytest-cov==5.0.0" + poetry run pip install "pytest-asyncio==0.21.1" + poetry run pip install "respx==0.22.0" + poetry run pip install "pydantic==2.11.0" + poetry run pip install "mcp==1.25.0" + poetry run pip install 'pytest-xdist==3.8.0' - - name: Setup litellm-enterprise as local package - run: | - poetry run pip install --force-reinstall --no-deps -e enterprise/ + - name: Setup litellm-enterprise as local package + run: | + poetry run pip install --force-reinstall --no-deps -e enterprise/ - - name: Run MCP tests - run: | - poetry run pytest tests/mcp_tests -x -vv -n 4 --cov=litellm --cov-report=xml --durations=5 + - name: Run MCP tests + run: | + poetry run pytest tests/mcp_tests -x -vv -n 4 --cov=litellm --cov-report=xml --durations=5 diff --git a/.github/workflows/test-model-map.yaml b/.github/workflows/test-model-map.yaml index ae5ac402e23..429f9e1ce0a 100644 --- a/.github/workflows/test-model-map.yaml +++ b/.github/workflows/test-model-map.yaml @@ -2,13 +2,18 @@ name: Validate model_prices_and_context_window.json on: pull_request: - branches: [ main ] + branches: [main] + +permissions: + contents: read jobs: validate-model-prices-json: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + with: + persist-credentials: false - name: Validate model_prices_and_context_window.json run: | diff --git a/.github/workflows/test-proxy-e2e-azure-batches.yml b/.github/workflows/test-proxy-e2e-azure-batches.yml index 4d74f3db0ac..7cbbe0b338f 100644 --- a/.github/workflows/test-proxy-e2e-azure-batches.yml +++ b/.github/workflows/test-proxy-e2e-azure-batches.yml @@ -9,6 +9,9 @@ concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} cancel-in-progress: true +permissions: + contents: read + jobs: proxy_e2e_azure_batches_tests: runs-on: ubuntu-latest @@ -30,18 +33,20 @@ jobs: --health-retries 5 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + with: + persist-credentials: false - name: Set up Python - uses: actions/setup-python@v5 + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version: "3.12" - name: Install Poetry - uses: snok/install-poetry@v1 + run: pip install 'poetry==2.3.2' - name: Cache Poetry dependencies - uses: actions/cache@v4 + uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.0.0 with: path: | ~/.cache/pypoetry @@ -56,14 +61,17 @@ jobs: run: | poetry config virtualenvs.in-project true poetry install --with dev,proxy-dev --extras "proxy" - poetry run pip install psycopg2-binary uvicorn fastapi httpx tenacity + poetry run pip install psycopg2-binary==2.9.11 uvicorn==0.42.0 fastapi==0.135.2 httpx==0.28.1 tenacity==9.1.4 - name: Setup litellm-enterprise run: | poetry run pip install --force-reinstall --no-deps -e enterprise/ - name: Generate Prisma client + env: + PRISMA_BINARY_CACHE_DIR: ${{ runner.temp }}/prisma-cache run: | + poetry run pip install nodejs-wheel-binaries==24.13.1 poetry run prisma generate --schema litellm/proxy/schema.prisma - name: Run Prisma migrations @@ -87,4 +95,3 @@ jobs: --tb=short \ --maxfail=3 \ --durations=10 - diff --git a/.github/workflows/test-unit-caching-redis.yml b/.github/workflows/test-unit-caching-redis.yml new file mode 100644 index 00000000000..ca274324f2f --- /dev/null +++ b/.github/workflows/test-unit-caching-redis.yml @@ -0,0 +1,38 @@ +name: "Unit Tests: Caching (Redis)" + +# Uses cloud Redis credentials — only runs on trusted branches, not PRs. +# This prevents external PRs from accessing Redis credentials. +on: + push: + branches: [main, "litellm_*"] + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + caching-redis: + uses: ./.github/workflows/_test-unit-services-base.yml + with: + # Redis-only tests that do NOT require provider API keys. + # Tests needing API keys (test_caching.py, test_caching_ssl.py, test_prometheus_service.py, + # test_router_caching.py) are in Phase 3 integration workflows. + test-path: >- + tests/local_testing/test_dual_cache.py + tests/local_testing/test_redis_batch_optimizations.py + tests/local_testing/test_router_utils.py + workers: 2 + reruns: 2 + timeout-minutes: 20 + enable-redis: true + enable-postgres: false + secrets: + REDIS_HOST: ${{ secrets.REDIS_HOST }} + REDIS_PORT: ${{ secrets.REDIS_PORT }} + REDIS_PASSWORD: ${{ secrets.REDIS_PASSWORD }} + DATABASE_URL: ${{ secrets.DATABASE_URL }} + POSTGRES_USER: ${{ secrets.POSTGRES_USER }} + POSTGRES_PASSWORD: ${{ secrets.POSTGRES_PASSWORD }} diff --git a/.github/workflows/test-unit-core-utils.yml b/.github/workflows/test-unit-core-utils.yml new file mode 100644 index 00000000000..2f3698fdf60 --- /dev/null +++ b/.github/workflows/test-unit-core-utils.yml @@ -0,0 +1,20 @@ +name: "Unit Tests: Core Utilities" + +on: + pull_request: + branches: [main] + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + core-utils: + uses: ./.github/workflows/_test-unit-base.yml + with: + test-path: "tests/test_litellm/litellm_core_utils" + workers: 2 + reruns: 1 diff --git a/.github/workflows/test-unit-documentation.yml b/.github/workflows/test-unit-documentation.yml new file mode 100644 index 00000000000..d8b30de6844 --- /dev/null +++ b/.github/workflows/test-unit-documentation.yml @@ -0,0 +1,67 @@ +name: "Unit Tests: Documentation Validation" + +on: + pull_request: + branches: [main] + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + documentation: + runs-on: ubuntu-latest + timeout-minutes: 10 + + steps: + - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + with: + persist-credentials: false + + - name: Set up Python + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: "3.12" + + - name: Install Poetry + run: pip install 'poetry==2.3.2' + + - name: Cache Poetry dependencies + uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 + with: + path: | + ~/.cache/pypoetry + ~/.cache/pip + .venv + key: ${{ runner.os }}-poetry-${{ hashFiles('poetry.lock') }} + restore-keys: | + ${{ runner.os }}-poetry- + + - name: Install dependencies + run: | + poetry config virtualenvs.in-project true + poetry install --with dev,proxy-dev --extras "proxy semantic-router" + poetry run pip install google-genai==1.22.0 \ + google-cloud-aiplatform==1.115.0 fastapi-offline==1.7.3 python-multipart==0.0.22 openapi-core==0.23.0 + + - name: Setup litellm-enterprise + run: | + poetry run pip install --force-reinstall --no-deps -e enterprise/ + + - name: Generate Prisma client + env: + PRISMA_BINARY_CACHE_DIR: ${{ runner.temp }}/prisma-cache + run: | + poetry run pip install nodejs-wheel-binaries==24.13.1 + poetry run prisma generate --schema litellm/proxy/schema.prisma + + # Run the same documentation tests that CircleCI ran (as direct Python scripts) + - name: Run documentation validation tests + run: | + poetry run python ./tests/documentation_tests/test_env_keys.py + poetry run python ./tests/documentation_tests/test_router_settings.py + poetry run python ./tests/documentation_tests/test_api_docs.py + poetry run python ./tests/documentation_tests/test_circular_imports.py diff --git a/.github/workflows/test-unit-enterprise-routing.yml b/.github/workflows/test-unit-enterprise-routing.yml new file mode 100644 index 00000000000..13ae3efedba --- /dev/null +++ b/.github/workflows/test-unit-enterprise-routing.yml @@ -0,0 +1,24 @@ +name: "Unit Tests: Enterprise, Google GenAI & Routing" + +on: + pull_request: + branches: [main] + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + enterprise-routing: + uses: ./.github/workflows/_test-unit-base.yml + with: + test-path: >- + tests/test_litellm/enterprise + tests/test_litellm/google_genai + tests/test_litellm/router_utils + tests/test_litellm/router_strategy + workers: 2 + reruns: 2 diff --git a/.github/workflows/test-unit-integrations.yml b/.github/workflows/test-unit-integrations.yml new file mode 100644 index 00000000000..2789f99d81c --- /dev/null +++ b/.github/workflows/test-unit-integrations.yml @@ -0,0 +1,20 @@ +name: "Unit Tests: Integrations (Callbacks & Logging)" + +on: + pull_request: + branches: [main] + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + integrations: + uses: ./.github/workflows/_test-unit-base.yml + with: + test-path: "tests/test_litellm/integrations" + workers: 2 + reruns: 3 diff --git a/.github/workflows/test-unit-llm-providers.yml b/.github/workflows/test-unit-llm-providers.yml new file mode 100644 index 00000000000..6c00272b0c8 --- /dev/null +++ b/.github/workflows/test-unit-llm-providers.yml @@ -0,0 +1,29 @@ +name: "Unit Tests: LLM Provider Transformations" + +on: + pull_request: + branches: [main] + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + vertex-ai: + name: Vertex AI + uses: ./.github/workflows/_test-unit-base.yml + with: + test-path: "tests/test_litellm/llms/vertex_ai" + workers: 1 + reruns: 2 + + other-providers: + name: All Other Providers + uses: ./.github/workflows/_test-unit-base.yml + with: + test-path: "tests/test_litellm/llms --ignore=tests/test_litellm/llms/vertex_ai" + workers: 2 + reruns: 2 diff --git a/.github/workflows/test-unit-misc.yml b/.github/workflows/test-unit-misc.yml new file mode 100644 index 00000000000..9228decd7cc --- /dev/null +++ b/.github/workflows/test-unit-misc.yml @@ -0,0 +1,31 @@ +name: "Unit Tests: MCP, Secrets, Containers & Misc" + +on: + pull_request: + branches: [main] + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + misc: + uses: ./.github/workflows/_test-unit-base.yml + with: + test-path: >- + tests/test_litellm/secret_managers + tests/test_litellm/a2a_protocol + tests/test_litellm/anthropic_interface + tests/test_litellm/completion_extras + tests/test_litellm/containers + tests/test_litellm/experimental_mcp_client + tests/test_litellm/images + tests/test_litellm/interactions + tests/test_litellm/passthrough + tests/test_litellm/vector_stores + tests/test_litellm/test_*.py + workers: 2 + reruns: 2 diff --git a/.github/workflows/test-unit-proxy-auth.yml b/.github/workflows/test-unit-proxy-auth.yml new file mode 100644 index 00000000000..e71821db701 --- /dev/null +++ b/.github/workflows/test-unit-proxy-auth.yml @@ -0,0 +1,20 @@ +name: "Unit Tests: Proxy Auth & Key Management" + +on: + pull_request: + branches: [main] + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + proxy-auth: + uses: ./.github/workflows/_test-unit-base.yml + with: + test-path: "tests/test_litellm/proxy/auth tests/test_litellm/proxy/hooks tests/test_litellm/proxy/policy_engine tests/test_litellm/proxy/client" + workers: 2 + reruns: 2 diff --git a/.github/workflows/test-unit-proxy-db.yml b/.github/workflows/test-unit-proxy-db.yml new file mode 100644 index 00000000000..bdfb6efeef1 --- /dev/null +++ b/.github/workflows/test-unit-proxy-db.yml @@ -0,0 +1,45 @@ +name: "Unit Tests: Proxy DB Operations" + +# Uses DATABASE_URL secret — only runs on trusted branches, not PRs. +on: + push: + branches: [main, "litellm_*"] + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + proxy-db: + strategy: + fail-fast: false + matrix: + include: + # Key generation tests must NOT run in parallel (event loop conflicts with logging worker) + - test-group: key-generation + test-path: "tests/proxy_unit_tests/test_key_generate_prisma.py" + workers: 0 + timeout: 30 + - test-group: auth-checks + test-path: "tests/proxy_unit_tests/test_auth_checks.py tests/proxy_unit_tests/test_user_api_key_auth.py" + workers: 8 + timeout: 20 + - test-group: remaining + test-path: "tests/proxy_unit_tests --ignore=tests/proxy_unit_tests/test_key_generate_prisma.py --ignore=tests/proxy_unit_tests/test_auth_checks.py --ignore=tests/proxy_unit_tests/test_user_api_key_auth.py" + workers: 8 + timeout: 20 + uses: ./.github/workflows/_test-unit-services-base.yml + with: + test-path: ${{ matrix.test-path }} + workers: ${{ matrix.workers }} + reruns: 2 + timeout-minutes: ${{ matrix.timeout }} + enable-redis: false + enable-postgres: true + secrets: + DATABASE_URL: ${{ secrets.DATABASE_URL }} + POSTGRES_USER: ${{ secrets.POSTGRES_USER }} + POSTGRES_PASSWORD: ${{ secrets.POSTGRES_PASSWORD }} diff --git a/.github/workflows/test-unit-proxy-endpoints.yml b/.github/workflows/test-unit-proxy-endpoints.yml new file mode 100644 index 00000000000..caff3b3ae06 --- /dev/null +++ b/.github/workflows/test-unit-proxy-endpoints.yml @@ -0,0 +1,35 @@ +name: "Unit Tests: Proxy API Endpoints" + +on: + pull_request: + branches: [main] + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + proxy-endpoints: + uses: ./.github/workflows/_test-unit-base.yml + with: + test-path: >- + tests/test_litellm/proxy/management_endpoints + tests/test_litellm/proxy/guardrails + tests/test_litellm/proxy/management_helpers + tests/test_litellm/proxy/anthropic_endpoints + tests/test_litellm/proxy/google_endpoints + tests/test_litellm/proxy/openai_files_endpoint + tests/test_litellm/proxy/response_api_endpoints + tests/test_litellm/proxy/image_endpoints + tests/test_litellm/proxy/vector_store_endpoints + tests/test_litellm/proxy/agent_endpoints + tests/test_litellm/proxy/discovery_endpoints + tests/test_litellm/proxy/health_endpoints + tests/test_litellm/proxy/public_endpoints + tests/test_litellm/proxy/prompts + tests/test_litellm/proxy/ui_crud_endpoints + workers: 2 + reruns: 2 diff --git a/.github/workflows/test-unit-proxy-infra.yml b/.github/workflows/test-unit-proxy-infra.yml new file mode 100644 index 00000000000..4dfbbe317ed --- /dev/null +++ b/.github/workflows/test-unit-proxy-infra.yml @@ -0,0 +1,28 @@ +name: "Unit Tests: Proxy Infrastructure" + +on: + pull_request: + branches: [main] + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + proxy-infra: + uses: ./.github/workflows/_test-unit-base.yml + with: + test-path: >- + tests/test_litellm/proxy/db + tests/test_litellm/proxy/middleware + tests/test_litellm/proxy/spend_tracking + tests/test_litellm/proxy/pass_through_endpoints + tests/test_litellm/proxy/_experimental + tests/test_litellm/proxy/experimental + tests/test_litellm/proxy/common_utils + tests/test_litellm/proxy/test_*.py + workers: 2 + reruns: 2 diff --git a/.github/workflows/test-unit-proxy-legacy.yml b/.github/workflows/test-unit-proxy-legacy.yml new file mode 100644 index 00000000000..a9391137263 --- /dev/null +++ b/.github/workflows/test-unit-proxy-legacy.yml @@ -0,0 +1,96 @@ +name: "Unit Tests: Proxy Legacy Tests" + +on: + pull_request: + branches: [main] + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + test: + runs-on: ubuntu-latest + timeout-minutes: 20 + strategy: + fail-fast: false + matrix: + test-group: + - name: "auth-and-jwt" + path: "tests/proxy_unit_tests/test_[a-j]*.py" + - name: "key-generation" + path: "tests/proxy_unit_tests/test_[k-o]*.py" + - name: "proxy-config" + path: "tests/proxy_unit_tests/test_prisma*.py tests/proxy_unit_tests/test_project*.py tests/proxy_unit_tests/test_prompt*.py tests/proxy_unit_tests/test_proxy_[c-r]*.py" + - name: "proxy-server" + path: "tests/proxy_unit_tests/test_proxy_server.py" + - name: "proxy-server-extras" + path: "tests/proxy_unit_tests/test_proxy_server_*.py tests/proxy_unit_tests/test_proxy_setting_guardrails.py" + - name: "proxy-utils" + path: "tests/proxy_unit_tests/test_proxy_utils.py" + - name: "proxy-token-counter" + path: "tests/proxy_unit_tests/test_proxy_token_counter.py" + - name: "proxy-response-and-misc" + path: "tests/proxy_unit_tests/test_[r-t]*.py" + - name: "proxy-user-auth-and-spend" + path: "tests/proxy_unit_tests/test_[u-z]*.py" + + name: ${{ matrix.test-group.name }} + + steps: + - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + with: + persist-credentials: false + + - name: Set up Python + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: "3.12" + + - name: Install Poetry + run: pip install 'poetry==2.3.2' + + - name: Cache Poetry dependencies + uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 + with: + path: | + ~/.cache/pypoetry + ~/.cache/pip + .venv + key: ${{ runner.os }}-poetry-${{ hashFiles('poetry.lock') }} + restore-keys: | + ${{ runner.os }}-poetry- + + - name: Install dependencies + run: | + poetry config virtualenvs.in-project true + poetry install --with dev,proxy-dev --extras "proxy semantic-router" + poetry run pip install google-genai==1.22.0 \ + google-cloud-aiplatform==1.115.0 fastapi-offline==1.7.3 python-multipart==0.0.22 openapi-core==0.23.0 + + - name: Setup litellm-enterprise + run: | + poetry run pip install --force-reinstall --no-deps -e enterprise/ + + - name: Generate Prisma client + env: + PRISMA_BINARY_CACHE_DIR: ${{ runner.temp }}/prisma-cache + run: | + poetry run pip install nodejs-wheel-binaries==24.13.1 + poetry run prisma generate --schema litellm/proxy/schema.prisma + + - name: Run tests - ${{ matrix.test-group.name }} + env: + TEST_PATH: ${{ matrix.test-group.path }} + run: | + poetry run pytest ${TEST_PATH} \ + --tb=short -vv \ + --maxfail=10 \ + -n 2 \ + --reruns 1 \ + --reruns-delay 1 \ + --dist=loadscope \ + --durations=20 diff --git a/.github/workflows/test-unit-responses-caching-types.yml b/.github/workflows/test-unit-responses-caching-types.yml new file mode 100644 index 00000000000..7f3acac2803 --- /dev/null +++ b/.github/workflows/test-unit-responses-caching-types.yml @@ -0,0 +1,20 @@ +name: "Unit Tests: Responses, Caching & Types" + +on: + pull_request: + branches: [main] + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + responses-caching-types: + uses: ./.github/workflows/_test-unit-base.yml + with: + test-path: "tests/test_litellm/responses tests/test_litellm/caching tests/test_litellm/types" + workers: 2 + reruns: 2 diff --git a/.github/workflows/test-unit-security.yml b/.github/workflows/test-unit-security.yml new file mode 100644 index 00000000000..b38c82b1c24 --- /dev/null +++ b/.github/workflows/test-unit-security.yml @@ -0,0 +1,28 @@ +name: "Unit Tests: Security" + +# Uses DATABASE_URL secret — only runs on trusted branches, not PRs. +on: + push: + branches: [main, "litellm_*"] + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + security: + uses: ./.github/workflows/_test-unit-services-base.yml + with: + test-path: "tests/proxy_security_tests/" + workers: 1 + reruns: 2 + timeout-minutes: 20 + enable-redis: false + enable-postgres: true + secrets: + DATABASE_URL: ${{ secrets.DATABASE_URL }} + POSTGRES_USER: ${{ secrets.POSTGRES_USER }} + POSTGRES_PASSWORD: ${{ secrets.POSTGRES_PASSWORD }} diff --git a/.github/workflows/test_server_root_path.yml b/.github/workflows/test_server_root_path.yml index c359e38bff9..47636ce8e92 100644 --- a/.github/workflows/test_server_root_path.yml +++ b/.github/workflows/test_server_root_path.yml @@ -17,13 +17,15 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v4 + uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + with: + persist-credentials: false - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 + uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12 - name: Build Docker image - uses: docker/build-push-action@v5 + uses: docker/build-push-action@0adf9959216b96bec444f325f1e493d4aa344497 #v6.14 with: context: . file: ./docker/Dockerfile.non_root diff --git a/.github/workflows/zizmor.yml b/.github/workflows/zizmor.yml new file mode 100644 index 00000000000..9a1e899fed5 --- /dev/null +++ b/.github/workflows/zizmor.yml @@ -0,0 +1,31 @@ +name: GitHub Actions Security Analysis + +on: + push: + branches: [main] + pull_request: + branches: [main] + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +permissions: {} + +jobs: + zizmor: + name: zizmor + runs-on: ubuntu-latest + timeout-minutes: 5 + permissions: + security-events: write + contents: read + actions: read + steps: + - name: Checkout repository + uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + with: + persist-credentials: false + + - name: Run zizmor + uses: zizmorcore/zizmor-action@71321a20a9ded102f6e9ce5718a2fcec2c4f70d8 # v0.5.2 diff --git a/.npmrc b/.npmrc new file mode 100644 index 00000000000..168e81a1c4e --- /dev/null +++ b/.npmrc @@ -0,0 +1,5 @@ +# Supply-chain hardening +# Packages needing lifecycle scripts: npm rebuild +ignore-scripts=true +# Protects local npm install only — npm ci (used in CI) ignores this +min-release-age=3d diff --git a/.semgrep/rules/security/no-claude-directory.yml b/.semgrep/rules/security/no-claude-directory.yml new file mode 100644 index 00000000000..7d120a7c23c --- /dev/null +++ b/.semgrep/rules/security/no-claude-directory.yml @@ -0,0 +1,18 @@ +rules: + - id: no-claude-directory-committed + message: > + .claude/ directory must not be committed to the repository. + It contains local Claude Code settings (permissions, worktree paths) that are + developer-machine-specific and may expose internal paths or credentials. + Add .claude/ to .gitignore instead. + severity: ERROR + languages: [generic] + paths: + include: + - "/.claude/**" + - "/.claude/*" + pattern-regex: '[\s\S]+' + metadata: + category: security + tags: [supply-chain, secrets] + confidence: HIGH diff --git a/README.md b/README.md index 67f2f3a2048..a6dc597574e 100644 --- a/README.md +++ b/README.md @@ -266,6 +266,7 @@ Support for more providers. Missing a provider or LLM Platform, raise a [feature + @@ -402,7 +403,7 @@ Support for more providers. Missing a provider or LLM Platform, raise a [feature # Enterprise For companies that need better security, user management and professional support -[Talk to founders](https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions) +[Talk to founders](https://enterprise.litellm.ai/demo) This covers: - ✅ **Features under the [LiteLLM Commercial License](https://docs.litellm.ai/docs/proxy/enterprise):** @@ -452,7 +453,6 @@ All these checks must pass before your PR can be merged. - [Schedule Demo 👋](https://calendly.com/d/4mp-gd3-k5k/berriai-1-1-onboarding-litellm-hosted-version) - [Community Discord 💭](https://discord.gg/wuPM9dRgDw) - [Community Slack 💭](https://www.litellm.ai/support) -- Our numbers 📞 +1 (770) 8783-106 / ‭+1 (412) 618-6238‬ - Our emails ✉️ ishaan@berri.ai / krrish@berri.ai # Why did we build this diff --git a/ci_cd/security_scans.sh b/ci_cd/security_scans.sh index 061e454465a..ee33393c8b1 100755 --- a/ci_cd/security_scans.sh +++ b/ci_cd/security_scans.sh @@ -160,7 +160,7 @@ run_grype_scans() { "CVE-2026-0775" # npm cli incorrect permission assignment - no fix available yet, npm is only used at build/prisma-generate time "GHSA-3ppc-4f35-3m26" # minimatch ReDoS via repeated wildcards - from nodejs_wheel bundled npm, not used in application runtime code "GHSA-83g3-92jg-28cx" # tar arbitrary file read/write via hardlink - from nodejs_wheel bundled npm, not used in application runtime code - "CVE-2026-25639" # axios - full fix requires 1.x major version bump; pinned to >=0.30.2 to clear other axios CVEs, upgrade to 1.x in follow-up + "CVE-2026-25639" # axios DoS via __proto__ in mergeConfig - transitive dev dep via @neondatabase/api-client, not imported in application code "CVE-2026-2297" # Python 3.13 SourcelessFileLoader audit hook bypass - no fix available in base image "GHSA-qffp-2rhf-9h96" # tar hardlink path traversal - from nodejs_wheel bundled npm, not used in application runtime code "CVE-2026-2673" # OpenSSL 3.6.1 TLS 1.3 key exchange group negotiation issue - no fix available yet diff --git a/cookbook/benchmark/readme.md b/cookbook/benchmark/readme.md index 57115eb96a9..afa59aa91ee 100644 --- a/cookbook/benchmark/readme.md +++ b/cookbook/benchmark/readme.md @@ -178,4 +178,4 @@ Benchmark Results for 'When will BerriAI IPO?': ``` ## Support -**🤝 Schedule a 1-on-1 Session:** Book a [1-on-1 session](https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions) with Krrish and Ishaan, the founders, to discuss any issues, provide feedback, or explore how we can improve LiteLLM for you. +**🤝 Schedule a 1-on-1 Session:** Book a [1-on-1 session](https://enterprise.litellm.ai/demo) with Krrish and Ishaan, the founders, to discuss any issues, provide feedback, or explore how we can improve LiteLLM for you. diff --git a/cookbook/codellama-server/README.MD b/cookbook/codellama-server/README.MD index b158bb083f2..82a7e62f40a 100644 --- a/cookbook/codellama-server/README.MD +++ b/cookbook/codellama-server/README.MD @@ -143,7 +143,6 @@ All responses from the server are returned in the following format (for all LLM - [Our calendar 👋](https://calendly.com/d/4mp-gd3-k5k/berriai-1-1-onboarding-litellm-hosted-version) - [Community Discord 💭](https://discord.gg/wuPM9dRgDw) -- Our numbers 📞 +1 (770) 8783-106 / +1 (412) 618-6238 - Our emails ✉️ ishaan@berri.ai / krrish@berri.ai ## Roadmap diff --git a/cookbook/litellm_proxy_server/readme.md b/cookbook/litellm_proxy_server/readme.md index d0b0592c433..2c1eab72c24 100644 --- a/cookbook/litellm_proxy_server/readme.md +++ b/cookbook/litellm_proxy_server/readme.md @@ -164,7 +164,6 @@ All responses from the server are returned in the following format (for all LLM - [Our calendar 👋](https://calendly.com/d/4mp-gd3-k5k/berriai-1-1-onboarding-litellm-hosted-version) - [Community Discord 💭](https://discord.gg/wuPM9dRgDw) -- Our numbers 📞 +1 (770) 8783-106 / +1 (412) 618-6238 - Our emails ✉️ ishaan@berri.ai / krrish@berri.ai ## Roadmap diff --git a/cosign.pub b/cosign.pub new file mode 100644 index 00000000000..2c2a555ab09 --- /dev/null +++ b/cosign.pub @@ -0,0 +1,4 @@ +-----BEGIN PUBLIC KEY----- +MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEKi4ivqGpE231OGH50PKbqy1Y1Kkb +POJC8+i2Wko82gBOUCe3M0Vw86H/4rhUhfoYEti4gdJ9wZbYmK0I2EE96g== +-----END PUBLIC KEY----- diff --git a/docker/Dockerfile.custom_ui b/docker/Dockerfile.custom_ui index c1bd9a383fa..9ab416944f5 100644 --- a/docker/Dockerfile.custom_ui +++ b/docker/Dockerfile.custom_ui @@ -51,7 +51,7 @@ ENV UI_BASE_PATH="/prod/ui" # Build the UI with the specified UI_BASE_PATH WORKDIR /app/ui/litellm-dashboard -RUN npm install +RUN npm ci RUN UI_BASE_PATH=$UI_BASE_PATH npm run build # Create the destination directory diff --git a/docker/Dockerfile.non_root b/docker/Dockerfile.non_root index db3981fb7e7..c224894779c 100644 --- a/docker/Dockerfile.non_root +++ b/docker/Dockerfile.non_root @@ -47,7 +47,7 @@ RUN mkdir -p /var/lib/litellm/ui && \ if [ -f "/app/enterprise/enterprise_ui/enterprise_colors.json" ]; then \ cp /app/enterprise/enterprise_ui/enterprise_colors.json ./ui_colors.json; \ fi && \ - npm install --legacy-peer-deps && \ + npm ci && \ npm run build && \ cp -r /app/ui/litellm-dashboard/out/* /var/lib/litellm/ui/ && \ mkdir -p /var/lib/litellm/assets && \ diff --git a/docker/build_admin_ui.sh b/docker/build_admin_ui.sh index 5373ad0e3d9..efb2bac3535 100755 --- a/docker/build_admin_ui.sh +++ b/docker/build_admin_ui.sh @@ -40,11 +40,22 @@ else exit 1 fi fi -curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.38.0/install.sh | bash +NVM_VERSION="v0.40.4" +NVM_CHECKSUM="4b7412c49960c7d31e8df72da90c1fb5b8cccb419ac99537b737028d497aba4f" +NVM_SCRIPT=$(mktemp) +trap 'rm -f "$NVM_SCRIPT"' EXIT +curl -fsSL "https://raw.githubusercontent.com/nvm-sh/nvm/${NVM_VERSION}/install.sh" -o "$NVM_SCRIPT" +if command -v sha256sum &>/dev/null; then + echo "${NVM_CHECKSUM} ${NVM_SCRIPT}" | sha256sum -c - +elif command -v shasum &>/dev/null; then + echo "${NVM_CHECKSUM} ${NVM_SCRIPT}" | shasum -a 256 -c - +else + echo "No sha256 tool found; cannot verify nvm checksum"; exit 1 +fi || { echo "nvm checksum verification failed"; exit 1; } +bash "$NVM_SCRIPT" source ~/.nvm/nvm.sh nvm install v18.17.0 nvm use v18.17.0 -npm install -g npm # copy _enterprise.json from this directory to /ui/litellm-dashboard, and rename it to ui_colors.json cp enterprise/enterprise_ui/enterprise_colors.json ui/litellm-dashboard/ui_colors.json diff --git a/docs/my-website/blog/ci_cd_v2_improvements/index.md b/docs/my-website/blog/ci_cd_v2_improvements/index.md new file mode 100644 index 00000000000..84f8f7bda6b --- /dev/null +++ b/docs/my-website/blog/ci_cd_v2_improvements/index.md @@ -0,0 +1,55 @@ +--- +slug: ci-cd-v2-improvements +title: "Announcing CI/CD v2 for LiteLLM" +date: 2026-03-30T21:30:00 +authors: + - krrish +description: "CI/CD v2 introduces isolated environments, stronger security gates, and safer release separation for LiteLLM." +tags: [engineering, ci-cd, security] +hide_table_of_contents: false +--- + +import Image from '@theme/IdealImage'; + +The CI/CD v2 is now live for LiteLLM. + + + +
+Building on the roadmap from our [security incident](https://docs.litellm.ai/blog/security-townhall-updates#roadmap), CI/CD v2 introduces isolated environments, stronger security gates, and safer release separation for LiteLLM. + +## What changed + +- Security scans and unit tests run in isolated environments. +- Validation and release are separated into different repositories, making it harder for an attacker to reach release credentials. +- Trusted Publishing for PyPI releases - this means no long-lived credentials are used to publish releases. +- Immutable Docker release tags - this means no tampering of Docker release tags after they are published [Learn more](https://docs.docker.com/docker-hub/repos/manage/hub-images/immutable-tags/). Note: work for GHCR docker releases is planned as well. + +## What's next + +Moving forward, we plan on: +- Adopting OpenSSF (this is a set of security criteria that projects should meet to demonstrate a strong security posture - [Learn more](https://baseline.openssf.org/versions/2026-02-19.html)) + - We've added Scorecard and Allstar to our Github + +- Adding SLSA Build Provenance to our CI/CD pipeline - this means we allow users to independently verify that a release came from us and prevent silent modifications of releases after they are published. + + +We hope that this will mean you can be confident that the releases you are using are safe and from us. + + +## The principle + +The new CI/CD pipeline reflects the principles, outlined below, and is designed to be more secure and reliable: + +- **Limit** what each package can access +- **Reduce** the number of sensitive environment variables +- **Avoid** compromised packages +- **Prevent** release tampering + + +## How to help: + +Help us plan April's stability sprint - https://github.com/BerriAI/litellm/issues/24825 \ No newline at end of file diff --git a/docs/my-website/blog/security_townhall_updates/index.md b/docs/my-website/blog/security_townhall_updates/index.md new file mode 100644 index 00000000000..b997de9c185 --- /dev/null +++ b/docs/my-website/blog/security_townhall_updates/index.md @@ -0,0 +1,190 @@ +--- +slug: security-townhall-updates +title: "Security Townhall Updates" +date: 2026-03-27T12:00:00 +authors: + - krrish + - ishaan-alt +description: "What happened, what we've done, and what comes next for LiteLLM's release and security processes." +tags: [security, incident-report] +hide_table_of_contents: false +--- + +import Image from '@theme/IdealImage'; + +Thank you to everyone who joined our town hall. + +We wanted to use that time to walk through what we know, what we've done so far, and how we're improving LiteLLM's release and security processes going forward. This post is a written version of that update. [Slides available here](https://drive.google.com/file/d/17hsSG7nk-OYL7VRCTbTa7McrWREtS9OO/view?usp=sharing) + +{/* truncate */} + +## What happened + +On March 24, 2026 at 10:39 UTC, LiteLLM v1.82.7 was pushed to PyPI. Version v1.82.8 was published soon after. Those packages were live for about 40 minutes before being quarantined by PyPI. By 16:00 UTC, the LiteLLM team had worked with PyPI to delete the affected packages. + +At this point, our understanding is that this was a supply-chain incident affecting those two published versions. + +## How did this happen? + +Our understanding is that the issue came from the [compromised Trivy security scanner](https://www.aquasec.com/blog/trivy-supply-chain-attack-what-you-need-to-know/) dependency in our CI/CD pipeline. + + + +There were three major contributing factors: + +### 1. Shared CI/CD environment + +At the time, everything was running on CircleCI, and all steps shared a common environment. That increased blast radius: if one component was compromised, it could potentially access credentials or context intended for other parts of the pipeline. + +### 2. Static credentials in environment variables + +Release credentials, including credentials for PyPI, GHCR, and Docker publishing, were available as static secrets in the environment. That meant a compromised step could access long-lived release credentials. + +### 3. Unpinned Trivy dependency + +In our security scanning component, we had an unpinned Trivy dependency. Our present understanding is that a compromised Trivy package ran during the scan, had access to environment variables, and enabled attackers to obtain those credentials. + +**In summary:** a compromised package in CI had access to secrets it should not have had, and those secrets were then used in the release path. + +## What we've already done + + +In the last 3 days, we've taken the following steps: + +### 1. Minimize Scope of Impact + +#### Prevented further key abuse + +We deleted or rotated all impacted or adjacent secret keys, including PyPI, GitHub, Docker, and related credentials. Out of an abundance of caution, we've also rotated LiteLLM maintainer accounts. + +#### Prevent branch attacks + +We removed roughly 6,000 open branches and added an auto-deletion policy for branches merged into `main`. This reduces the surface area for branch-based abuse. + +#### Pinned CI/CD dependencies + +We've pinned all Github Actions, and are working on pinning all CircleCI dependencies as well. + +#### Paused releases + +We've paused new releases until we've confirmed codebase security and put stronger release controls in place. + +### 2. Secured LiteLLM + +#### Forensic analysis + +We are working with Google's Mandiant cybersecurity team to confirm the source of the attack and verify the security of the codebase. We also confirmed that no malicious code was pushed to `main`. + +#### Confirm Application Security + +In parallel, we are working with whitehat hackers at [Veria Labs](https://verialabs.com/) to verify application security and review improvements to our CI/CD process. + +We have also confirmed that the last 20 LiteLLM releases contain no indicators of compromise, and that no unauthenticated attacks can be made against LiteLLM Proxy based on our current investigation. [Check Security Blog for release verification.](https://docs.litellm.ai/blog/security-update-march-2026#verified-safe-versions) + +#### Created a security working group + +We created a new security working group inside LiteLLM focused on: + +- Building threat models +- Auditing the build process and dependencies + +If you're interested in joining the security working group, please file an issue [here](https://github.com/BerriAI/litellm-security-wg). + +### 3. Improved CI/CD + +We've already begun making structural changes to how releases are built and published. These align with our goals (covered in the next section) around isolated environments, ephemeral credentials, and release auditing. + +## Roadmap + +We plan on following 4 guiding principles for our new CI/CD pipeline: + +1. **Limit** what each package can access +2. **Reduce** the number of sensitive environment variables +3. **Avoid** compromised packages +4. **Prevent** release tampering + + +### Isolated environments + + + +We are breaking our CI/CD into 4 semantic concepts: + +1. Unit tests +2. Integration tests +3. Security scans +4. Release publishing + +And will be running each of these in isolated environments. + +This will limit the damage that any single compromised component can cause. + +### Ephemeral credentials + +We plan to move to ephemeral credentials for PyPI (Trusted Publisher) and GHCR (Token-based authentication) releases. This will reduce the risk of credentials being leaked or compromised. + +We have already begun doing this: + +- PyPI Trusted Publisher on GitHub Actions [PR](https://github.com/BerriAI/litellm/pull/24654) +- GHCR Token-based authentication on GitHub Actions [PR](https://github.com/BerriAI/litellm/pull/24683) + +### Release auditing + +Our goal is to allow users to independently verify that a release came from us and prevent silent modifications of releases after they are published. + +This will ensure, your releases are safe, even when: +- Stolen PyPI/GHCR credentials are used to publish malicious releases +- Tampered registry artifacts are published +- Tag mutations are made after the release is published + +We believe that [Cosign](https://github.com/sigstore/cosign) is a good fit for this, and have already begun working on it [PR](https://github.com/BerriAI/litellm/pull/24683). + + +### Avoid Compromised Packages + +- Move to pinned, verified SHAs for packages and actions used in CI/CD, avoiding `latest` wherever possible. +- Add a cooldown period before upgrading to a new version of a package - allows more time to investigate and verify the new version. + +We've added zizmor to help us catch issues such as unpinned dependencies and credential leakage. [commit](https://github.com/BerriAI/litellm/commit/a671275f5c5b0e1fb1adacdf3b6ef779aaa5d56c). + + +## Frequently Asked Questions + +**Q: Did you observe any lateral movement into your corporate environment during this incident?** + +A: No. Our investigation to date, conducted in coordination with external security experts, has found no evidence of lateral movement into our internal corporate systems. The incident was isolated to the CI/CD pipeline and the release path for specific versions (v1.82.7 and v1.82.8). As a proactive measure, we have rotated all potentially impacted or adjacent secrets—including PyPI, GitHub, and Docker credentials—and updated maintainer account security to ensure continued isolation. + +**Q: Do you expect delays in future product releases due to these new security measures?** + +A: We are committed to balancing security with speed. While we have temporarily paused releases to implement stronger controls, we are moving quickly to automate our new security protocols. We are currently implementing isolated CI/CD environments, ephemeral credentials (via Trusted Publishers), and release auditing with Cosign. These improvements are designed to be integrated into our automated pipeline, allowing us to maintain a fast release cadence while ensuring every package is verified and secure. + +**Q: Were older packages impacted?** + +Our current findings show no indicators of compromise in the last 20 versions of LiteLLM. This was manually verified by our team and independently reviewed by Veria Labs. + +We have also published the verified versions for users to use. [Check Security Blog for release verification.](https://docs.litellm.ai/blog/security-update-march-2026#verified-safe-versions) + + + +## Questions & Support + +If you believe your systems may be affected, contact us immediately: + +- **Security:** security@berri.ai +- **Support:** support@berri.ai +- **Slack:** Reach out to the LiteLLM team directly [here](https://join.slack.com/t/litellmossslack/shared_invite/zt-3o7nkuyfr-p_kbNJj8taRfXGgQI1~YyA) + +## Hiring + +We are currently hiring for: + +- DevOps Engineer - to keep ci/cd secure and running smoothly +- Security Engineer - to keep the application secure + +If you're interest in joining, please apply [here](https://jobs.ashbyhq.com/litellm) \ No newline at end of file diff --git a/docs/my-website/blog/security_townhall_updates/shared_ci_cd_environment.png b/docs/my-website/blog/security_townhall_updates/shared_ci_cd_environment.png new file mode 100644 index 00000000000..29ec195b7fb Binary files /dev/null and b/docs/my-website/blog/security_townhall_updates/shared_ci_cd_environment.png differ diff --git a/docs/my-website/blog/security_update_march_2026/index.md b/docs/my-website/blog/security_update_march_2026/index.md index 20cf5e8f3ff..1c298fe372f 100644 --- a/docs/my-website/blog/security_update_march_2026/index.md +++ b/docs/my-website/blog/security_update_march_2026/index.md @@ -12,16 +12,27 @@ hide_table_of_contents: false import Tabs from '@theme/Tabs'; import TabItem from '@theme/TabItem'; +import VersionVerificationTable from '@site/src/components/VersionVerificationTable'; > **Status:** Active investigation -> **Last updated:** March 24, 2026, 2:00 PM ET +> **Last updated:** March 27, 2026 + +> **Update (March 30):** A new **clean** version of LiteLLM is now available (v1.83.0). This was released by our new [CI/CD v2](https://docs.litellm.ai/blog/ci-cd-v2-improvements) pipeline which added isolated environments, stronger security gates, and safer release separation for LiteLLM. + +> **Update (March 27):** Review Townhall updates, including explanation of the incident, what we've done, and what comes next. [Learn more](https://docs.litellm.ai/blog/security-townhall-updates) + +> **Update (March 27):** Added [Verified safe versions](#verified-safe-versions) section with SHA-256 checksums for all audited PyPI and Docker releases. + +> **Update (March 26):** Added `checkmarx[.]zone` to [Indicators of compromise](#indicators-of-compromise-iocs) + +> **Update (March 25):** Added community-contributed scripts for scanning GitHub Actions and GitLab CI pipelines for the compromised versions. See [How to check if you are affected](#how-to-check-if-you-are-affected). s/o [@Zach Fury](https://www.linkedin.com/in/fryware/) for these scripts. ## TLDR; -- The compromised PyPI packages were **litellm==1.82.7** and **litellm==1.82.8**. Those packages have now been removed from PyPI. -- We believe that the compromise originated from the Trivy dependency used in our CI/CD security scanning workflow. +- The compromised PyPI packages were **litellm==1.82.7** and **litellm==1.82.8**. Those packages were live on March 24, 2026 from 10:39 UTC for about 40 minutes before being quarantined by PyPI. +- We believe that the compromise originated from the [Trivy dependency](https://www.aquasec.com/blog/trivy-supply-chain-attack-what-you-need-to-know/) used in our CI/CD security scanning workflow. - Customers running the official LiteLLM Proxy Docker image were not impacted. That deployment path pins dependencies in requirements.txt and does not rely on the compromised PyPI packages. -- We are pausing new LiteLLM releases until we complete a broader supply-chain review and confirm the release path is safe. +- ~~We have paused all new LiteLLM releases until we complete a broader supply-chain review and confirm the release path is safe.~~ **Updated:** We have now released a new **safe** version of LiteLLM (v1.83.0) by our new [CI/CD v2](https://docs.litellm.ai/blog/ci-cd-v2-improvements) pipeline which added isolated environments, stronger security gates, and safer release separation for LiteLLM. We have also verified the codebase is safe and no malicious code was pushed to `main`. ## Overview @@ -91,9 +102,548 @@ pip show litellm Go to the proxy base url, and check the version of the installed LiteLLM. ![Proxy version check](../../img/security_update_march_2026/proxy_version.png) + + + +Scans all repositories in a GitHub organization for workflow jobs that installed the compromised versions. + +**Requirements:** Python 3 and `requests` (`pip install requests`). + +**Setup:** + +```bash +export GITHUB_TOKEN="your-github-pat" +``` + +**Run:** + +```bash +python find_litellm_github.py +``` + +Set the `ORG` variable in the script to your GitHub organization name. + +Both scripts default to scanning jobs from **today**. Adjust the `WINDOW_START` and `WINDOW_END` constants to cover **March 24, 2026** (the incident date) if running on a different day. + +
+View full script (find_litellm_github.py) + +```python +#!/usr/bin/env python3 +""" +Scan all GitHub Actions jobs in a GitHub org that ran between +0800-1244 UTC today and identify any that installed litellm 1.82.7 or 1.82.8. + +Adjust WINDOW_START / WINDOW_END to cover March 24, 2026 if running later. +""" + +import io +import os +import re +import sys +import zipfile +from concurrent.futures import ThreadPoolExecutor, as_completed +from datetime import datetime, timezone + +import requests + +GITHUB_URL = "https://api.github.com" +ORG = "your-org" # <-- set to your GitHub organization +TOKEN = os.environ.get("GITHUB_TOKEN", "") + +TODAY = datetime.now(timezone.utc).date() +WINDOW_START = datetime(TODAY.year, TODAY.month, TODAY.day, 8, 0, 0, tzinfo=timezone.utc) +WINDOW_END = datetime(TODAY.year, TODAY.month, TODAY.day, 12, 44, 0, tzinfo=timezone.utc) + +TARGET_VERSIONS = {"1.82.7", "1.82.8"} +VERSION_PATTERN = re.compile(r"litellm[=\-](\d+\.\d+\.\d+)", re.IGNORECASE) + +SESSION = requests.Session() +SESSION.headers.update({ + "Authorization": f"Bearer {TOKEN}", + "Accept": "application/vnd.github+json", + "X-GitHub-Api-Version": "2022-11-28", +}) + + +def get_paginated(url, params=None): + params = dict(params or {}) + params.setdefault("per_page", 100) + page = 1 + while True: + params["page"] = page + resp = SESSION.get(url, params=params, timeout=30) + if resp.status_code == 404: + return + resp.raise_for_status() + data = resp.json() + if isinstance(data, dict): + items = next((v for v in data.values() if isinstance(v, list)), []) + else: + items = data + if not items: + break + yield from items + if len(items) < params["per_page"]: + break + page += 1 + + +def parse_ts(ts_str): + if not ts_str: + return None + return datetime.fromisoformat(ts_str.replace("Z", "+00:00")) + + +def get_repos(): + repos = [] + for r in get_paginated(f"{GITHUB_URL}/orgs/{ORG}/repos", {"type": "all"}): + repos.append({"id": r["id"], "name": r["name"], "full_name": r["full_name"]}) + return repos + + +def get_runs_in_window(repo_full_name): + created_filter = ( + f"{WINDOW_START.strftime('%Y-%m-%dT%H:%M:%SZ')}" + f"..{WINDOW_END.strftime('%Y-%m-%dT%H:%M:%SZ')}" + ) + url = f"{GITHUB_URL}/repos/{repo_full_name}/actions/runs" + runs = [] + for run in get_paginated(url, {"created": created_filter, "per_page": 100}): + ts = parse_ts(run.get("run_started_at") or run.get("created_at")) + if ts and WINDOW_START <= ts <= WINDOW_END: + runs.append(run) + return runs + + +def get_jobs_for_run(repo_full_name, run_id): + url = f"{GITHUB_URL}/repos/{repo_full_name}/actions/runs/{run_id}/jobs" + jobs = [] + for job in get_paginated(url, {"filter": "all"}): + ts = parse_ts(job.get("started_at")) + if ts and WINDOW_START <= ts <= WINDOW_END: + jobs.append(job) + return jobs + + +def fetch_job_log(repo_full_name, job_id): + url = f"{GITHUB_URL}/repos/{repo_full_name}/actions/jobs/{job_id}/logs" + resp = SESSION.get(url, timeout=60, allow_redirects=True) + if resp.status_code in (403, 404, 410): + return "" + resp.raise_for_status() + + content_type = resp.headers.get("Content-Type", "") + if "zip" in content_type or resp.content[:2] == b"PK": + try: + with zipfile.ZipFile(io.BytesIO(resp.content)) as zf: + parts = [] + for name in sorted(zf.namelist()): + with zf.open(name) as f: + parts.append(f.read().decode("utf-8", errors="replace")) + return "\n".join(parts) + except zipfile.BadZipFile: + pass + return resp.text + + +def check_job(repo_full_name, job): + job_id = job["id"] + job_name = job["name"] + run_id = job["run_id"] + started = job.get("started_at", "") + + log_text = fetch_job_log(repo_full_name, job_id) + if not log_text: + return None + + found_versions = set() + context_lines = [] + for line in log_text.splitlines(): + m = VERSION_PATTERN.search(line) + if m: + ver = m.group(1) + if ver in TARGET_VERSIONS: + found_versions.add(ver) + context_lines.append(line.strip()) + + if not found_versions: + return None + + return { + "repo": repo_full_name, + "run_id": run_id, + "job_id": job_id, + "job_name": job_name, + "started_at": started, + "versions": sorted(found_versions), + "context": context_lines[:10], + "job_url": job.get("html_url", f"https://github.com/{repo_full_name}/actions/runs/{run_id}"), + } + + +def main(): + if not TOKEN: + print("ERROR: Set GITHUB_TOKEN environment variable.", file=sys.stderr) + sys.exit(1) + + print(f"Time window : {WINDOW_START.isoformat()} -> {WINDOW_END.isoformat()}") + print(f"Hunting for : litellm {', '.join(sorted(TARGET_VERSIONS))}") + print() + + print(f"Fetching repositories for org '{ORG}'...") + repos = get_repos() + print(f" Found {len(repos)} repositories") + print() + + jobs_to_check = [] + + print("Scanning workflow runs for time window...") + for repo in repos: + full_name = repo["full_name"] + try: + runs = get_runs_in_window(full_name) + except requests.HTTPError as e: + print(f" WARN: {full_name} - {e}", file=sys.stderr) + continue + if not runs: + continue + print(f" {full_name}: {len(runs)} run(s) in window") + for run in runs: + try: + jobs = get_jobs_for_run(full_name, run["id"]) + except requests.HTTPError as e: + print(f" WARN: run {run['id']} - {e}", file=sys.stderr) + continue + for job in jobs: + jobs_to_check.append((full_name, job)) + + total = len(jobs_to_check) + print(f"\nFetching logs for {total} job(s)...") + print() + + hits = [] + with ThreadPoolExecutor(max_workers=8) as pool: + futures = { + pool.submit(check_job, full_name, job): (full_name, job["id"]) + for full_name, job in jobs_to_check + } + done = 0 + for future in as_completed(futures): + done += 1 + full_name, jid = futures[future] + try: + result = future.result() + except Exception as e: + print(f" ERROR {full_name} job {jid}: {e}", file=sys.stderr) + continue + if result: + hits.append(result) + print( + f" [{done}/{total}] {full_name} job {jid}" + + (f" *** HIT: litellm {result['versions']} ***" if result else ""), + flush=True, + ) + + print() + print("=" * 72) + print(f"RESULTS: {len(hits)} job(s) installed litellm {' or '.join(sorted(TARGET_VERSIONS))}") + print("=" * 72) + + if not hits: + print("No matches found.") + return + + for h in sorted(hits, key=lambda x: x["started_at"]): + print() + print(f" Repo : {h['repo']}") + print(f" Job : {h['job_name']} (#{h['job_id']})") + print(f" Run ID : {h['run_id']}") + print(f" Started : {h['started_at']}") + print(f" Versions : litellm {', '.join(h['versions'])}") + print(f" URL : {h['job_url']}") + print(f" Log lines :") + for line in h["context"]: + print(f" {line}") + + +if __name__ == "__main__": + main() +``` + +
+ + + + +Scans all projects in a GitLab group (including subgroups) for CI/CD jobs that installed the compromised versions. + +**Requirements:** Python 3 and `requests` (`pip install requests`). + +**Setup:** + +```bash +export GITLAB_TOKEN="your-gitlab-pat" +``` + +**Run:** + +```bash +python find_litellm_jobs.py +``` + +Set the `GROUP_NAME` variable in the script to your GitLab group name. + +Both scripts default to scanning jobs from **today**. Adjust the `WINDOW_START` and `WINDOW_END` constants to cover **March 24, 2026** (the incident date) if running on a different day. + +
+View full script (find_litellm_jobs.py) + +```python +#!/usr/bin/env python3 +""" +Scan all GitLab CI/CD jobs in a GitLab group that ran between +0800-1244 UTC today and identify any that installed litellm 1.82.7 or 1.82.8. + +Adjust WINDOW_START / WINDOW_END to cover March 24, 2026 if running later. +""" + +import os +import re +import sys +from concurrent.futures import ThreadPoolExecutor, as_completed +from datetime import datetime, timezone + +import requests + +GITLAB_URL = "https://gitlab.com" +GROUP_NAME = "YourGroup" # <-- set to your GitLab group name +TOKEN = os.environ.get("GITLAB_TOKEN", "") + +TODAY = datetime.now(timezone.utc).date() +WINDOW_START = datetime(TODAY.year, TODAY.month, TODAY.day, 8, 0, 0, tzinfo=timezone.utc) +WINDOW_END = datetime(TODAY.year, TODAY.month, TODAY.day, 12, 44, 0, tzinfo=timezone.utc) + +TARGET_VERSIONS = {"1.82.7", "1.82.8"} +VERSION_PATTERN = re.compile(r"litellm[=\-](\d+\.\d+\.\d+)", re.IGNORECASE) + +HEADERS = {"PRIVATE-TOKEN": TOKEN} +SESSION = requests.Session() +SESSION.headers.update(HEADERS) + + +def get_paginated(url, params=None): + params = dict(params or {}) + params.setdefault("per_page", 100) + page = 1 + while True: + params["page"] = page + resp = SESSION.get(url, params=params, timeout=30) + resp.raise_for_status() + data = resp.json() + if not data: + break + yield from data + if len(data) < params["per_page"]: + break + page += 1 + + +def get_group_id(group_name): + resp = SESSION.get(f"{GITLAB_URL}/api/v4/groups/{group_name}", timeout=30) + resp.raise_for_status() + return resp.json()["id"] + + +def get_all_projects(group_id): + projects = [] + for p in get_paginated( + f"{GITLAB_URL}/api/v4/groups/{group_id}/projects", + {"include_subgroups": "true", "archived": "false"}, + ): + projects.append({"id": p["id"], "name": p["path_with_namespace"]}) + return projects + + +def parse_ts(ts_str): + if not ts_str: + return None + ts_str = ts_str.replace("Z", "+00:00") + return datetime.fromisoformat(ts_str) + + +def jobs_in_window(project_id): + matching = [] + url = f"{GITLAB_URL}/api/v4/projects/{project_id}/jobs" + params = {"per_page": 100, "scope[]": ["success", "failed", "canceled", "running"]} + + page = 1 + while True: + params["page"] = page + resp = SESSION.get(url, params=params, timeout=30) + if resp.status_code == 403: + return matching + resp.raise_for_status() + jobs = resp.json() + if not jobs: + break + + stop_early = False + for job in jobs: + ts = parse_ts(job.get("started_at") or job.get("created_at")) + if ts is None: + continue + if ts > WINDOW_END: + continue + if ts < WINDOW_START: + stop_early = True + continue + matching.append(job) + + if stop_early or len(jobs) < 100: + break + page += 1 + + return matching + + +def fetch_trace(project_id, job_id): + url = f"{GITLAB_URL}/api/v4/projects/{project_id}/jobs/{job_id}/trace" + resp = SESSION.get(url, timeout=60) + if resp.status_code in (403, 404): + return "" + resp.raise_for_status() + return resp.text + + +def check_job(project_name, project_id, job): + job_id = job["id"] + job_name = job["name"] + ref = job.get("ref", "") + started = job.get("started_at", job.get("created_at", "")) + + trace = fetch_trace(project_id, job_id) + if not trace: + return None + + found_versions = set() + for match in VERSION_PATTERN.finditer(trace): + ver = match.group(1) + if ver in TARGET_VERSIONS: + found_versions.add(ver) + + if not found_versions: + return None + + context_lines = [] + for line in trace.splitlines(): + if VERSION_PATTERN.search(line): + ver_match = VERSION_PATTERN.search(line) + if ver_match and ver_match.group(1) in TARGET_VERSIONS: + context_lines.append(line.strip()) + + return { + "project": project_name, + "project_id": project_id, + "job_id": job_id, + "job_name": job_name, + "ref": ref, + "started_at": started, + "versions": sorted(found_versions), + "context": context_lines[:10], + "job_url": f"{GITLAB_URL}/{project_name}/-/jobs/{job_id}", + } + + +def main(): + if not TOKEN: + print("ERROR: Set GITLAB_TOKEN environment variable.", file=sys.stderr) + sys.exit(1) + + print(f"Time window : {WINDOW_START.isoformat()} -> {WINDOW_END.isoformat()}") + print(f"Hunting for : litellm {', '.join(sorted(TARGET_VERSIONS))}") + print() + + print(f"Resolving group '{GROUP_NAME}'...") + group_id = get_group_id(GROUP_NAME) + + print("Fetching projects...") + projects = get_all_projects(group_id) + print(f" Found {len(projects)} projects") + print() + + all_jobs_to_check = [] + + print("Scanning job listings for time window...") + for proj in projects: + try: + jobs = jobs_in_window(proj["id"]) + except requests.HTTPError as e: + print(f" WARN: {proj['name']} - {e}", file=sys.stderr) + continue + if jobs: + print(f" {proj['name']}: {len(jobs)} job(s) in window") + for j in jobs: + all_jobs_to_check.append((proj["name"], proj["id"], j)) + + total = len(all_jobs_to_check) + print(f"\nFetching traces for {total} job(s)...") + print() + + hits = [] + with ThreadPoolExecutor(max_workers=10) as pool: + futures = { + pool.submit(check_job, pname, pid, job): (pname, job["id"]) + for pname, pid, job in all_jobs_to_check + } + done = 0 + for future in as_completed(futures): + done += 1 + pname, jid = futures[future] + try: + result = future.result() + except Exception as e: + print(f" ERROR checking {pname} job {jid}: {e}", file=sys.stderr) + continue + if result: + hits.append(result) + print(f" [{done}/{total}] checked {pname} job {jid}" + + (f" *** HIT: litellm {result['versions']} ***" if result else ""), + flush=True) + + print() + print("=" * 72) + print(f"RESULTS: {len(hits)} job(s) installed litellm {' or '.join(sorted(TARGET_VERSIONS))}") + print("=" * 72) + + if not hits: + print("No matches found.") + return + + for h in sorted(hits, key=lambda x: x["started_at"]): + print() + print(f" Project : {h['project']}") + print(f" Job : {h['job_name']} (#{h['job_id']})") + print(f" Branch/tag: {h['ref']}") + print(f" Started : {h['started_at']}") + print(f" Versions : litellm {', '.join(h['versions'])}") + print(f" URL : {h['job_url']}") + print(f" Log lines :") + for line in h["context"]: + print(f" {line}") + + +if __name__ == "__main__": + main() +``` + +
+ +*CI/CD scripts contributed by the community ([original gist](https://gist.github.com/fryz/93ec8d4898ffe5b5ac5706a208823ef3)). Review before running.* + ## Indicators of compromise (IoCs) @@ -102,6 +652,8 @@ Review affected systems for the following indicators: - `litellm_init.pth` present in your `site-packages` - Outbound traffic or requests to `models.litellm[.]cloud` This domain is **not** affiliated with LiteLLM +- Outbound traffic or requests to `checkmarx[.]zone` + This domain is **not** affiliated with LiteLLM ## Immediate actions for affected users @@ -156,6 +708,72 @@ The LiteLLM AI Gateway team has already taken the following steps: - Engaged Google's Mandiant security team to assist with forensic analysis of the build and publishing chain +## Verified safe versions + +We have audited every LiteLLM release published between v1.78.0 and v1.82.6 across both PyPI and Docker. Each artifact was verified by: + +1. Downloading the published artifact and computing its SHA-256 digest +2. Scanning for the known [indicators of compromise](#indicators-of-compromise-iocs) (IOCs) +3. Comparing the artifact contents against the corresponding Git commit in the BerriAI/litellm repository + +**All versions listed below are confirmed clean.** + + + + + + + + + + + + + + + ## Questions and support If you believe your systems may be affected, contact us immediately: diff --git a/docs/my-website/blog/vanta_compliance_recertification/index.md b/docs/my-website/blog/vanta_compliance_recertification/index.md new file mode 100644 index 00000000000..d05c113967f --- /dev/null +++ b/docs/my-website/blog/vanta_compliance_recertification/index.md @@ -0,0 +1,18 @@ +--- +slug: vanta-compliance-recertification +title: "LiteLLM + Vanta: SOC 2 Type 2 and ISO 27001 Recertification" +date: 2026-03-30T10:00:00 +authors: + - krrish +description: "LiteLLM is partnering with Vanta on SOC 2 Type 2 and ISO 27001 recertification and engaging independent auditors for verification." +tags: [security, compliance] +hide_table_of_contents: true +--- + +![LiteLLM x Vanta SOC-2 Recertification](/img/blog/vanta_soc2_recertification.png) + +We are partnering with [Vanta](https://www.vanta.com/) to recertify LiteLLM's compliance for SOC 2 Type 2 and ISO 27001. + +As part of this process, we are also identifying independent auditors to validate and verify our compliance posture. + +This is part of our commitment to being the most secure and transparent AI Gateway possible. diff --git a/docs/my-website/docs/enterprise.md b/docs/my-website/docs/enterprise.md index 6dccf7ff4e7..a3fc9e38b6e 100644 --- a/docs/my-website/docs/enterprise.md +++ b/docs/my-website/docs/enterprise.md @@ -4,7 +4,7 @@ import Image from '@theme/IdealImage'; :::info - ✨ SSO is free for up to 5 users. After that, an enterprise license is required. [Get Started with Enterprise here](https://www.litellm.ai/enterprise) -- Who is Enterprise for? Companies giving access to 100+ users **OR** 10+ AI use-cases. If you're not sure, [get in touch with us](https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions) to discuss your needs. +- Who is Enterprise for? Companies giving access to 100+ users **OR** 10+ AI use-cases. If you're not sure, [get in touch with us](https://enterprise.litellm.ai/demo) to discuss your needs. ::: For companies that need SSO, user management and professional support for LiteLLM Proxy @@ -36,7 +36,7 @@ Manage Yourself - you can deploy our Docker Image or build a custom image from o ### What’s the cost of the Self-Managed Enterprise edition? -Self-Managed Enterprise deployments require our team to understand your exact needs. [Get in touch with us to learn more](https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions) +Self-Managed Enterprise deployments require our team to understand your exact needs. [Get in touch with us to learn more](https://enterprise.litellm.ai/demo) ### How does deployment with Enterprise License work? @@ -106,7 +106,7 @@ Professional Support can assist with LLM/Provider integrations, deployment, upgr Pricing is based on usage. We can figure out a price that works for your team, on the call. -[**Contact Us to learn more**](https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions) +[**Contact Us to learn more**](https://enterprise.litellm.ai/demo) diff --git a/docs/my-website/docs/fine_tuning.md b/docs/my-website/docs/fine_tuning.md index d0bd98a76f9..52e96f28688 100644 --- a/docs/my-website/docs/fine_tuning.md +++ b/docs/my-website/docs/fine_tuning.md @@ -6,7 +6,7 @@ import TabItem from '@theme/TabItem'; :::info -This is an Enterprise only endpoint [Get Started with Enterprise here](https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions) +This is an Enterprise only endpoint [Get Started with Enterprise here](https://enterprise.litellm.ai/demo) ::: diff --git a/docs/my-website/docs/learn/sdk_quickstart.md b/docs/my-website/docs/learn/sdk_quickstart.md index bdf7b63eb5d..0fb8c3f02a5 100644 --- a/docs/my-website/docs/learn/sdk_quickstart.md +++ b/docs/my-website/docs/learn/sdk_quickstart.md @@ -11,7 +11,7 @@ Use this path if you are integrating LiteLLM directly into application code. ## 1. Install LiteLLM ```bash -pip install litellm +pip install litellm==1.82.6 ``` ## 2. Set Provider Credentials diff --git a/docs/my-website/docs/observability/arize_integration.md b/docs/my-website/docs/observability/arize_integration.md index b3ccf98ea3b..4486fb2b718 100644 --- a/docs/my-website/docs/observability/arize_integration.md +++ b/docs/my-website/docs/observability/arize_integration.md @@ -194,5 +194,4 @@ print(response) - [Schedule Demo 👋](https://calendly.com/d/4mp-gd3-k5k/berriai-1-1-onboarding-litellm-hosted-version) - [Community Discord 💭](https://discord.gg/wuPM9dRgDw) -- Our numbers 📞 +1 (770) 8783-106 / +1 (412) 618-6238 - Our emails ✉️ ishaan@berri.ai / krrish@berri.ai diff --git a/docs/my-website/docs/observability/gcs_bucket_integration.md b/docs/my-website/docs/observability/gcs_bucket_integration.md index 69b956950e5..5f8d42508ae 100644 --- a/docs/my-website/docs/observability/gcs_bucket_integration.md +++ b/docs/my-website/docs/observability/gcs_bucket_integration.md @@ -6,7 +6,7 @@ Log LLM Logs to [Google Cloud Storage Buckets](https://cloud.google.com/storage? :::info -✨ This is an Enterprise only feature [Get Started with Enterprise here](https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions) +✨ This is an Enterprise only feature [Get Started with Enterprise here](https://enterprise.litellm.ai/demo) ::: @@ -79,5 +79,4 @@ curl --location 'http://0.0.0.0:4000/chat/completions' \ - [Schedule Demo 👋](https://calendly.com/d/4mp-gd3-k5k/berriai-1-1-onboarding-litellm-hosted-version) - [Community Discord 💭](https://discord.gg/wuPM9dRgDw) -- Our numbers 📞 +1 (770) 8783-106 / ‭+1 (412) 618-6238‬ - Our emails ✉️ ishaan@berri.ai / krrish@berri.ai diff --git a/docs/my-website/docs/observability/langfuse_integration.md b/docs/my-website/docs/observability/langfuse_integration.md index d3c5a44d481..32849ebdb92 100644 --- a/docs/my-website/docs/observability/langfuse_integration.md +++ b/docs/my-website/docs/observability/langfuse_integration.md @@ -342,5 +342,4 @@ Be aware that if you are continuing an existing trace, and you set `update_trace - [Schedule Demo 👋](https://calendly.com/d/4mp-gd3-k5k/berriai-1-1-onboarding-litellm-hosted-version) - [Community Discord 💭](https://discord.gg/wuPM9dRgDw) -- Our numbers 📞 +1 (770) 8783-106 / ‭+1 (412) 618-6238‬ - Our emails ✉️ ishaan@berri.ai / krrish@berri.ai diff --git a/docs/my-website/docs/observability/langsmith_integration.md b/docs/my-website/docs/observability/langsmith_integration.md index cada4122b20..bf867319de8 100644 --- a/docs/my-website/docs/observability/langsmith_integration.md +++ b/docs/my-website/docs/observability/langsmith_integration.md @@ -225,5 +225,4 @@ environment_variables: - [Schedule Demo 👋](https://calendly.com/d/4mp-gd3-k5k/berriai-1-1-onboarding-litellm-hosted-version) - [Community Discord 💭](https://discord.gg/wuPM9dRgDw) -- Our numbers 📞 +1 (770) 8783-106 / ‭+1 (412) 618-6238‬ - Our emails ✉️ ishaan@berri.ai / krrish@berri.ai diff --git a/docs/my-website/docs/observability/logfire_integration.md b/docs/my-website/docs/observability/logfire_integration.md index a1bd43a4bc4..00652c0f1a1 100644 --- a/docs/my-website/docs/observability/logfire_integration.md +++ b/docs/my-website/docs/observability/logfire_integration.md @@ -63,5 +63,4 @@ response = litellm.completion( - [Schedule Demo 👋](https://calendly.com/d/4mp-gd3-k5k/berriai-1-1-onboarding-litellm-hosted-version) - [Community Discord 💭](https://discord.gg/wuPM9dRgDw) -- Our numbers 📞 +1 (770) 8783-106 / ‭+1 (412) 618-6238‬ - Our emails ✉️ ishaan@berri.ai / krrish@berri.ai diff --git a/docs/my-website/docs/observability/lunary_integration.md b/docs/my-website/docs/observability/lunary_integration.md index 8d28321c807..4f1dca9d4af 100644 --- a/docs/my-website/docs/observability/lunary_integration.md +++ b/docs/my-website/docs/observability/lunary_integration.md @@ -176,5 +176,4 @@ You can find more details about the different ways of making requests to the Lit - [Schedule Demo 👋](https://calendly.com/d/4mp-gd3-k5k/berriai-1-1-onboarding-litellm-hosted-version) - [Community Discord 💭](https://discord.gg/wuPM9dRgDw) -- Our numbers 📞 +1 (770) 8783-106 / ‭+1 (412) 618-6238‬ - Our emails ✉️ ishaan@berri.ai / krrish@berri.ai diff --git a/docs/my-website/docs/observability/opik_integration.md b/docs/my-website/docs/observability/opik_integration.md index d28c46f0b4b..5b5cbe0f185 100644 --- a/docs/my-website/docs/observability/opik_integration.md +++ b/docs/my-website/docs/observability/opik_integration.md @@ -261,5 +261,4 @@ All requests made with this key will automatically be tracked in the "TestProjec - [Schedule Demo 👋](https://calendly.com/d/4mp-gd3-k5k/berriai-1-1-onboarding-litellm-hosted-version) - [Community Discord 💭](https://discord.gg/wuPM9dRgDw) -- Our numbers 📞 +1 (770) 8783-106 / ‭+1 (412) 618-6238‬ - Our emails ✉️ ishaan@berri.ai / krrish@berri.ai diff --git a/docs/my-website/docs/observability/phoenix_integration.md b/docs/my-website/docs/observability/phoenix_integration.md index 191f1f8044a..67ba815557e 100644 --- a/docs/my-website/docs/observability/phoenix_integration.md +++ b/docs/my-website/docs/observability/phoenix_integration.md @@ -127,5 +127,4 @@ Depending on which Phoenix Cloud version or deployment you are using, you should - [Schedule Demo 👋](https://calendly.com/d/4mp-gd3-k5k/berriai-1-1-onboarding-litellm-hosted-version) - [Community Discord 💭](https://discord.gg/wuPM9dRgDw) -- Our numbers 📞 +1 (770) 8783-106 / ‭+1 (412) 618-6238‬ - Our emails ✉️ ishaan@berri.ai / krrish@berri.ai diff --git a/docs/my-website/docs/observability/promptlayer_integration.md b/docs/my-website/docs/observability/promptlayer_integration.md index 7f62a316972..9462e755f74 100644 --- a/docs/my-website/docs/observability/promptlayer_integration.md +++ b/docs/my-website/docs/observability/promptlayer_integration.md @@ -84,5 +84,4 @@ Credits to [Nick Bradford](https://github.com/nsbradford), from [Vim-GPT](https: - [Schedule Demo 👋](https://calendly.com/d/4mp-gd3-k5k/berriai-1-1-onboarding-litellm-hosted-version) - [Community Discord 💭](https://discord.gg/wuPM9dRgDw) -- Our numbers 📞 +1 (770) 8783-106 / ‭+1 (412) 618-6238‬ - Our emails ✉️ ishaan@berri.ai / krrish@berri.ai \ No newline at end of file diff --git a/docs/my-website/docs/observability/slack_integration.md b/docs/my-website/docs/observability/slack_integration.md index 0ca7f616683..468d8b5945c 100644 --- a/docs/my-website/docs/observability/slack_integration.md +++ b/docs/my-website/docs/observability/slack_integration.md @@ -101,5 +101,4 @@ response = litellm.completion( - [Schedule Demo 👋](https://calendly.com/d/4mp-gd3-k5k/berriai-1-1-onboarding-litellm-hosted-version) - [Community Discord 💭](https://discord.gg/wuPM9dRgDw) -- Our numbers 📞 +1 (770) 8783-106 / ‭+1 (412) 618-6238‬ - Our emails ✉️ ishaan@berri.ai / krrish@berri.ai diff --git a/docs/my-website/docs/observability/sumologic_integration.md b/docs/my-website/docs/observability/sumologic_integration.md index c30ee94dad4..87e20ca57ed 100644 --- a/docs/my-website/docs/observability/sumologic_integration.md +++ b/docs/my-website/docs/observability/sumologic_integration.md @@ -328,5 +328,4 @@ If you get authentication errors, regenerate the HTTP Source URL in Sumo Logic: - [Schedule Demo 👋](https://calendly.com/d/4mp-gd3-k5k/berriai-1-1-onboarding-litellm-hosted-version) - [Community Discord 💭](https://discord.gg/wuPM9dRgDw) -- Our numbers 📞 +1 (770) 8783-106 / ‭+1 (412) 618-6238‬ - Our emails ✉️ ishaan@berri.ai / krrish@berri.ai diff --git a/docs/my-website/docs/observability/supabase_integration.md b/docs/my-website/docs/observability/supabase_integration.md index fd3f1c3d5a0..c29871d752f 100644 --- a/docs/my-website/docs/observability/supabase_integration.md +++ b/docs/my-website/docs/observability/supabase_integration.md @@ -105,5 +105,4 @@ litellm.modify_integration("supabase",{"table_name": "litellm_logs"}) - [Schedule Demo 👋](https://calendly.com/d/4mp-gd3-k5k/berriai-1-1-onboarding-litellm-hosted-version) - [Community Discord 💭](https://discord.gg/wuPM9dRgDw) -- Our numbers 📞 +1 (770) 8783-106 / ‭+1 (412) 618-6238‬ - Our emails ✉️ ishaan@berri.ai / krrish@berri.ai diff --git a/docs/my-website/docs/observability/wandb_integration.md b/docs/my-website/docs/observability/wandb_integration.md index 37057f43db5..3c1a3363957 100644 --- a/docs/my-website/docs/observability/wandb_integration.md +++ b/docs/my-website/docs/observability/wandb_integration.md @@ -57,5 +57,4 @@ response = litellm.completion( - [Schedule Demo 👋](https://calendly.com/d/4mp-gd3-k5k/berriai-1-1-onboarding-litellm-hosted-version) - [Community Discord 💭](https://discord.gg/wuPM9dRgDw) -- Our numbers 📞 +1 (770) 8783-106 / ‭+1 (412) 618-6238‬ - Our emails ✉️ ishaan@berri.ai / krrish@berri.ai \ No newline at end of file diff --git a/docs/my-website/docs/providers/gemini.md b/docs/my-website/docs/providers/gemini.md index c8c9114ea87..87ab5ad40f4 100644 --- a/docs/my-website/docs/providers/gemini.md +++ b/docs/my-website/docs/providers/gemini.md @@ -11,6 +11,7 @@ import TabItem from '@theme/TabItem'; | Provider Doc | [Google AI Studio ↗](https://aistudio.google.com/) | | API Endpoint for Provider | https://generativelanguage.googleapis.com | | Supported OpenAI Endpoints | `/chat/completions`, [`/embeddings`](../embedding/supported_embedding#gemini-ai-embedding-models), `/completions`, [`/videos`](./gemini/videos.md), [`/images/edits`](../image_edits.md) | +| Lyria (music) | [Cost map & notes](./gemini/music.md) | | Pass-through Endpoint | [Supported](../pass_through/google_ai_studio.md) |
diff --git a/docs/my-website/docs/providers/gemini/music.md b/docs/my-website/docs/providers/gemini/music.md new file mode 100644 index 00000000000..f3968f2db39 --- /dev/null +++ b/docs/my-website/docs/providers/gemini/music.md @@ -0,0 +1,28 @@ +# Gemini — Lyria (music generation) + +Google Lyria 3 preview models are listed in LiteLLM’s [model cost map](https://github.com/BerriAI/litellm/blob/main/model_prices_and_context_window.json) under the `gemini/` provider for metadata and spend tracking. + +| Property | Details | +|----------|---------| +| Provider route | `gemini/` | +| Models | `gemini/lyria-3-clip-preview`, `gemini/lyria-3-pro-preview` | +| Provider docs | [Gemini API pricing / models ↗](https://ai.google.dev/gemini-api/docs/pricing) | + +## Models + +| Model | Notes | +|-------|--------| +| `gemini/lyria-3-clip-preview` | ~30s clip; paid tier listed as per generated song in Google’s pricing | +| `gemini/lyria-3-pro-preview` | Full song; paid tier listed as per generated song in Google’s pricing | + +Input context limit in the cost map: **131,072** tokens. For modalities, limits, and features, see [Google’s Gemini API docs ↗](https://ai.google.dev/gemini-api/docs/models). + +## LiteLLM behavior + +- **Cost map**: Per-song paid pricing is stored as `output_cost_per_image` on those entries (flat per generation unit). Token-based completion cost may not reflect music billing until a dedicated path exists. +- **API calls**: Use the Gemini API as documented by Google. LiteLLM does not ship a separate `music_generation` helper like Veo’s `video_generation`. + +## Auth + +Same as other Gemini API models: `GEMINI_API_KEY` or `GOOGLE_API_KEY`. + diff --git a/docs/my-website/docs/providers/openai.md b/docs/my-website/docs/providers/openai.md index 2907cdf9f47..1f4a1687e8b 100644 --- a/docs/my-website/docs/providers/openai.md +++ b/docs/my-website/docs/providers/openai.md @@ -581,6 +581,90 @@ curl -X POST 'http://0.0.0.0:4000/chat/completions' \ See [OpenAI Reasoning documentation](https://platform.openai.com/docs/guides/reasoning) for more details on organization verification requirements. +### Multi-turn Conversations with `reasoning_items` + +For multi-turn conversations you need `reasoning_items`: structured blocks that include the `encrypted_content` token OpenAI uses to restore reasoning state on the next request. Pass `include=["reasoning.encrypted_content"]` on every call where you want that token returned. + + + + +```python showLineNumbers title="Non-streaming: round-trip reasoning_items" +import litellm + +messages = [{"role": "user", "content": "Solve this step by step: 2 + 2"}] + +# Turn 1 — get reasoning_items (encrypted_content); +response = litellm.completion( + model="openai/responses/gpt-5-mini", + messages=messages, + reasoning_effort="low", + include=["reasoning.encrypted_content"], +) + +assistant_msg = response.choices[0].message + +# Turn 2 — pass reasoning_items back; LiteLLM converts to the correct Responses API format +messages.append({ + "role": "assistant", + "content": assistant_msg.content, + "reasoning_items": assistant_msg.reasoning_items, +}) +messages.append({"role": "user", "content": "Now summarize your reasoning."}) + +response2 = litellm.completion( + model="openai/responses/gpt-5-mini", + messages=messages, + reasoning_effort="low", + include=["reasoning.encrypted_content"], +) +``` + + + + +`reasoning_items` (with `encrypted_content`) arrive on the final chunk when the full response completes: + +```python showLineNumbers title="Streaming: collect and round-trip reasoning_items" +import litellm + +messages = [{"role": "user", "content": "Solve this step by step: 2 + 2"}] + +collected_content = [] +collected_reasoning_items = [] + +stream = litellm.completion( + model="openai/responses/gpt-5-mini", + messages=messages, + stream=True, + reasoning_effort="low", + include=["reasoning.encrypted_content"], +) + +for chunk in stream: + delta = chunk.choices[0].delta + if delta.content: + collected_content.append(delta.content) + if getattr(delta, "reasoning_items", None): + collected_reasoning_items.extend(delta.reasoning_items) + +messages.append({ + "role": "assistant", + "content": "".join(collected_content), + "reasoning_items": collected_reasoning_items or None, +}) +messages.append({"role": "user", "content": "Continue the conversation."}) + +response2 = litellm.completion( + model="openai/responses/gpt-5-mini", + messages=messages, + reasoning_effort="low", + include=["reasoning.encrypted_content"], +) +``` + + + + ### Verbosity Control for GPT-5 Models The `verbosity` parameter controls the length and detail of responses from GPT-5 family models. It accepts three values: `"low"`, `"medium"`, or `"high"`. diff --git a/docs/my-website/docs/proxy/config_settings.md b/docs/my-website/docs/proxy/config_settings.md index 02f5c2be9c7..cc9090c2de6 100644 --- a/docs/my-website/docs/proxy/config_settings.md +++ b/docs/my-website/docs/proxy/config_settings.md @@ -279,6 +279,33 @@ router_settings: | forward_client_headers_to_llm_api | boolean | If true, forwards the client headers (any `x-` headers and `anthropic-beta` headers) to the backend LLM call | | maximum_spend_logs_retention_period | str | Used to set the max retention time for spend logs in the db, after which they will be auto-purged | | maximum_spend_logs_retention_interval | str | Used to set the interval in which the spend log cleanup task should run in. | +| alert_type_config | dict | Configuration mapping alert types to their handler settings | +| always_include_stream_usage | boolean | If true, includes usage metrics in every streaming response chunk | +| auto_redirect_ui_login_to_sso | boolean | If true, automatically redirects UI login page to SSO provider | +| control_plane_url | string | URL of the control plane for cross-instance state sharing | +| custom_auth_run_common_checks | boolean | If true, runs standard auth validation checks alongside custom auth handlers | +| custom_ui_sso_sign_in_handler | string | Custom handler for SSO sign-in logic in the UI | +| database_connection_pool_timeout | integer | Database connection pool timeout in seconds | +| disable_error_logs | boolean | If true, suppresses error tracking and storage in the database | +| enable_health_check_routing | boolean | If true, enables health check-driven request routing to avoid unhealthy deployments | +| enable_mcp_registry | boolean | If true, enables access to the centralized MCP server registry | +| enforce_rbac | boolean | If true, enables role-based access control (RBAC) for all proxy operations | +| forward_llm_provider_auth_headers | boolean | If true, forwards provider-specific auth headers to LLM API calls | +| health_check_concurrency | integer | Maximum number of concurrent health check operations | +| health_check_staleness_threshold | integer | Maximum age in seconds for health check results before marking deployments as stale | +| maximum_spend_logs_cleanup_cron | string | Cron expression for scheduling automatic spend log cleanup tasks | +| mcp_client_side_auth_header_name | string | HTTP header name for client-side MCP server credentials | +| mcp_internal_ip_ranges | list | CIDR ranges considered internal for non-public MCP server access control | +| mcp_required_fields | list | List of required field names for MCP server submissions | +| mcp_trusted_proxy_ranges | list | CIDR ranges of proxies trusted to forward X-Forwarded-For headers for MCP | +| require_end_user_mcp_access_defined | boolean | If true, requires end users to have explicit MCP access permissions defined | +| role_permissions | list | List of role-based permission configurations | +| search_tools | list | List of search tool configurations for enabling web search capabilities | +| token_rate_limit_type | string | Rate limit counting method: "total", "output", or "input" tokens | +| use_redis_transaction_buffer | boolean | If true, buffers database transactions in Redis before writing | +| use_shared_health_check | boolean | If true, uses Redis-backed shared health check state across multiple proxy instances | +| user_header_mappings | dict | Map custom request headers to user IDs using lookup rules | +| user_header_name | string | HTTP header name to extract user identity from requests | ### router_settings - Reference @@ -367,6 +394,8 @@ router_settings: | ignore_invalid_deployments | boolean | If true, ignores invalid deployments. Default for proxy is True - to prevent invalid models from blocking other models from being loaded. | | search_tools | List[SearchToolTypedDict] | List of search tool configurations for Search API integration. Each tool specifies a search_tool_name and litellm_params with search_provider, api_key, api_base, etc. [Further Docs](../search/index.md) | | guardrail_list | List[GuardrailTypedDict] | List of guardrail configurations for guardrail load balancing. Enables load balancing across multiple guardrail deployments with the same guardrail_name. [Further Docs](./guardrails/guardrail_load_balancing.md) | +| enable_health_check_routing | boolean | If true, enables health check-driven deployment filtering to avoid routing requests to unhealthy deployments | +| health_check_staleness_threshold | integer | Maximum age in seconds for cached health check results before marking deployments as stale | ### environment variables - Reference @@ -804,6 +833,7 @@ router_settings: | LITELLM_OTEL_INTEGRATION_ENABLE_EVENTS | Optionally enable semantic logs for OTEL | LITELLM_OTEL_INTEGRATION_ENABLE_METRICS | Optionally enable emantic metrics for OTEL | LITELLM_ENABLE_PYROSCOPE | If true, enables Pyroscope CPU profiling. Profiles are sent to PYROSCOPE_SERVER_ADDRESS. Off by default. See [Pyroscope profiling](/proxy/pyroscope_profiling). +| LITELLM_ENABLE_TEAM_STALE_ALIAS_BYPASS | When `true`, if a team's legacy `model_aliases` entry maps a public model name to an internal `model_name__` deployment, pre-call handling can skip that rewrite when team-scoped sibling deployments exist for the public name—so load balancing / `order` apply across siblings. Default is `false` for backwards compatibility. See [Team-scoped models and legacy aliases](./load_balancing#team-scoped-models-and-legacy-model_aliases). When stale aliases are detected and this flag is off, the proxy may log a one-time warning. | PYROSCOPE_APP_NAME | Application name reported to Pyroscope. Required when LITELLM_ENABLE_PYROSCOPE is true. No default. | PYROSCOPE_SERVER_ADDRESS | Pyroscope server URL to send profiles to. Required when LITELLM_ENABLE_PYROSCOPE is true. No default. | PYROSCOPE_SAMPLE_RATE | Optional. Sample rate for Pyroscope profiling (integer). No default; when unset, the pyroscope-io library default is used. diff --git a/docs/my-website/docs/proxy/cost_tracking.md b/docs/my-website/docs/proxy/cost_tracking.md index f28eec287d4..f9e22cfecd3 100644 --- a/docs/my-website/docs/proxy/cost_tracking.md +++ b/docs/my-website/docs/proxy/cost_tracking.md @@ -163,7 +163,7 @@ Use this when you want non-proxy admins to access `/spend` endpoints :::info -Schedule a [meeting with us to get your Enterprise License](https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions) +Schedule a [meeting with us to get your Enterprise License](https://enterprise.litellm.ai/demo) ::: diff --git a/docs/my-website/docs/proxy/email.md b/docs/my-website/docs/proxy/email.md index 86a79cbcfc8..ba737c6782c 100644 --- a/docs/my-website/docs/proxy/email.md +++ b/docs/my-website/docs/proxy/email.md @@ -203,7 +203,7 @@ After regenerating the key, the user will receive an email notification with: :::info -Customizing Email Branding is an Enterprise Feature [Get in touch with us for a Free Trial](https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions) +Customizing Email Branding is an Enterprise Feature [Get in touch with us for a Free Trial](https://enterprise.litellm.ai/demo) ::: diff --git a/docs/my-website/docs/proxy/enterprise.md b/docs/my-website/docs/proxy/enterprise.md index 4b525837a20..09b103ca4a0 100644 --- a/docs/my-website/docs/proxy/enterprise.md +++ b/docs/my-website/docs/proxy/enterprise.md @@ -5,7 +5,7 @@ import TabItem from '@theme/TabItem'; # ✨ Enterprise Features :::tip -To get a license, get in touch with us [here](https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions) +To get a license, get in touch with us [here](https://enterprise.litellm.ai/demo) ::: diff --git a/docs/my-website/docs/proxy/guardrails/aporia_api.md b/docs/my-website/docs/proxy/guardrails/aporia_api.md index ceafc19a1cc..e6ff0d5fed3 100644 --- a/docs/my-website/docs/proxy/guardrails/aporia_api.md +++ b/docs/my-website/docs/proxy/guardrails/aporia_api.md @@ -139,7 +139,7 @@ curl -i http://localhost:4000/v1/chat/completions \ :::info -✨ This is an Enterprise only feature [Contact us to get a free trial](https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions) +✨ This is an Enterprise only feature [Contact us to get a free trial](https://enterprise.litellm.ai/demo) ::: diff --git a/docs/my-website/docs/proxy/guardrails/custom_guardrail.md b/docs/my-website/docs/proxy/guardrails/custom_guardrail.md index 638cae9c835..37579ad870d 100644 --- a/docs/my-website/docs/proxy/guardrails/custom_guardrail.md +++ b/docs/my-website/docs/proxy/guardrails/custom_guardrail.md @@ -417,7 +417,7 @@ curl -i -X POST http://localhost:4000/v1/chat/completions \ :::info -✨ This is an Enterprise only feature [Contact us to get a free trial](https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions) +✨ This is an Enterprise only feature [Contact us to get a free trial](https://enterprise.litellm.ai/demo) ::: diff --git a/docs/my-website/docs/proxy/guardrails/guardrails_ai.md b/docs/my-website/docs/proxy/guardrails/guardrails_ai.md index 55d586aee7b..19ae34014a4 100644 --- a/docs/my-website/docs/proxy/guardrails/guardrails_ai.md +++ b/docs/my-website/docs/proxy/guardrails/guardrails_ai.md @@ -59,7 +59,7 @@ curl -i http://localhost:4000/v1/chat/completions \ :::info -✨ This is an Enterprise only feature [Contact us to get a free trial](https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions) +✨ This is an Enterprise only feature [Contact us to get a free trial](https://enterprise.litellm.ai/demo) ::: diff --git a/docs/my-website/docs/proxy/health.md b/docs/my-website/docs/proxy/health.md index 2764a6f0d4f..530bea3d06b 100644 --- a/docs/my-website/docs/proxy/health.md +++ b/docs/my-website/docs/proxy/health.md @@ -314,6 +314,89 @@ general_settings: health_check_details: False ``` +## Health Check Driven Routing + +By default, background health checks are observability-only — they populate the `/health` endpoint but don't affect routing. Unhealthy deployments still receive traffic until request failures trigger cooldown. + +With `enable_health_check_routing: true`, the router **excludes deployments that failed their last background health check** before selecting a candidate. This gives you proactive failover instead of reactive cooldown. + +### How it works + +1. Background health checks run on their configured interval +2. After each cycle, every deployment is marked healthy or unhealthy +3. On each incoming request, the router filters out unhealthy deployments **before** cooldown filtering and load balancing +4. If all deployments are unhealthy, the filter is bypassed (safety net — never causes a total outage) +5. If health state is stale (older than `health_check_staleness_threshold`), it is ignored + +### Quick start + +```yaml +model_list: + - model_name: gpt-4 + litellm_params: + model: openai/gpt-4 + api_key: os.environ/OPENAI_API_KEY + - model_name: gpt-4 + litellm_params: + model: openai/gpt-4 + api_key: os.environ/OPENAI_API_KEY_SECONDARY + +general_settings: + background_health_checks: true + health_check_interval: 60 + enable_health_check_routing: true +``` + +### Configuration + +| Setting | Where | Default | Description | +|---------|-------|---------|-------------| +| `enable_health_check_routing` | `general_settings` | `false` | Enable/disable health-check-driven routing | +| `health_check_staleness_threshold` | `general_settings` | `health_check_interval * 2` | Seconds before health state is considered stale and ignored | +| `background_health_checks` | `general_settings` | `false` | Must be `true` for health check routing to work | +| `health_check_interval` | `general_settings` | `300` | Seconds between health check cycles | + +### Interaction with cooldown + +Health check filtering and cooldown are **additive**. A deployment can be excluded by either mechanism: + +- **Health check filter** — proactive, runs on the configured interval, excludes deployments that failed the last check +- **Cooldown** — reactive, triggered by request failures, excludes deployments for a short TTL + +This means request failures still provide fast detection between health check intervals. + +### Staleness + +If a health check result is older than `health_check_staleness_threshold`, it is ignored and the deployment is treated as eligible. This prevents stale data from permanently excluding a deployment if the health check loop stops or slows down. + +The default staleness threshold is `health_check_interval * 2`. For a 60s interval, health state expires after 120s. + +### Example: custom staleness + +```yaml +general_settings: + background_health_checks: true + health_check_interval: 30 + enable_health_check_routing: true + health_check_staleness_threshold: 90 # ignore health state older than 90s +``` + +### Debugging + +Run the proxy with `--detailed_debug` and look for: + +``` +health_check_routing_state_updated healthy=3 unhealthy=1 +``` + +This is logged after each health check cycle when routing state is written. + +If the safety net triggers (all deployments unhealthy), you'll see: + +``` +All deployments marked unhealthy by health checks, bypassing health filter +``` + ## Health Check Timeout The health check timeout is set in `litellm/constants.py` and defaults to 60 seconds. diff --git a/docs/my-website/docs/proxy/high_availability_control_plane.md b/docs/my-website/docs/proxy/high_availability_control_plane.md index 324fba3a180..4cc6d2952fb 100644 --- a/docs/my-website/docs/proxy/high_availability_control_plane.md +++ b/docs/my-website/docs/proxy/high_availability_control_plane.md @@ -42,9 +42,9 @@ The **High Availability Control Plane** takes a different approach: -The **control plane** is a LiteLLM instance that serves the admin UI and knows about all the workers. It does not proxy LLM requests, it is purely for administration. +The **control plane** is a LiteLLM instance that serves the admin UI and knows about all the workers. It is **not a router** — it does not proxy or route any LLM requests. It exists purely so admins can switch between workers and manage them from a single UI. -Each **worker** is a fully independent LiteLLM proxy that handles LLM requests for its region or team. Workers have their own users, keys, teams, and budgets. +Each **worker** is a fully independent LiteLLM proxy that handles LLM requests for its region or team. Workers have their own database, Redis, users, keys, teams, and budgets. No infrastructure is shared between workers. ## Setup diff --git a/docs/my-website/docs/proxy/ip_address.md b/docs/my-website/docs/proxy/ip_address.md index 8f042d9f183..4c469b81e0b 100644 --- a/docs/my-website/docs/proxy/ip_address.md +++ b/docs/my-website/docs/proxy/ip_address.md @@ -3,7 +3,7 @@ :::info -You need a LiteLLM License to unlock this feature. [Grab time](https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions), to get one today! +You need a LiteLLM License to unlock this feature. [Grab time](https://enterprise.litellm.ai/demo), to get one today! ::: diff --git a/docs/my-website/docs/proxy/jwt_key_mapping.md b/docs/my-website/docs/proxy/jwt_key_mapping.md new file mode 100644 index 00000000000..452bf821016 --- /dev/null +++ b/docs/my-website/docs/proxy/jwt_key_mapping.md @@ -0,0 +1,318 @@ +# JWT → Virtual Key Mapping + +:::info Enterprise + +JWT → Virtual Key Mapping is an Enterprise feature. + +[Get a free trial](https://enterprise.litellm.ai/demo) + +::: + +Map JWT tokens to LiteLLM virtual keys — so every JWT client gets the same granular controls as a virtual key: model restrictions, spend limits, rate limits, guardrails, and full spend tracking. + +**Why this matters:** Standard JWT auth maps a JWT to a *team*. That's a shared boundary — all clients under a team share the same limits. With JWT → Virtual Key Mapping, each individual JWT client (identified by a claim like `client_id`, `azp`, or `sub`) maps to its own virtual key. You get per-client accountability without issuing API keys to your users. + +**Common use case:** Your company uses SSO/OIDC. Developers use Claude Code with their identity tokens. You want to enforce per-developer model access and spend limits without giving each person a LiteLLM API key. + +--- + +## How It Works + +```mermaid +sequenceDiagram + participant Client as Client (Claude Code / API) + participant Proxy as LiteLLM Proxy + participant OIDC as OIDC Provider + participant DB as Mapping Table + + Client->>Proxy: POST /v1/chat/completions
Authorization: Bearer + + Proxy->>OIDC: Verify JWT signature + OIDC-->>Proxy: Valid ✓ + + Proxy->>Proxy: Extract claim
(e.g. client_id = "alice@corp.com") + + Proxy->>DB: Look up (claim_name, claim_value) + alt Mapping found + DB-->>Proxy: virtual_key_id = sk-abc123 + Proxy->>Proxy: Apply virtual key permissions
(models, budget, rate limits) + Proxy-->>Client: 200 OK + else No mapping — fallback_team_mapping + Proxy->>Proxy: Fall through to team JWT auth + Proxy-->>Client: 200 OK + else No mapping — reject + Proxy-->>Client: 403 Forbidden + else No mapping — auto_register + Proxy->>DB: Create new virtual key + mapping + Proxy-->>Client: 200 OK + end +``` + +--- + +## Setup + +### Prerequisites + +Complete [OIDC JWT Auth setup](./token_auth.md) first — you need `JWT_PUBLIC_KEY_URL` configured and `enable_jwt_auth: True` in your proxy config. + +### Step 1. Configure the JWT claim to map on + +Add `jwt_client_id_field` to your `litellm_jwtauth` config. This is the JWT claim LiteLLM uses as the lookup key: + +```yaml +general_settings: + master_key: sk-1234 + enable_jwt_auth: True + litellm_jwtauth: + team_id_jwt_field: "team_id" # existing team mapping (optional) + user_id_jwt_field: "sub" + jwt_client_id_field: "client_id" # 👈 claim used for key mapping + unregistered_jwt_client_behavior: "fallback_team_mapping" # see below +``` + +**`unregistered_jwt_client_behavior`** controls what happens when a JWT has no registered mapping: + +| Value | Behavior | +|-------|----------| +| `fallback_team_mapping` | Fall through to team-based JWT auth (default — backward compatible) | +| `reject` | Return 403 if no mapping found | +| `auto_register` | Auto-create a virtual key + mapping on first encounter | + +### Step 2. Register a JWT client → virtual key mapping + +**Option A: Single call (creates key + mapping atomically)** + +```bash +curl -X POST 'http://0.0.0.0:4000/jwt_client/new' \ + -H 'Authorization: Bearer ' \ + -H 'Content-Type: application/json' \ + -d '{ + "jwt_claim_name": "client_id", + "jwt_claim_value": "dev-alice", + "models": ["claude-sonnet-4-5", "claude-haiku-4-5"], + "max_budget": 50.0, + "budget_duration": "30d", + "rpm_limit": 100, + "tpm_limit": 50000, + "team_id": "engineering" + }' +``` + +Response includes the virtual key token (only shown on creation): + +```json +{ + "key": "sk-abc123...", + "key_id": "key_123", + "mapping_id": "mapping_456", + "jwt_claim_name": "client_id", + "jwt_claim_value": "dev-alice" +} +``` + +**Option B: Map an existing virtual key** + +```bash +curl -X POST 'http://0.0.0.0:4000/jwt/key/mapping/new' \ + -H 'Authorization: Bearer ' \ + -H 'Content-Type: application/json' \ + -d '{ + "jwt_claim_name": "client_id", + "jwt_claim_value": "dev-alice", + "virtual_key_id": "key_123" + }' +``` + +### Step 3. Test it + +```bash +# Get a JWT from your OIDC provider (must have client_id: dev-alice) +JWT_TOKEN="eyJhbG..." + +curl -X POST 'http://0.0.0.0:4000/v1/chat/completions' \ + -H "Authorization: Bearer $JWT_TOKEN" \ + -H 'Content-Type: application/json' \ + -d '{ + "model": "claude-sonnet-4-5", + "messages": [{"role": "user", "content": "Hello"}] + }' +``` + +The request is now tracked against `dev-alice`'s virtual key — spend, rate limits, and model access enforced per-client. + +--- + +## Walkthrough: Admin grants granular access, team uses Claude Code + +This is the full flow for an engineering team using Claude Code with company SSO. + +### Admin setup + +**1. Create a team for engineering** + +```bash +curl -X POST 'http://0.0.0.0:4000/team/new' \ + -H 'Authorization: Bearer ' \ + -H 'Content-Type: application/json' \ + -d '{ + "team_alias": "engineering", + "models": ["claude-sonnet-4-5", "claude-haiku-4-5"] + }' +``` + +**2. Register each developer with their own key and spend limit** + +```bash +# Alice — senior eng, higher budget +curl -X POST 'http://0.0.0.0:4000/jwt_client/new' \ + -H 'Authorization: Bearer ' \ + -H 'Content-Type: application/json' \ + -d '{ + "jwt_claim_name": "client_id", + "jwt_claim_value": "alice@corp.com", + "team_id": "engineering", + "models": ["claude-sonnet-4-5", "claude-haiku-4-5"], + "max_budget": 200.0, + "budget_duration": "30d", + "rpm_limit": 200 + }' + +# Bob — contractor, tighter limits +curl -X POST 'http://0.0.0.0:4000/jwt_client/new' \ + -H 'Authorization: Bearer ' \ + -H 'Content-Type: application/json' \ + -d '{ + "jwt_claim_name": "client_id", + "jwt_claim_value": "bob@contractor.com", + "team_id": "engineering", + "models": ["claude-haiku-4-5"], + "max_budget": 20.0, + "budget_duration": "30d", + "rpm_limit": 30 + }' +``` + +**3. Configure Claude Code to use the proxy** + +Set the proxy as the API base in your team's Claude Code config: + +```bash +# Point Claude Code at the LiteLLM proxy instead of Anthropic directly. +# ANTHROPIC_API_KEY here is the bearer token sent to the proxy — set it to +# the user's SSO/OIDC JWT token (obtained from your IdP at login). +export ANTHROPIC_API_KEY="" +export ANTHROPIC_BASE_URL="http://your-litellm-proxy:4000" +``` + +Or in `~/.claude/settings.json`: + +```json +{ + "env": { + "ANTHROPIC_BASE_URL": "http://your-litellm-proxy:4000" + } +} +``` + +**4. Developers authenticate with SSO as usual** + +When Alice runs Claude Code, her JWT (issued by your IdP with `client_id: alice@corp.com`) goes to the proxy. LiteLLM looks up the mapping, finds her virtual key, and enforces her specific limits — her $200/month budget, 200 RPM cap, and access to Sonnet and Haiku only. + +Bob's token maps to his own key — $20/month, Haiku only, 30 RPM. + +No API keys distributed. No shared limits. Full per-developer spend visibility in the LiteLLM dashboard. + +--- + +## Managing mappings + +**View a mapping + its key settings** + +```bash +curl 'http://0.0.0.0:4000/jwt/key/mapping/info?jwt_claim_name=client_id&jwt_claim_value=alice@corp.com' \ + -H 'Authorization: Bearer ' +``` + +Response includes the linked key's `models`, `max_budget`, `spend`, `rpm_limit`, `expires`, etc. + +**Update a mapping** + +```bash +curl -X POST 'http://0.0.0.0:4000/jwt_client/update' \ + -H 'Authorization: Bearer ' \ + -H 'Content-Type: application/json' \ + -d '{ + "jwt_claim_name": "client_id", + "jwt_claim_value": "alice@corp.com", + "max_budget": 300.0 + }' +``` + +**Delete a mapping** + +```bash +curl -X DELETE 'http://0.0.0.0:4000/jwt/key/mapping/delete' \ + -H 'Authorization: Bearer ' \ + -H 'Content-Type: application/json' \ + -d '{ + "jwt_claim_name": "client_id", + "jwt_claim_value": "alice@corp.com" + }' +``` + +--- + +## Security + +JWT-bound keys are locked down: + +- Non-admin users cannot call `/key/update`, `/key/delete`, or `/key/regenerate` on a JWT-bound key. These return 403. +- JWT-bound keys are automatically restricted to `llm_api_routes` — they can make LLM calls but cannot manage other keys or admin resources. +- Only proxy admins can create, update, or delete mappings. + +--- + +## Multi-IdP support + +If you have users across multiple identity providers that share the same claim values (e.g. two services both have `sub: user-123` from different issuers), set `issuer` when creating the mapping: + +```bash +curl -X POST 'http://0.0.0.0:4000/jwt_client/new' \ + -H 'Authorization: Bearer ' \ + -H 'Content-Type: application/json' \ + -d '{ + "jwt_claim_name": "sub", + "jwt_claim_value": "user-123", + "issuer": "https://idp-a.corp.com", + "models": ["claude-sonnet-4-5"], + "max_budget": 50.0 + }' +``` + +Mappings are unique per `(claim_name, claim_value, issuer)` — so `user-123` from IdP A and `user-123` from IdP B resolve to different virtual keys. + +--- + +## What JWT clients can and can't do vs virtual keys + +| Capability | Virtual Key | JWT → Key Mapping | +|---|---|---| +| Per-client model access | ✅ | ✅ | +| Per-client spend budget | ✅ | ✅ | +| Per-client RPM/TPM limits | ✅ | ✅ | +| Team membership | ✅ | ✅ | +| Spend tracking in dashboard | ✅ | ✅ | +| Guardrails | ✅ | ✅ | +| Key rotation | ✅ | ✅ (admin only) | +| Key expiry | ✅ | ✅ | +| No API key distribution needed | ❌ | ✅ | +| Works with existing SSO/OIDC | ❌ | ✅ | + +--- + +## Related + +- [OIDC JWT Auth](./token_auth.md) — base JWT auth setup required before using this feature +- [Virtual Keys](./virtual_keys.md) — full virtual key documentation +- [Access Control](./access_control.md) — model and team access control diff --git a/docs/my-website/docs/proxy/load_balancing.md b/docs/my-website/docs/proxy/load_balancing.md index 74b3e8a5117..93f3d944340 100644 --- a/docs/my-website/docs/proxy/load_balancing.md +++ b/docs/my-website/docs/proxy/load_balancing.md @@ -324,17 +324,58 @@ model_list: litellm_params: model: azure/gpt-4-fallback api_key: os.environ/AZURE_API_KEY_2 - order: 2 # 👈 Used when order=1 is unavailable - -router_settings: - enable_pre_call_checks: true # 👈 Required for 'order' to work + order: 2 # 👈 Used when order=1 fails ``` -:::important -The `order` parameter requires `enable_pre_call_checks: true` in `router_settings`. -::: +### How order-based fallback works -If `order=1` deployment is unavailable (e.g., rate-limited), the router falls back to `order=2` deployments. +When a request to an `order=1` deployment fails (connection error, 404, 429, etc.), the router automatically tries `order=2` deployments, then `order=3`, and so on. Each order level gets its own set of retries before escalating to the next. + +If all order levels are exhausted, the router falls through to any configured [model-level fallbacks](#fallbacks). + +```yaml +model_list: + - model_name: gpt-4 + litellm_params: + model: azure/gpt-4-primary + api_key: os.environ/AZURE_API_KEY + order: 1 + + - model_name: gpt-4 + litellm_params: + model: azure/gpt-4-secondary + api_key: os.environ/AZURE_API_KEY_2 + order: 2 + + - model_name: gpt-4-fallback + litellm_params: + model: openai/gpt-4 + api_key: os.environ/OPENAI_API_KEY + +router_settings: + fallbacks: + - gpt-4: + - gpt-4-fallback # tried after all order levels fail +``` + +The fallback chain for the above config: `order=1` → `order=2` → `gpt-4-fallback`. + +For 429 (rate limit) errors specifically, the failed deployment is immediately placed on cooldown. If all `order=1` deployments are on cooldown, the router picks `order=2` deployments directly during retries without waiting for the fallback path. + +### Team-scoped models and legacy `model_aliases` {#team-scoped-models-and-legacy-model_aliases} + +Team-scoped deployments are identified by `model_info.team_id` and `model_info.team_public_model_name`. Requests should use the **public** model name; the router resolves all sibling deployments (same public name, different `api_base` / `order`, etc.) for routing, failover, and deployment `order`. + +For router internals: when a `team_id` is in scope, optimized lookups key off `(team_id, team_public_model_name)`. If code passes an internal deployment id (e.g. `model_name__`) instead of the public name, routing still works via the usual deployment-name paths, but the team-specific fast path applies only to the public name. + +**Legacy teams:** Older proxy versions could persist `model_aliases` on the team row mapping a public name to a single internal deployment id (`model_name__`). On each request, pre-call logic may still rewrite `model` to that internal name **before** routing, which collapses to one deployment and can make newer sibling deployments unreachable. + +**Migration options:** + +1. **Recommended for upgrades:** Set environment variable `LITELLM_ENABLE_TEAM_STALE_ALIAS_BYPASS=true` so that when sibling team deployments exist for the public name, the stale alias rewrite is skipped and team-scoped routing (including `order` and failover) applies. See the [Environment variables](./config_settings) table in the proxy settings doc. +2. **Data cleanup:** Remove obsolete `model_aliases` entries for team public names from the team record in the database so only `team_public_model_name` + team model list drive access. + +If a stale alias is detected and the bypass is **not** enabled, the proxy may emit a **one-time** warning in logs explaining that sibling deployments may be unreachable until the flag is set or aliases are cleaned up. ### When You'll See Load Balancing in Action diff --git a/docs/my-website/docs/proxy/logging.md b/docs/my-website/docs/proxy/logging.md index 74a79776fbd..2f81498799a 100644 --- a/docs/my-website/docs/proxy/logging.md +++ b/docs/my-website/docs/proxy/logging.md @@ -1109,7 +1109,7 @@ Log LLM Logs to [Google Cloud Storage Buckets](https://cloud.google.com/storage? :::info -✨ This is an Enterprise only feature [Get Started with Enterprise here](https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions) +✨ This is an Enterprise only feature [Get Started with Enterprise here](https://enterprise.litellm.ai/demo) ::: @@ -1194,7 +1194,7 @@ Log LLM Logs/SpendLogs to [Google Cloud Storage PubSub Topic](https://cloud.goog :::info -✨ This is an Enterprise only feature [Get Started with Enterprise here](https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions) +✨ This is an Enterprise only feature [Get Started with Enterprise here](https://enterprise.litellm.ai/demo) ::: @@ -1497,7 +1497,7 @@ Log LLM Logs to [Azure Data Lake Storage](https://learn.microsoft.com/en-us/azur :::info -✨ This is an Enterprise only feature [Get Started with Enterprise here](https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions) +✨ This is an Enterprise only feature [Get Started with Enterprise here](https://enterprise.litellm.ai/demo) ::: diff --git a/docs/my-website/docs/proxy/multiple_admins.md b/docs/my-website/docs/proxy/multiple_admins.md index 8d39674df19..83d0c5863df 100644 --- a/docs/my-website/docs/proxy/multiple_admins.md +++ b/docs/my-website/docs/proxy/multiple_admins.md @@ -20,7 +20,7 @@ LiteLLM tracks changes to the following entities and actions: :::tip -Requires Enterprise License, Get in touch with us [here](https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions) +Requires Enterprise License, Get in touch with us [here](https://enterprise.litellm.ai/demo) ::: @@ -56,6 +56,40 @@ On the LiteLLM UI, navigate to Logs -> Audit Logs. You should see the audit log /> +## Export Audit Logs to External Storage + +You can export audit logs to an external storage backend (e.g. S3) in addition to storing them in the database. Logs are batched and uploaded asynchronously, so they do not block your proxy requests. + +### S3 Example + +Add `audit_log_callbacks` and `s3_callback_params` to your `litellm_settings`: + +```yaml +litellm_settings: + store_audit_logs: true + audit_log_callbacks: ["s3_v2"] + s3_callback_params: + s3_bucket_name: my-audit-logs-bucket # AWS Bucket Name + s3_region_name: us-west-2 # AWS Region + s3_aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID + s3_aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY + s3_path: litellm-audit # [OPTIONAL] prefix path in the bucket +``` + +Audit logs are written as JSON files to: + +``` +s3:///audit_logs//_.json +# or, when s3_path is set: +s3:////audit_logs//_.json +``` + +:::info + +Both `store_audit_logs: true` and `audit_log_callbacks` must be set. If `store_audit_logs` is not enabled, the callbacks will not fire. + +::: + ## Advanced ### Attribute Management changes to Users diff --git a/docs/my-website/docs/proxy/oauth2.md b/docs/my-website/docs/proxy/oauth2.md index 41c4110e447..204a01538cc 100644 --- a/docs/my-website/docs/proxy/oauth2.md +++ b/docs/my-website/docs/proxy/oauth2.md @@ -4,7 +4,7 @@ Use this if you want to use an Oauth2.0 token to make `/chat`, `/embeddings` req :::info -This is an Enterprise Feature - [get in touch with us if you want a free trial to test if this feature meets your needs]((https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions)) +This is an Enterprise Feature - [get in touch with us if you want a free trial to test if this feature meets your needs]((https://enterprise.litellm.ai/demo)) ::: diff --git a/docs/my-website/docs/proxy/pass_through.md b/docs/my-website/docs/proxy/pass_through.md index f47d7064140..700bfb0831d 100644 --- a/docs/my-website/docs/proxy/pass_through.md +++ b/docs/my-website/docs/proxy/pass_through.md @@ -422,6 +422,5 @@ general_settings: [Community Discord 💭](https://discord.gg/wuPM9dRgDw) -Our numbers 📞 +1 (770) 8783-106 / ‭+1 (412) 618-6238‬ Our emails ✉️ ishaan@berri.ai / krrish@berri.ai diff --git a/docs/my-website/docs/proxy/prod.md b/docs/my-website/docs/proxy/prod.md index 26cb484cbe9..d40a0343106 100644 --- a/docs/my-website/docs/proxy/prod.md +++ b/docs/my-website/docs/proxy/prod.md @@ -47,7 +47,7 @@ export LITELLM_LOG="ERROR" :::info -Need Help or want dedicated support ? Talk to a founder [here]: (https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions) +Need Help or want dedicated support ? Talk to a founder [here]: (https://enterprise.litellm.ai/demo) ::: diff --git a/docs/my-website/docs/proxy/public_routes.md b/docs/my-website/docs/proxy/public_routes.md index d5f3941751f..e53548349dc 100644 --- a/docs/my-website/docs/proxy/public_routes.md +++ b/docs/my-website/docs/proxy/public_routes.md @@ -5,7 +5,7 @@ import TabItem from '@theme/TabItem'; :::info -Requires a LiteLLM Enterprise License. [Get a free trial](https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions). +Requires a LiteLLM Enterprise License. [Get a free trial](https://enterprise.litellm.ai/demo). ::: diff --git a/docs/my-website/docs/proxy/tag_routing.md b/docs/my-website/docs/proxy/tag_routing.md index a1ae52e5e45..57d16a59b54 100644 --- a/docs/my-website/docs/proxy/tag_routing.md +++ b/docs/my-website/docs/proxy/tag_routing.md @@ -315,7 +315,7 @@ LiteLLM Proxy supports team-based tag routing, allowing you to associate specifi :::info -This is an enterprise feature, [Contact us here to get a free trial](https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions) +This is an enterprise feature, [Contact us here to get a free trial](https://enterprise.litellm.ai/demo) ::: diff --git a/docs/my-website/docs/proxy/team_logging.md b/docs/my-website/docs/proxy/team_logging.md index 2ad7e2a4a8e..3f57d0d6d8b 100644 --- a/docs/my-website/docs/proxy/team_logging.md +++ b/docs/my-website/docs/proxy/team_logging.md @@ -26,7 +26,7 @@ Team 3 -> Disabled Logging (for GDPR compliance) :::info -✨ This is an Enterprise only feature [Get Started with Enterprise here](https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions) +✨ This is an Enterprise only feature [Get Started with Enterprise here](https://enterprise.litellm.ai/demo) ::: @@ -248,7 +248,7 @@ Use the `/key/generate` or `/key/update` endpoints to add logging callbacks to a :::info -✨ This is an Enterprise only feature [Get Started with Enterprise here](https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions) +✨ This is an Enterprise only feature [Get Started with Enterprise here](https://enterprise.litellm.ai/demo) ::: diff --git a/docs/my-website/docs/proxy/team_model_add.md b/docs/my-website/docs/proxy/team_model_add.md index 7db59a3300e..bb4238055b5 100644 --- a/docs/my-website/docs/proxy/team_model_add.md +++ b/docs/my-website/docs/proxy/team_model_add.md @@ -5,7 +5,7 @@ This is an Enterprise feature. [Enterprise Pricing](https://www.litellm.ai/#pricing) -[Contact us here to get a free trial](https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions) +[Contact us here to get a free trial](https://enterprise.litellm.ai/demo) ::: diff --git a/docs/my-website/docs/proxy/token_auth.md b/docs/my-website/docs/proxy/token_auth.md index 7364ae0fb56..c287ab03644 100644 --- a/docs/my-website/docs/proxy/token_auth.md +++ b/docs/my-website/docs/proxy/token_auth.md @@ -11,11 +11,17 @@ Use JWT's to auth admins / users / projects into the proxy. [Enterprise Pricing](https://www.litellm.ai/#pricing) -[Contact us here to get a free trial](https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions) +[Contact us here to get a free trial](https://enterprise.litellm.ai/demo) ::: +:::tip JWT → Virtual Key Mapping + +Want per-user model restrictions, spend limits, and rate limits without distributing API keys? See **[JWT → Virtual Key Mapping](./jwt_key_mapping.md)** — enterprise-grade granular access control for JWT-authenticated users (e.g. Claude Code + SSO). + +::: + ## Usage ### Step 1. Setup Proxy diff --git a/docs/my-website/docs/proxy_server.md b/docs/my-website/docs/proxy_server.md index e23d64e443b..7b6f15a604b 100644 --- a/docs/my-website/docs/proxy_server.md +++ b/docs/my-website/docs/proxy_server.md @@ -813,5 +813,4 @@ Thread Stats Avg Stdev Max +/- Stdev - [Schedule Demo 👋](https://calendly.com/d/4mp-gd3-k5k/berriai-1-1-onboarding-litellm-hosted-version) - [Community Discord 💭](https://discord.gg/wuPM9dRgDw) -- Our numbers 📞 +1 (770) 8783-106 / ‭+1 (412) 618-6238‬ - Our emails ✉️ ishaan@berri.ai / krrish@berri.ai diff --git a/docs/my-website/docs/routing.md b/docs/my-website/docs/routing.md index 67e7f681147..5aa655ae212 100644 --- a/docs/my-website/docs/routing.md +++ b/docs/my-website/docs/routing.md @@ -842,6 +842,8 @@ Traffic mirroring allows you to "mimic" production traffic to a secondary (silen Set `order` in `litellm_params` to prioritize deployments. Lower values = higher priority. When multiple deployments share the same `order`, the routing strategy picks among them. +When a request to an `order=1` deployment fails (connection error, 404, 429, etc.), the router automatically tries `order=2` deployments, then `order=3`, and so on. Each order level gets its own set of retries before escalating to the next. If all order levels are exhausted, the router falls through to any configured [fallbacks](#fallbacks). + @@ -862,18 +864,14 @@ model_list = [ "litellm_params": { "model": "azure/gpt-4-fallback", "api_key": os.getenv("AZURE_API_KEY_2"), - "order": 2, # 👈 Used when order=1 is unavailable + "order": 2, # 👈 Tried when order=1 fails }, }, ] -router = Router(model_list=model_list, enable_pre_call_checks=True) # 👈 Required for 'order' to work +router = Router(model_list=model_list) ``` -:::important -The `order` parameter requires `enable_pre_call_checks=True` to be set on the Router. -::: - @@ -889,10 +887,7 @@ model_list: litellm_params: model: azure/gpt-4-fallback api_key: os.environ/AZURE_API_KEY_2 - order: 2 # 👈 Used when order=1 is unavailable - -router_settings: - enable_pre_call_checks: true # 👈 Required for 'order' to work + order: 2 # 👈 Tried when order=1 fails ``` diff --git a/docs/my-website/docs/secret.md b/docs/my-website/docs/secret.md index c5c80311475..57f576fd56a 100644 --- a/docs/my-website/docs/secret.md +++ b/docs/my-website/docs/secret.md @@ -6,7 +6,7 @@ [Enterprise Pricing](https://www.litellm.ai/#pricing) -[Contact us here to get a free trial](https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions) +[Contact us here to get a free trial](https://enterprise.litellm.ai/demo) ::: diff --git a/docs/my-website/docs/secret_managers/aws_kms.md b/docs/my-website/docs/secret_managers/aws_kms.md index 7f69d91fe87..806223a2539 100644 --- a/docs/my-website/docs/secret_managers/aws_kms.md +++ b/docs/my-website/docs/secret_managers/aws_kms.md @@ -6,7 +6,7 @@ [Enterprise Pricing](https://www.litellm.ai/#pricing) -[Contact us here to get a free trial](https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions) +[Contact us here to get a free trial](https://enterprise.litellm.ai/demo) ::: diff --git a/docs/my-website/docs/secret_managers/aws_secret_manager.md b/docs/my-website/docs/secret_managers/aws_secret_manager.md index c49797a15dd..a7e24ea69ae 100644 --- a/docs/my-website/docs/secret_managers/aws_secret_manager.md +++ b/docs/my-website/docs/secret_managers/aws_secret_manager.md @@ -9,7 +9,7 @@ import TabItem from '@theme/TabItem'; [Enterprise Pricing](https://www.litellm.ai/#pricing) -[Contact us here to get a free trial](https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions) +[Contact us here to get a free trial](https://enterprise.litellm.ai/demo) ::: diff --git a/docs/my-website/docs/secret_managers/azure_key_vault.md b/docs/my-website/docs/secret_managers/azure_key_vault.md index 81aeaa32159..4ea53d2ea9e 100644 --- a/docs/my-website/docs/secret_managers/azure_key_vault.md +++ b/docs/my-website/docs/secret_managers/azure_key_vault.md @@ -6,7 +6,7 @@ [Enterprise Pricing](https://www.litellm.ai/#pricing) -[Contact us here to get a free trial](https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions) +[Contact us here to get a free trial](https://enterprise.litellm.ai/demo) ::: diff --git a/docs/my-website/docs/secret_managers/cyberark.md b/docs/my-website/docs/secret_managers/cyberark.md index 0a17c0afc30..cd7c0ea5d25 100644 --- a/docs/my-website/docs/secret_managers/cyberark.md +++ b/docs/my-website/docs/secret_managers/cyberark.md @@ -8,7 +8,7 @@ import Image from '@theme/IdealImage'; [Enterprise Pricing](https://www.litellm.ai/#pricing) -[Contact us here to get a free trial](https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions) +[Contact us here to get a free trial](https://enterprise.litellm.ai/demo) ::: diff --git a/docs/my-website/docs/secret_managers/google_kms.md b/docs/my-website/docs/secret_managers/google_kms.md index 31fd6195bdb..152ecbaae80 100644 --- a/docs/my-website/docs/secret_managers/google_kms.md +++ b/docs/my-website/docs/secret_managers/google_kms.md @@ -6,7 +6,7 @@ [Enterprise Pricing](https://www.litellm.ai/#pricing) -[Contact us here to get a free trial](https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions) +[Contact us here to get a free trial](https://enterprise.litellm.ai/demo) ::: diff --git a/docs/my-website/docs/secret_managers/google_secret_manager.md b/docs/my-website/docs/secret_managers/google_secret_manager.md index 81878b7e398..f3e7367e8a4 100644 --- a/docs/my-website/docs/secret_managers/google_secret_manager.md +++ b/docs/my-website/docs/secret_managers/google_secret_manager.md @@ -6,7 +6,7 @@ [Enterprise Pricing](https://www.litellm.ai/#pricing) -[Contact us here to get a free trial](https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions) +[Contact us here to get a free trial](https://enterprise.litellm.ai/demo) ::: diff --git a/docs/my-website/docs/secret_managers/hashicorp_vault.md b/docs/my-website/docs/secret_managers/hashicorp_vault.md index 52d9b556200..11e25e88a7d 100644 --- a/docs/my-website/docs/secret_managers/hashicorp_vault.md +++ b/docs/my-website/docs/secret_managers/hashicorp_vault.md @@ -8,7 +8,7 @@ import Image from '@theme/IdealImage'; [Enterprise Pricing](https://www.litellm.ai/#pricing) -[Contact us here to get a free trial](https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions) +[Contact us here to get a free trial](https://enterprise.litellm.ai/demo) ::: diff --git a/docs/my-website/docs/secret_managers/overview.md b/docs/my-website/docs/secret_managers/overview.md index bf7386ab89c..f02362f4932 100644 --- a/docs/my-website/docs/secret_managers/overview.md +++ b/docs/my-website/docs/secret_managers/overview.md @@ -8,7 +8,7 @@ import Image from '@theme/IdealImage'; [Enterprise Pricing](https://www.litellm.ai/#pricing) -[Contact us here to get a free trial](https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions) +[Contact us here to get a free trial](https://enterprise.litellm.ai/demo) ::: diff --git a/docs/my-website/docs/troubleshoot.md b/docs/my-website/docs/troubleshoot.md index 1539e1959f7..1afa35df8e4 100644 --- a/docs/my-website/docs/troubleshoot.md +++ b/docs/my-website/docs/troubleshoot.md @@ -50,7 +50,6 @@ Full error logs, stack traces, and any images from service metrics (CPU, memory, [Community Discord 💭](https://discord.gg/wuPM9dRgDw) [Community Slack 💭](https://www.litellm.ai/support) -Our numbers 📞 +1 (770) 8783-106 / +1 (412) 618-6238 Our emails ✉️ ishaan@berri.ai / krrish@berri.ai diff --git a/docs/my-website/docs/tutorials/compare_llms.md b/docs/my-website/docs/tutorials/compare_llms.md index 02877b46607..0252263a16e 100644 --- a/docs/my-website/docs/tutorials/compare_llms.md +++ b/docs/my-website/docs/tutorials/compare_llms.md @@ -82,7 +82,7 @@ Benchmark Results for 'When will BerriAI IPO?': +-----------------+----------------------------------------------------------------------------------+---------------------------+------------+ ``` ## Support -**🤝 Schedule a 1-on-1 Session:** Book a [1-on-1 session](https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions) with Krrish and Ishaan, the founders, to discuss any issues, provide feedback, or explore how we can improve LiteLLM for you. +**🤝 Schedule a 1-on-1 Session:** Book a [1-on-1 session](https://enterprise.litellm.ai/demo) with Krrish and Ishaan, the founders, to discuss any issues, provide feedback, or explore how we can improve LiteLLM for you.
Stripeimage Google ADK Greptile OpenHands