diff --git a/.circleci/config.yml b/.circleci/config.yml index 790efc79862..307247651e7 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -39,7 +39,7 @@ commands: # conflicts with transitive dep pins like openai<2 and pydantic>=2.11.5) pip install "pytest-mock==3.12.0" "pytest==7.3.1" "pytest-retry==1.6.3" \ "pytest-asyncio==0.21.1" "respx==0.22.0" "hypercorn==0.17.3" \ - "pydantic==2.11.0" "mcp==1.25.0" "requests-mock>=1.12.1" \ + "pydantic==2.12.5" "mcp==1.26.0" "requests-mock>=1.12.1" \ "responses==0.25.7" "pytest-xdist==3.6.1" "pytest-timeout==2.2.0" \ "pytest-cov==5.0.0" "semantic_router==0.1.10" "fastapi-offline==1.7.3" \ "a2a" "parameterized>=0.9.0" @@ -150,16 +150,16 @@ jobs: python -m pip install --upgrade pip python -m pip install -r .circleci/requirements.txt pip install "pytest==7.3.1" "pytest-retry==1.6.3" "pytest-asyncio==0.21.1" "pytest-cov==5.0.0" \ - "mypy==1.18.2" "google-generativeai==0.3.2" "google-cloud-aiplatform==1.43.0" pyarrow \ - "boto3==1.36.0" "aioboto3==13.4.0" langchain lunary==0.2.5 \ - "azure-identity==1.16.1" "langfuse==2.59.7" "logfire==0.29.0" numpydoc \ + "mypy==1.18.2" "google-generativeai==0.3.2" "google-cloud-aiplatform==1.133.0" pyarrow \ + "boto3==1.42.80" langchain lunary==0.2.5 \ + "azure-identity==1.25.3" "langfuse==2.59.7" "logfire==0.29.0" numpydoc \ traceloop-sdk==0.21.1 openai==1.100.1 prisma==0.11.0 \ "detect_secrets==1.5.0" "respx==0.22.0" fastapi \ - "gunicorn==21.2.0" "aiodynamo==23.10.1" "asyncio==3.4.3" \ - "apscheduler==3.10.4" "PyGithub==1.59.1" argon2-cffi "pytest-mock==3.12.0" \ - python-multipart prometheus-client==0.20.0 "pydantic==2.10.2" \ - "diskcache==5.6.1" "Pillow==10.3.0" "jsonschema==4.22.0" \ - "pytest-xdist==3.6.1" "pytest-timeout==2.2.0" "websockets==13.1.0" + "gunicorn==23.0.0" "aiodynamo==23.10.1" "asyncio==3.4.3" \ + "apscheduler==3.11.2" "PyGithub==1.59.1" argon2-cffi "pytest-mock==3.12.0" \ + python-multipart prometheus-client==0.20.0 "pydantic==2.12.5" \ + "diskcache==5.6.1" "Pillow==12.1.1" "jsonschema==4.23.0" \ + "pytest-xdist==3.6.1" "pytest-timeout==2.2.0" "websockets==15.0.1" pip install semantic_router --no-deps pip install aurelio_sdk --no-deps pip uninstall posthog -y @@ -245,16 +245,16 @@ jobs: python -m pip install --upgrade pip python -m pip install -r .circleci/requirements.txt pip install "pytest==7.3.1" "pytest-retry==1.6.3" "pytest-asyncio==0.21.1" "pytest-cov==5.0.0" \ - "mypy==1.18.2" "google-generativeai==0.3.2" "google-cloud-aiplatform==1.43.0" pyarrow \ - "boto3==1.36.0" "aioboto3==13.4.0" langchain lunary==0.2.5 \ - "azure-identity==1.16.1" "langfuse==2.59.7" "logfire==0.29.0" numpydoc \ + "mypy==1.18.2" "google-generativeai==0.3.2" "google-cloud-aiplatform==1.133.0" pyarrow \ + "boto3==1.42.80" langchain lunary==0.2.5 \ + "azure-identity==1.25.3" "langfuse==2.59.7" "logfire==0.29.0" numpydoc \ traceloop-sdk==0.21.1 openai==1.100.1 prisma==0.11.0 \ "detect_secrets==1.5.0" "respx==0.22.0" fastapi \ - "gunicorn==21.2.0" "aiodynamo==23.10.1" "asyncio==3.4.3" \ - "apscheduler==3.10.4" "PyGithub==1.59.1" argon2-cffi "pytest-mock==3.12.0" \ - python-multipart prometheus-client==0.20.0 "pydantic==2.10.2" \ - "diskcache==5.6.1" "Pillow==10.3.0" "jsonschema==4.22.0" \ - "pytest-xdist==3.6.1" "pytest-timeout==2.2.0" "websockets==13.1.0" + "gunicorn==23.0.0" "aiodynamo==23.10.1" "asyncio==3.4.3" \ + "apscheduler==3.11.2" "PyGithub==1.59.1" argon2-cffi "pytest-mock==3.12.0" \ + python-multipart prometheus-client==0.20.0 "pydantic==2.12.5" \ + "diskcache==5.6.1" "Pillow==12.1.1" "jsonschema==4.23.0" \ + "pytest-xdist==3.6.1" "pytest-timeout==2.2.0" "websockets==15.0.1" pip install semantic_router --no-deps pip install aurelio_sdk --no-deps pip uninstall posthog -y @@ -346,42 +346,41 @@ jobs: pip install "pytest-cov==5.0.0" pip install "mypy==1.18.2" pip install "google-generativeai==0.3.2" - pip install "google-cloud-aiplatform==1.43.0" + pip install "google-cloud-aiplatform==1.133.0" pip install pyarrow - pip install "boto3==1.36.0" - pip install "aioboto3==13.4.0" + pip install "boto3==1.42.80" pip install langchain pip install lunary==0.2.5 - pip install "azure-identity==1.16.1" + pip install "azure-identity==1.25.3" pip install "langfuse==2.59.7" pip install "logfire==0.29.0" pip install numpydoc pip install traceloop-sdk==0.21.1 - pip install opentelemetry-api==1.25.0 - pip install opentelemetry-sdk==1.25.0 - pip install opentelemetry-exporter-otlp==1.25.0 + pip install opentelemetry-api==1.28.0 + pip install opentelemetry-sdk==1.28.0 + pip install opentelemetry-exporter-otlp==1.28.0 pip install openai==1.100.1 pip install prisma==0.11.0 pip install "detect_secrets==1.5.0" - pip install "httpx==0.24.1" + pip install "httpx==0.28.1" pip install "respx==0.22.0" pip install fastapi - pip install "gunicorn==21.2.0" - pip install "anyio==4.2.0" + pip install "gunicorn==23.0.0" + pip install "anyio==4.8.0" pip install "aiodynamo==23.10.1" pip install "asyncio==3.4.3" - pip install "apscheduler==3.10.4" + pip install "apscheduler==3.11.2" pip install "PyGithub==1.59.1" pip install argon2-cffi pip install "pytest-mock==3.12.0" pip install python-multipart pip install google-cloud-aiplatform pip install prometheus-client==0.20.0 - pip install "pydantic==2.10.2" + pip install "pydantic==2.12.5" pip install "diskcache==5.6.1" - pip install "Pillow==10.3.0" - pip install "jsonschema==4.22.0" - pip install "websockets==13.1.0" + pip install "Pillow==12.1.1" + pip install "jsonschema==4.23.0" + pip install "websockets==15.0.1" - setup_litellm_enterprise_pip - save_cache: paths: @@ -406,119 +405,6 @@ jobs: # Store test results - store_test_results: path: test-results - caching_unit_tests: - docker: - - image: cimg/python:3.11 - auth: - username: ${DOCKERHUB_USERNAME} - password: ${DOCKERHUB_PASSWORD} - resource_class: large - working_directory: ~/project - parallelism: 2 - - steps: - - checkout - - setup_google_dns - - run: - name: DNS lookup for Redis host - command: | - sudo apt-get update - sudo apt-get install -y dnsutils - dig redis-19899.c239.us-east-1-2.ec2.redns.redis-cloud.com +short - - run: - name: Show git commit hash - command: | - echo "Git commit hash: $CIRCLE_SHA1" - - - restore_cache: - keys: - - v2-caching-deps-{{ checksum ".circleci/requirements.txt" }} - - v2-caching-deps- - - run: - name: Install Dependencies - command: | - python -m pip install --upgrade pip - python -m pip install -r .circleci/requirements.txt - pip install "pytest==7.3.1" - pip install "pytest-retry==1.6.3" - pip install "pytest-asyncio==0.21.1" - pip install "pytest-cov==5.0.0" - pip install "mypy==1.18.2" - pip install "google-generativeai==0.3.2" - pip install "google-cloud-aiplatform==1.43.0" - pip install pyarrow - pip install "boto3==1.36.0" - pip install "aioboto3==13.4.0" - pip install langchain - pip install lunary==0.2.5 - pip install "azure-identity==1.16.1" - pip install "langfuse==2.59.7" - pip install "logfire==0.29.0" - pip install numpydoc - pip install traceloop-sdk==0.21.1 - pip install opentelemetry-api==1.25.0 - pip install opentelemetry-sdk==1.25.0 - pip install opentelemetry-exporter-otlp==1.25.0 - pip install openai==1.100.1 - pip install prisma==0.11.0 - pip install "detect_secrets==1.5.0" - pip install "httpx==0.24.1" - pip install "respx==0.22.0" - pip install fastapi - pip install "gunicorn==21.2.0" - pip install "anyio==4.2.0" - pip install "aiodynamo==23.10.1" - pip install "asyncio==3.4.3" - pip install "apscheduler==3.10.4" - pip install "PyGithub==1.59.1" - pip install argon2-cffi - pip install "pytest-mock==3.12.0" - pip install python-multipart - pip install google-cloud-aiplatform - pip install prometheus-client==0.20.0 - pip install "pydantic==2.10.2" - pip install "diskcache==5.6.1" - pip install "Pillow==10.3.0" - pip install "jsonschema==4.22.0" - pip install "websockets==13.1.0" - pip install "pytest-xdist==3.6.1" - - setup_litellm_enterprise_pip - - save_cache: - paths: - - /home/circleci/.pyenv/versions - - /home/circleci/.local - key: v2-caching-deps-{{ checksum ".circleci/requirements.txt" }} - - run: - name: Run prisma ./docker/entrypoint.sh - command: | - set +e - chmod +x docker/entrypoint.sh - ./docker/entrypoint.sh - set -e - - # Run pytest and generate JUnit XML report - - run: - name: Run tests - command: | - pwd - ls - mkdir -p test-results - - TEST_FILES=$(circleci tests glob "tests/local_testing/**/test_*.py") - - echo "$TEST_FILES" | circleci tests run \ - --split-by=timings \ - --verbose \ - --command="xargs python -m pytest \ - -v \ - --junitxml=test-results/junit.xml \ - --durations=5 \ - -k 'caching or cache'" - no_output_timeout: 15m - - # Store test results - - store_test_results: - path: test-results auth_ui_unit_tests: docker: - image: cimg/python:3.11 @@ -664,376 +550,6 @@ jobs: # Store test results - store_test_results: path: test-results - litellm_security_tests: - docker: - - image: cimg/python:3.13 - auth: - username: ${DOCKERHUB_USERNAME} - password: ${DOCKERHUB_PASSWORD} - - image: cimg/postgres:14.0 - environment: - POSTGRES_USER: postgres - POSTGRES_PASSWORD: postgres - POSTGRES_DB: circle_test - resource_class: xlarge - working_directory: ~/project - environment: - DATABASE_URL: "postgresql://postgres:postgres@localhost:5432/circle_test" - steps: - - checkout - - setup_google_dns - - run: - name: Show git commit hash - command: | - echo "Git commit hash: $CIRCLE_SHA1" - - setup_remote_docker: - docker_layer_caching: true - - restore_cache: - keys: - - v3-litellm-uv-deps-{{ checksum "requirements.txt" }}-{{ checksum ".circleci/config.yml" }} - - run: - name: Install Dependencies - command: | - python -m pip install --upgrade pip uv - uv pip install --system -r requirements.txt - pip install "pytest==7.3.1" "pytest-retry==1.6.3" "pytest-mock==3.12.0" \ - "pytest-asyncio==0.21.1" "pytest-cov==5.0.0" - - save_cache: - paths: - - ~/.local/lib - - ~/.local/bin - - ~/.cache/uv - key: v3-litellm-uv-deps-{{ checksum "requirements.txt" }}-{{ checksum ".circleci/config.yml" }} - - run: - name: Install dockerize - command: | - wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz - sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz - rm dockerize-linux-amd64-v0.6.1.tar.gz - - run: - name: Wait for PostgreSQL to be ready - command: dockerize -wait tcp://localhost:5432 -timeout 1m - - run: - name: Run Security Scans - command: | - chmod +x ci_cd/security_scans.sh - ./ci_cd/security_scans.sh - - run: - name: Run prisma ./docker/entrypoint.sh - command: | - set +e - chmod +x docker/entrypoint.sh - ./docker/entrypoint.sh - set -e - # Run pytest and generate JUnit XML report - - run: - name: Run tests - command: | - python -m pytest tests/proxy_security_tests -v -x --junitxml=test-results/junit.xml --durations=5 - no_output_timeout: 15m - # Store test results - - store_test_results: - path: test-results - # Split proxy unit tests into 3 jobs for faster execution and better debugging - # test_key_generate_prisma runs separately without parallel execution to avoid event loop issues with logging worker - litellm_proxy_unit_testing_key_generation: - docker: - - image: cimg/python:3.11 - auth: - username: ${DOCKERHUB_USERNAME} - password: ${DOCKERHUB_PASSWORD} - working_directory: ~/project - resource_class: medium - steps: - - checkout - - setup_google_dns - - run: - name: Show git commit hash - command: | - echo "Git commit hash: $CIRCLE_SHA1" - - run: - name: Install PostgreSQL - command: | - sudo apt-get update - sudo apt-get install -y postgresql-14 postgresql-contrib-14 - - restore_cache: - keys: - - v1-dependencies-{{ checksum ".circleci/requirements.txt" }} - - run: - name: Install Dependencies - command: | - python -m pip install --upgrade pip - python -m pip install -r .circleci/requirements.txt - pip install "pytest==7.3.1" - pip install "pytest-retry==1.6.3" - pip install "pytest-asyncio==0.21.1" - pip install "pytest-cov==5.0.0" - pip install "pytest-timeout==2.2.0" - pip install "pytest-forked==1.6.0" - pip install "mypy==1.18.2" - pip install "google-generativeai==0.3.2" - pip install "google-cloud-aiplatform==1.43.0" - pip install "google-genai==1.22.0" - pip install pyarrow - pip install "boto3==1.36.0" - pip install "aioboto3==13.4.0" - pip install langchain - pip install lunary==0.2.5 - pip install "azure-identity==1.16.1" - pip install "langfuse==2.59.7" - pip install "logfire==0.29.0" - pip install numpydoc - pip install traceloop-sdk==0.21.1 - pip install opentelemetry-api==1.25.0 - pip install opentelemetry-sdk==1.25.0 - pip install opentelemetry-exporter-otlp==1.25.0 - pip install openai==1.100.1 - pip install prisma==0.11.0 - pip install "detect_secrets==1.5.0" - pip install "httpx==0.24.1" - pip install "respx==0.22.0" - pip install fastapi - pip install "gunicorn==21.2.0" - pip install "anyio==4.2.0" - pip install "aiodynamo==23.10.1" - pip install "asyncio==3.4.3" - pip install "apscheduler==3.10.4" - pip install "PyGithub==1.59.1" - pip install argon2-cffi - pip install "pytest-mock==3.12.0" - pip install python-multipart - pip install google-cloud-aiplatform - pip install prometheus-client==0.20.0 - pip install "pydantic==2.10.2" - pip install "diskcache==5.6.1" - pip install "Pillow==10.3.0" - pip install "jsonschema==4.22.0" - pip install "pytest-postgresql==7.0.1" - pip install "fakeredis==2.28.1" - - setup_litellm_enterprise_pip - - save_cache: - paths: - - ./venv - key: v1-dependencies-{{ checksum ".circleci/requirements.txt" }} - - run: - name: Run prisma ./docker/entrypoint.sh - command: | - set +e - chmod +x docker/entrypoint.sh - ./docker/entrypoint.sh - set -e - - run: - name: Run key generation tests (no parallel execution to avoid event loop issues) - command: | - pwd - ls - # Run without -n flag to avoid pytest-xdist event loop conflicts with logging worker - python -m pytest tests/proxy_unit_tests/test_key_generate_prisma.py --cov=litellm --cov-report=xml --junitxml=test-results/junit-key-generation.xml --durations=10 --timeout=300 -vv --log-cli-level=INFO - no_output_timeout: 15m - - run: - name: Rename the coverage files - command: | - mv coverage.xml litellm_proxy_unit_tests_key_generation_coverage.xml - mv .coverage litellm_proxy_unit_tests_key_generation_coverage - - store_test_results: - path: test-results - - persist_to_workspace: - root: . - paths: - - litellm_proxy_unit_tests_key_generation_coverage.xml - - litellm_proxy_unit_tests_key_generation_coverage - litellm_proxy_unit_testing_part1: - docker: - - image: cimg/python:3.11 - auth: - username: ${DOCKERHUB_USERNAME} - password: ${DOCKERHUB_PASSWORD} - working_directory: ~/project - resource_class: xlarge - steps: - - checkout - - setup_google_dns - - run: - name: Show git commit hash - command: | - echo "Git commit hash: $CIRCLE_SHA1" - - run: - name: Install PostgreSQL - command: | - sudo apt-get update - sudo apt-get install -y postgresql-14 postgresql-contrib-14 - - restore_cache: - keys: - - v1-dependencies-{{ checksum ".circleci/requirements.txt" }} - - run: - name: Install Dependencies - command: | - python -m pip install --upgrade pip - python -m pip install -r .circleci/requirements.txt - pip install "pytest==7.3.1" - pip install "pytest-retry==1.6.3" - pip install "pytest-asyncio==0.21.1" - pip install "pytest-cov==5.0.0" - pip install "pytest-timeout==2.2.0" - pip install "pytest-forked==1.6.0" - pip install "mypy==1.18.2" - pip install "google-generativeai==0.3.2" - pip install "google-cloud-aiplatform==1.43.0" - pip install "google-genai==1.22.0" - pip install pyarrow - pip install "boto3==1.36.0" - pip install "aioboto3==13.4.0" - pip install langchain - pip install lunary==0.2.5 - pip install "azure-identity==1.16.1" - pip install "langfuse==2.59.7" - pip install "logfire==0.29.0" - pip install numpydoc - pip install traceloop-sdk==0.21.1 - pip install opentelemetry-api==1.25.0 - pip install opentelemetry-sdk==1.25.0 - pip install opentelemetry-exporter-otlp==1.25.0 - pip install openai==1.100.1 - pip install prisma==0.11.0 - pip install "detect_secrets==1.5.0" - pip install "httpx==0.24.1" - pip install "respx==0.22.0" - pip install fastapi - pip install "gunicorn==21.2.0" - pip install "anyio==4.2.0" - pip install "aiodynamo==23.10.1" - pip install "asyncio==3.4.3" - pip install "apscheduler==3.10.4" - pip install "PyGithub==1.59.1" - pip install argon2-cffi - pip install "pytest-mock==3.12.0" - pip install python-multipart - pip install google-cloud-aiplatform - pip install prometheus-client==0.20.0 - pip install "pydantic==2.10.2" - pip install "diskcache==5.6.1" - pip install "Pillow==10.3.0" - pip install "jsonschema==4.22.0" - pip install "pytest-postgresql==7.0.1" - pip install "fakeredis==2.28.1" - pip install "pytest-xdist==3.6.1" - - setup_litellm_enterprise_pip - - save_cache: - paths: - - ./venv - key: v1-dependencies-{{ checksum ".circleci/requirements.txt" }} - - run: - name: Run prisma ./docker/entrypoint.sh - command: | - set +e - chmod +x docker/entrypoint.sh - ./docker/entrypoint.sh - set -e - - run: - name: Run proxy unit tests (part 1 - auth checks) - command: | - pwd - ls - python -m pytest tests/proxy_unit_tests/test_auth_checks.py tests/proxy_unit_tests/test_user_api_key_auth.py --junitxml=test-results/junit-part1.xml --durations=10 -n 8 --timeout=300 -v - no_output_timeout: 15m - - store_test_results: - path: test-results - litellm_proxy_unit_testing_part2: - docker: - - image: cimg/python:3.11 - auth: - username: ${DOCKERHUB_USERNAME} - password: ${DOCKERHUB_PASSWORD} - working_directory: ~/project - resource_class: xlarge - steps: - - checkout - - setup_google_dns - - run: - name: Show git commit hash - command: | - echo "Git commit hash: $CIRCLE_SHA1" - - run: - name: Install PostgreSQL - command: | - sudo apt-get update - sudo apt-get install -y postgresql-14 postgresql-contrib-14 - - restore_cache: - keys: - - v1-dependencies-{{ checksum ".circleci/requirements.txt" }} - - run: - name: Install Dependencies - command: | - python -m pip install --upgrade pip - python -m pip install -r .circleci/requirements.txt - pip install "pytest==7.3.1" - pip install "pytest-retry==1.6.3" - pip install "pytest-asyncio==0.21.1" - pip install "pytest-cov==5.0.0" - pip install "pytest-timeout==2.2.0" - pip install "pytest-forked==1.6.0" - pip install "mypy==1.18.2" - pip install "google-generativeai==0.3.2" - pip install "google-cloud-aiplatform==1.43.0" - pip install "google-genai==1.22.0" - pip install pyarrow - pip install "boto3==1.36.0" - pip install "aioboto3==13.4.0" - pip install langchain - pip install lunary==0.2.5 - pip install "azure-identity==1.16.1" - pip install "langfuse==2.59.7" - pip install "logfire==0.29.0" - pip install numpydoc - pip install traceloop-sdk==0.21.1 - pip install opentelemetry-api==1.25.0 - pip install opentelemetry-sdk==1.25.0 - pip install opentelemetry-exporter-otlp==1.25.0 - pip install openai==1.100.1 - pip install prisma==0.11.0 - pip install "detect_secrets==1.5.0" - pip install "httpx==0.24.1" - pip install "respx==0.22.0" - pip install fastapi - pip install "gunicorn==21.2.0" - pip install "anyio==4.2.0" - pip install "aiodynamo==23.10.1" - pip install "asyncio==3.4.3" - pip install "apscheduler==3.10.4" - pip install "PyGithub==1.59.1" - pip install argon2-cffi - pip install "pytest-mock==3.12.0" - pip install python-multipart - pip install google-cloud-aiplatform - pip install prometheus-client==0.20.0 - pip install "pydantic==2.10.2" - pip install "diskcache==5.6.1" - pip install "Pillow==10.3.0" - pip install "jsonschema==4.22.0" - pip install "pytest-postgresql==7.0.1" - pip install "fakeredis==2.28.1" - pip install "pytest-xdist==3.6.1" - - setup_litellm_enterprise_pip - - save_cache: - paths: - - ./venv - key: v1-dependencies-{{ checksum ".circleci/requirements.txt" }} - - run: - name: Run prisma ./docker/entrypoint.sh - command: | - set +e - chmod +x docker/entrypoint.sh - ./docker/entrypoint.sh - set -e - - run: - name: Run proxy unit tests (part 2 - remaining tests) - command: | - pwd - ls - python -m pytest tests/proxy_unit_tests --ignore=tests/proxy_unit_tests/test_key_generate_prisma.py --ignore=tests/proxy_unit_tests/test_auth_checks.py --ignore=tests/proxy_unit_tests/test_user_api_key_auth.py --junitxml=test-results/junit-part2.xml --durations=10 -n 8 --timeout=300 -v - no_output_timeout: 15m - - store_test_results: - path: test-results litellm_assistants_api_testing: # Runs all tests with the "assistants" keyword docker: - image: cimg/python:3.13.1 @@ -1183,8 +699,8 @@ jobs: pip install "pytest-cov==5.0.0" pip install "pytest-asyncio==0.21.1" pip install "respx==0.22.0" - pip install "pydantic==2.11.0" - pip install "mcp==1.25.0" + pip install "pydantic==2.12.5" + pip install "mcp==1.26.0" pip install "pytest-xdist==3.6.1" # Run pytest and generate JUnit XML report - run: @@ -1229,7 +745,7 @@ jobs: pip install "pytest-cov==5.0.0" pip install "pytest-asyncio==0.21.1" pip install "respx==0.22.0" - pip install "pydantic==2.11.0" + pip install "pydantic==2.12.5" pip install "a2a-sdk" # Run pytest and generate JUnit XML report - run: @@ -1274,8 +790,8 @@ jobs: pip install "pytest-cov==5.0.0" pip install "pytest-asyncio==0.21.1" pip install "respx==0.22.0" - pip install "pydantic==2.10.2" - pip install "boto3==1.36.0" + pip install "pydantic==2.12.5" + pip install "boto3==1.42.80" pip install "semantic_router==0.1.10" --no-deps pip install aurelio_sdk pip install "pytest-xdist==3.6.1" @@ -1324,7 +840,7 @@ jobs: pip install "pytest-cov==5.0.0" pip install "pytest-asyncio==0.21.1" pip install "respx==0.22.0" - pip install "pydantic==2.10.2" + pip install "pydantic==2.12.5" # Run pytest and generate JUnit XML report - run: name: Run tests @@ -1507,101 +1023,6 @@ jobs: no_output_timeout: 15m - store_test_results: path: test-results - litellm_mapped_tests_llms: - docker: - - image: cimg/python:3.11 - auth: - username: ${DOCKERHUB_USERNAME} - password: ${DOCKERHUB_PASSWORD} - working_directory: ~/project - resource_class: large - steps: - - setup_litellm_test_deps - - run: - name: Run LLM provider tests - command: | - python -m pytest tests/test_litellm/llms --junitxml=test-results/junit-llms.xml --durations=10 -n 4 --maxfail=5 --timeout=300 -vv --log-cli-level=WARNING - no_output_timeout: 15m - - store_test_results: - path: test-results - litellm_mapped_tests_core: - docker: - - image: cimg/python:3.11 - auth: - username: ${DOCKERHUB_USERNAME} - password: ${DOCKERHUB_PASSWORD} - working_directory: ~/project - resource_class: large - steps: - - setup_litellm_test_deps - - run: - name: Run core tests - command: | - python -m pytest tests/test_litellm --ignore=tests/test_litellm/proxy --ignore=tests/test_litellm/llms --ignore=tests/test_litellm/integrations --ignore=tests/test_litellm/litellm_core_utils --ignore=tests/test_litellm/experimental_mcp_client --junitxml=test-results/junit-core.xml --durations=10 -n 4 --maxfail=5 --timeout=300 -vv --log-cli-level=WARNING - no_output_timeout: 15m - - store_test_results: - path: test-results - litellm_mapped_tests_litellm_core_utils: - docker: - - image: cimg/python:3.11 - auth: - username: ${DOCKERHUB_USERNAME} - password: ${DOCKERHUB_PASSWORD} - working_directory: ~/project - resource_class: large - steps: - - setup_litellm_test_deps - - run: - name: Run litellm_core_utils tests - command: | - python -m pytest tests/test_litellm/litellm_core_utils --junitxml=test-results/junit-litellm-core-utils.xml --durations=10 -n 4 --maxfail=5 --timeout=300 -vv --log-cli-level=WARNING - no_output_timeout: 15m - - store_test_results: - path: test-results - litellm_mapped_tests_mcps: - docker: - - image: cimg/python:3.11 - auth: - username: ${DOCKERHUB_USERNAME} - password: ${DOCKERHUB_PASSWORD} - working_directory: ~/project - resource_class: medium - steps: - - setup_litellm_test_deps - - run: - name: Run MCP client tests - command: | - python -m pytest tests/test_litellm/experimental_mcp_client --cov=litellm --cov-report=xml --junitxml=test-results/junit-mcps.xml --durations=10 -n 2 --maxfail=5 --timeout=300 -vv --log-cli-level=WARNING - no_output_timeout: 15m - - run: - name: Rename the coverage files - command: | - mv coverage.xml litellm_mcps_tests_coverage.xml - mv .coverage litellm_mcps_tests_coverage - - store_test_results: - path: test-results - - persist_to_workspace: - root: . - paths: - - litellm_mcps_tests_coverage.xml - - litellm_mcps_tests_coverage - litellm_mapped_tests_integrations: - docker: - - image: cimg/python:3.11 - auth: - username: ${DOCKERHUB_USERNAME} - password: ${DOCKERHUB_PASSWORD} - working_directory: ~/project - resource_class: large - steps: - - setup_litellm_test_deps - - run: - name: Run integrations tests - command: | - python -m pytest tests/test_litellm/integrations --junitxml=test-results/junit-integrations.xml --durations=10 -n 4 --maxfail=5 --timeout=300 -vv --log-cli-level=WARNING - no_output_timeout: 15m - - store_test_results: - path: test-results litellm_mapped_enterprise_tests: docker: - image: cimg/python:3.11 @@ -1626,8 +1047,8 @@ jobs: pip install "pytest-asyncio==0.21.1" pip install "respx==0.22.0" pip install "hypercorn==0.17.3" - pip install "pydantic==2.11.0" - pip install "mcp==1.25.0" + pip install "pydantic==2.12.5" + pip install "mcp==1.26.0" pip install "requests-mock>=1.12.1" pip install "responses==0.25.7" pip install "pytest-xdist==3.6.1" @@ -1668,7 +1089,7 @@ jobs: pip install "pytest-asyncio==0.21.1" pip install "pytest-cov==5.0.0" pip install "google-generativeai==0.3.2" - pip install "google-cloud-aiplatform==1.43.0" + pip install "google-cloud-aiplatform==1.133.0" pip install "pytest-xdist==3.6.1" # Run pytest and generate JUnit XML report - run: @@ -1715,7 +1136,7 @@ jobs: pip install "pytest-asyncio==0.21.1" pip install "pytest-cov==5.0.0" pip install "google-generativeai==0.3.2" - pip install "google-cloud-aiplatform==1.43.0" + pip install "google-cloud-aiplatform==1.133.0" pip install pytest-mock pip install "pytest-xdist==3.6.1" # Run pytest and generate JUnit XML report @@ -1837,9 +1258,9 @@ jobs: pip install pytest-mock pip install "respx==0.22.0" pip install "google-generativeai==0.3.2" - pip install "google-cloud-aiplatform==1.43.0" + pip install "google-cloud-aiplatform==1.133.0" pip install "mlflow==2.17.2" - pip install "anthropic==0.52.0" + pip install "anthropic==0.54.0" pip install "blockbuster==1.5.24" pip install "pytest-xdist==3.6.1" pip install "pytest-timeout==2.2.0" @@ -1930,11 +1351,11 @@ jobs: pip install aiohttp pip install openai pip install click - pip install "boto3==1.36.0" + pip install "boto3==1.42.80" pip install jinja2 - pip install "tokenizers==0.20.0" + pip install "tokenizers==0.22.2" pip install "uvloop==0.21.0" - pip install "fastuuid==0.12.0" + pip install "fastuuid==0.14.0" pip install jsonschema - setup_litellm_enterprise_pip - run: @@ -1967,7 +1388,7 @@ jobs: pip install "pytest-asyncio==0.21.1" pip install "pytest-cov==5.0.0" pip install "tomli==2.2.1" - pip install "mcp==1.25.0" + pip install "mcp==1.26.0" - run: name: Run tests command: | @@ -2137,6 +1558,25 @@ jobs: pip install "pytest-asyncio==0.21.1" pip install aiohttp pip install apscheduler + - run: + name: Install dockerize + command: | + sudo wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz + sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz + sudo rm dockerize-linux-amd64-v0.6.1.tar.gz + - run: + name: Start PostgreSQL Database + command: | + docker run -d \ + --name postgres-db \ + -e POSTGRES_USER=postgres \ + -e POSTGRES_PASSWORD=postgres \ + -e POSTGRES_DB=litellm_test \ + -p 5432:5432 \ + postgres:14 + - run: + name: Wait for PostgreSQL to be ready + command: dockerize -wait tcp://localhost:5432 -timeout 1m - attach_workspace: at: ~/project - run: @@ -2145,29 +1585,41 @@ jobs: zstd -d litellm-docker-database.tar.zst --stdout | docker load docker images | grep litellm-docker-database - run: - name: Run Docker container + name: Seed database with real schema + command: | + docker run -d \ + -p 4001:4000 \ + -e DATABASE_URL="postgresql://postgres:postgres@host.docker.internal:5432/litellm_test" \ + -e LITELLM_MASTER_KEY="sk-1234" \ + --name schema-seed \ + --add-host=host.docker.internal:host-gateway \ + -v $(pwd)/litellm/proxy/example_config_yaml/simple_config.yaml:/app/config.yaml \ + litellm-docker-database:ci \ + --config /app/config.yaml \ + --port 4000 \ + --use_prisma_db_push + - run: + name: Wait for schema seed to complete + command: dockerize -wait http://localhost:4001 -timeout 5m + - run: + name: Stop schema seed container + command: docker stop schema-seed && docker rm schema-seed + - run: + name: Run Docker container with bad schema and disabled updates command: | docker run -d \ -p 4000:4000 \ - -e DATABASE_URL=$PROXY_DATABASE_URL \ + -e DATABASE_URL="postgresql://postgres:postgres@host.docker.internal:5432/litellm_test" \ -e DEFAULT_NUM_WORKERS_LITELLM_PROXY=1 \ -e DISABLE_SCHEMA_UPDATE="True" \ + --name my-app \ + --add-host=host.docker.internal:host-gateway \ -v $(pwd)/litellm/proxy/example_config_yaml/bad_schema.prisma:/app/schema.prisma \ -v $(pwd)/litellm/proxy/example_config_yaml/bad_schema.prisma:/app/litellm/proxy/schema.prisma \ -v $(pwd)/litellm/proxy/example_config_yaml/disable_schema_update.yaml:/app/config.yaml \ - --name my-app \ litellm-docker-database:ci \ --config /app/config.yaml \ --port 4000 - - run: - name: Install curl and dockerize - command: | - sudo apt-get update - sudo apt-get install -y curl - sudo wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz - sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz - sudo rm dockerize-linux-amd64-v0.6.1.tar.gz - - run: name: Wait for container to be ready command: dockerize -wait http://localhost:4000 -timeout 1m @@ -2226,10 +1678,9 @@ jobs: pip install "pytest-asyncio==0.21.1" pip install "mypy==1.18.2" pip install "google-generativeai==0.3.2" - pip install "google-cloud-aiplatform==1.43.0" + pip install "google-cloud-aiplatform==1.133.0" pip install pyarrow - pip install "boto3==1.36.0" - pip install "aioboto3==13.4.0" + pip install "boto3==1.42.80" pip install langchain pip install "langfuse>=2.0.0" pip install "logfire==0.29.0" @@ -2237,9 +1688,9 @@ jobs: pip install prisma pip install fastapi pip install jsonschema - pip install "httpx==0.24.1" - pip install "gunicorn==21.2.0" - pip install "anyio==3.7.1" + pip install "httpx==0.28.1" + pip install "gunicorn==23.0.0" + pip install "anyio==4.8.0" pip install "aiodynamo==23.10.1" pip install "asyncio==3.4.3" pip install "PyGithub==1.59.1" @@ -2370,10 +1821,9 @@ jobs: pip install "mypy==1.18.2" pip install "jsonlines==4.0.0" pip install "google-generativeai==0.3.2" - pip install "google-cloud-aiplatform==1.43.0" + pip install "google-cloud-aiplatform==1.133.0" pip install pyarrow - pip install "boto3==1.36.0" - pip install "aioboto3==13.4.0" + pip install "boto3==1.42.80" pip install langchain pip install "langchain_mcp_adapters==0.0.5" pip install "langfuse>=2.0.0" @@ -2382,9 +1832,9 @@ jobs: pip install prisma pip install fastapi pip install jsonschema - pip install "httpx==0.24.1" - pip install "gunicorn==21.2.0" - pip install "anyio==3.7.1" + pip install "httpx==0.28.1" + pip install "gunicorn==23.0.0" + pip install "anyio==4.8.0" pip install "aiodynamo==23.10.1" pip install "asyncio==3.4.3" pip install "PyGithub==1.59.1" @@ -2516,10 +1966,9 @@ jobs: pip install "pytest-asyncio==0.21.1" pip install "mypy==1.18.2" pip install "google-generativeai==0.3.2" - pip install "google-cloud-aiplatform==1.43.0" + pip install "google-cloud-aiplatform==1.133.0" pip install pyarrow - pip install "boto3==1.36.0" - pip install "aioboto3==13.4.0" + pip install "boto3==1.42.80" pip install langchain pip install "langfuse>=2.0.0" pip install "logfire==0.29.0" @@ -2527,9 +1976,9 @@ jobs: pip install prisma pip install fastapi pip install jsonschema - pip install "httpx==0.24.1" - pip install "gunicorn==21.2.0" - pip install "anyio==3.7.1" + pip install "httpx==0.28.1" + pip install "gunicorn==23.0.0" + pip install "anyio==4.8.0" pip install "aiodynamo==23.10.1" pip install "asyncio==3.4.3" pip install "PyGithub==1.59.1" @@ -2575,9 +2024,6 @@ jobs: -e OPENAI_API_KEY=$OPENAI_API_KEY \ -e LITELLM_LICENSE=$LITELLM_LICENSE \ -e OTEL_EXPORTER="in_memory" \ - -e APORIA_API_BASE_2=$APORIA_API_BASE_2 \ - -e APORIA_API_KEY_2=$APORIA_API_KEY_2 \ - -e APORIA_API_BASE_1=$APORIA_API_BASE_1 \ -e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID \ -e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY \ -e DEFAULT_NUM_WORKERS_LITELLM_PROXY=1 \ @@ -2585,7 +2031,6 @@ jobs: -e DD_API_KEY=$DD_API_KEY \ -e DD_SITE=$DD_SITE \ -e AWS_REGION_NAME=$AWS_REGION_NAME \ - -e APORIA_API_KEY_1=$APORIA_API_KEY_1 \ -e COHERE_API_KEY=$COHERE_API_KEY \ -e GCS_FLUSH_INTERVAL="1" \ --add-host host.docker.internal:host-gateway \ @@ -3061,6 +2506,20 @@ jobs: name: Build Docker image command: | docker build -t my-app:latest -f docker/build_from_pip/Dockerfile.build_from_pip . + - run: + name: Start PostgreSQL Database + command: | + docker run -d \ + --name postgres-db \ + -e POSTGRES_USER=postgres \ + -e POSTGRES_PASSWORD=postgres \ + -e POSTGRES_DB=circle_test \ + -p 5432:5432 \ + postgres:14 + - run: + name: Wait for PostgreSQL to be ready + command: | + timeout 60s bash -c 'until docker exec postgres-db pg_isready -U postgres -d circle_test; do sleep 2; done' - run: name: Run Docker container # intentionally give bad redis credentials here @@ -3068,7 +2527,7 @@ jobs: command: | docker run -d \ -p 4000:4000 \ - -e DATABASE_URL=$PROXY_DATABASE_URL \ + -e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \ -e REDIS_HOST=$REDIS_HOST \ -e REDIS_PASSWORD=$REDIS_PASSWORD \ -e REDIS_PORT=$REDIS_PORT \ @@ -3076,18 +2535,15 @@ jobs: -e OPENAI_API_KEY=$OPENAI_API_KEY \ -e LITELLM_LICENSE=$LITELLM_LICENSE \ -e OTEL_EXPORTER="in_memory" \ - -e APORIA_API_BASE_2=$APORIA_API_BASE_2 \ - -e APORIA_API_KEY_2=$APORIA_API_KEY_2 \ - -e APORIA_API_BASE_1=$APORIA_API_BASE_1 \ -e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID \ -e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY \ -e AWS_REGION_NAME=$AWS_REGION_NAME \ - -e APORIA_API_KEY_1=$APORIA_API_KEY_1 \ -e COHERE_API_KEY=$COHERE_API_KEY \ -e USE_DDTRACE=True \ -e DD_API_KEY=$DD_API_KEY \ -e DD_SITE=$DD_SITE \ -e GCS_FLUSH_INTERVAL="1" \ + --add-host host.docker.internal:host-gateway \ --name my-app \ -v $(pwd)/docker/build_from_pip/litellm_config.yaml:/app/config.yaml \ my-app:latest \ @@ -3118,8 +2574,11 @@ jobs: - run: name: Stop and remove first container command: | - docker stop my-app - docker rm my-app + docker stop my-app || true + docker rm my-app || true + docker stop postgres-db || true + docker rm postgres-db || true + when: always proxy_pass_through_endpoint_tests: machine: image: ubuntu-2204:2023.10.1 @@ -3148,16 +2607,16 @@ jobs: pip install "pytest==7.3.1" pip install "pytest-retry==1.6.3" pip install "pytest-asyncio==0.21.1" - pip install "google-cloud-aiplatform==1.43.0" + pip install "google-cloud-aiplatform==1.133.0" pip install aiohttp pip install "openai==1.100.1" pip install "assemblyai==0.37.0" python -m pip install --upgrade pip - pip install "pydantic==2.10.2" + pip install "pydantic==2.12.5" pip install "pytest==7.3.1" pip install "pytest-mock==3.12.0" pip install "pytest-asyncio==0.21.1" - pip install "boto3==1.36.0" + pip install "boto3==1.42.80" pip install "mypy==1.18.2" pip install pyarrow pip install numpydoc @@ -3165,11 +2624,11 @@ jobs: pip install fastapi pip install jsonschema pip install "httpx==0.27.0" - pip install "anyio==3.7.1" + pip install "anyio==4.8.0" pip install "asyncio==3.4.3" pip install "PyGithub==1.59.1" pip install "google-cloud-aiplatform==1.59.0" - pip install "anthropic==0.52.0" + pip install "anthropic==0.54.0" pip install "langchain_mcp_adapters==0.0.5" pip install "langchain_openai==0.2.1" pip install "langgraph==0.3.18" @@ -3336,7 +2795,7 @@ jobs: conda activate myenv pip install "pytest==7.3.1" pip install "pytest-asyncio==0.21.1" - pip install "boto3==1.36.0" + pip install "boto3==1.42.80" pip install "httpx==0.27.0" pip install "claude-agent-sdk" pip install -r requirements.txt @@ -3543,93 +3002,6 @@ jobs: - codecov/upload: file: ./coverage.xml - publish_to_pypi: - docker: - - image: cimg/python:3.8 - working_directory: ~/project - - environment: - TWINE_USERNAME: __token__ - - steps: - - checkout - - - run: - name: Copy model_prices_and_context_window File to model_prices_and_context_window_backup - command: | - cp model_prices_and_context_window.json litellm/model_prices_and_context_window_backup.json - - - run: - name: Checkout code - command: git checkout $CIRCLE_SHA1 - - # Check if setup.py is modified and publish to PyPI - - run: - name: PyPI publish - command: | - echo "Install TOML package." - python -m pip install toml - VERSION=$(python -c "import toml; print(toml.load('pyproject.toml')['tool']['poetry']['version'])") - PACKAGE_NAME=$(python -c "import toml; print(toml.load('pyproject.toml')['tool']['poetry']['name'])") - if ! pip show -v $PACKAGE_NAME | grep -q "Version: ${VERSION}"; then - echo "pyproject.toml modified" - echo -e "[pypi]\nusername = $PYPI_PUBLISH_USERNAME\npassword = $PYPI_PUBLISH_PASSWORD" > ~/.pypirc - python -m pip install --upgrade pip - pip install build - pip install wheel - pip install --upgrade twine setuptools - rm -rf build dist - - echo "Building package" - python -m build - - echo "Twine upload to dist" - echo "Contents of dist directory:" - ls dist/ - twine upload --verbose dist/* - else - echo "Version ${VERSION} of package is already published on PyPI." - - # Check if corresponding Docker nightly image exists - NIGHTLY_TAG="v${VERSION}-nightly" - echo "Checking for Docker nightly image: litellm/litellm:${NIGHTLY_TAG}" - - # Check Docker Hub for the nightly image - if curl -s "https://hub.docker.com/v2/repositories/litellm/litellm/tags/${NIGHTLY_TAG}" | grep -q "name"; then - echo "Docker nightly image ${NIGHTLY_TAG} exists. This release was already completed successfully." - echo "Skipping PyPI publish and continuing to ensure Docker images are up to date." - circleci step halt - else - echo "ERROR: PyPI package ${VERSION} exists but Docker nightly image ${NIGHTLY_TAG} does not exist!" - echo "This indicates an incomplete release. Please investigate." - exit 1 - fi - fi - - run: - name: Trigger Github Action for new Docker Container + Trigger Load Testing - command: | - echo "Install TOML package." - python3 -m pip install toml - VERSION=$(python3 -c "import toml; print(toml.load('pyproject.toml')['tool']['poetry']['version'])") - echo "LiteLLM Version ${VERSION}" - - # Determine which branch to use for Docker build - if [[ "$CIRCLE_BRANCH" =~ ^litellm_release_day_.* ]]; then - BUILD_BRANCH="$CIRCLE_BRANCH" - echo "Using release branch: $BUILD_BRANCH" - else - BUILD_BRANCH="main" - echo "Using default branch: $BUILD_BRANCH" - fi - - curl -X POST \ - -H "Accept: application/vnd.github.v3+json" \ - -H "Authorization: Bearer $GITHUB_TOKEN" \ - "https://api.github.com/repos/BerriAI/litellm/actions/workflows/ghcr_deploy.yml/dispatches" \ - -d "{\"ref\":\"${BUILD_BRANCH}\", \"inputs\":{\"tag\":\"v${VERSION}-nightly\", \"commit_hash\":\"$CIRCLE_SHA1\"}}" - echo "triggering load testing server for version ${VERSION} and commit ${CIRCLE_SHA1}" - curl -X POST "https://proxyloadtester-production.up.railway.app/start/load/test?version=${VERSION}&commit_hash=${CIRCLE_SHA1}&release_type=nightly" - publish_proxy_extras: docker: - image: cimg/python:3.8 @@ -3861,31 +3233,9 @@ jobs: name: Install Playwright Browsers command: | npx playwright install - - run: - name: Install Neon CLI - command: | - npm i -g neonctl - - run: - name: Create Neon branch - command: | - export EXPIRES_AT=$(date -u -d "+3 hours" +"%Y-%m-%dT%H:%M:%SZ") - echo "Expires at: $EXPIRES_AT" - neon branches create \ - --project-id $NEON_PROJECT_ID \ - --name preview/commit-${CIRCLE_SHA1:0:7}-<< parameters.browser >> \ - --expires-at $EXPIRES_AT \ - --parent br-fancy-paper-ad1olsb3 \ - --api-key $NEON_API_KEY || true - run: name: Run Docker container command: | - E2E_UI_TEST_DATABASE_URL=$(neon connection-string \ - --project-id $NEON_PROJECT_ID \ - --api-key $NEON_API_KEY \ - --branch preview/commit-${CIRCLE_SHA1:0:7}-<< parameters.browser >> \ - --database-name yuneng-trial-db \ - --role neondb_owner) - echo $E2E_UI_TEST_DATABASE_URL docker run -d \ -p 4000:4000 \ -e DATABASE_URL=$E2E_UI_TEST_DATABASE_URL \ @@ -3942,37 +3292,39 @@ jobs: - setup_google_dns - attach_workspace: at: ~/project + - run: + name: Install dockerize + command: | + sudo wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz + sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz + sudo rm dockerize-linux-amd64-v0.6.1.tar.gz + - run: + name: Start PostgreSQL Database + command: | + docker run -d \ + --name postgres-db \ + -e POSTGRES_USER=postgres \ + -e POSTGRES_PASSWORD=postgres \ + -e POSTGRES_DB=litellm_schema_sync \ + -p 5432:5432 \ + postgres:14 + - run: + name: Wait for PostgreSQL to be ready + command: dockerize -wait tcp://localhost:5432 -timeout 1m - run: name: Load Docker Database Image command: | zstd -d litellm-docker-database.tar.zst --stdout | docker load docker images | grep litellm-docker-database - run: - name: Install Neon CLI + name: Run schema sync via prisma db push command: | - npm i -g neonctl - - run: - name: Install curl and dockerize - command: | - sudo apt-get update - sudo apt-get install -y curl - sudo wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz - sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz - sudo rm dockerize-linux-amd64-v0.6.1.tar.gz - - run: - name: Sync schema on base e2e database - command: | - BASE_DATABASE_URL=$(neon connection-string \ - --project-id $NEON_PROJECT_ID \ - --api-key $NEON_API_KEY \ - --branch br-fancy-paper-ad1olsb3 \ - --database-name yuneng-trial-db \ - --role neondb_owner) docker run -d \ -p 4000:4000 \ - -e DATABASE_URL=$BASE_DATABASE_URL \ + -e DATABASE_URL="postgresql://postgres:postgres@host.docker.internal:5432/litellm_schema_sync" \ -e LITELLM_MASTER_KEY="sk-1234" \ --name schema-sync \ + --add-host=host.docker.internal:host-gateway \ -v $(pwd)/litellm/proxy/example_config_yaml/simple_config.yaml:/app/config.yaml \ litellm-docker-database:ci \ --config /app/config.yaml \ @@ -4089,34 +3441,14 @@ workflows: only: - main - /litellm_.*/ - - caching_unit_tests: - filters: - branches: - only: - main - /litellm_.*/ - - litellm_proxy_unit_testing_key_generation: - filters: - branches: - only: - main - /litellm_.*/ - - litellm_proxy_unit_testing_part1: - filters: - branches: - only: - main - /litellm_.*/ - - litellm_proxy_unit_testing_part2: - filters: - branches: - only: - main - /litellm_.*/ - - litellm_security_tests: - filters: - branches: - only: - main - /litellm_.*/ - litellm_assistants_api_testing: @@ -4170,7 +3502,6 @@ workflows: - main - /litellm_.*/ - prisma_schema_sync: - context: e2e_ui_tests requires: - build_docker_database_image filters: @@ -4178,32 +3509,32 @@ workflows: only: - main - /litellm_.*/ - - e2e_ui_testing: - name: e2e_ui_testing_chromium - browser: chromium - context: e2e_ui_tests - requires: - - ui_build - - build_docker_database_image - - prisma_schema_sync - filters: - branches: - only: - - main - - /litellm_.*/ - - e2e_ui_testing: - name: e2e_ui_testing_firefox - browser: firefox - context: e2e_ui_tests - requires: - - ui_build - - build_docker_database_image - - prisma_schema_sync - filters: - branches: - only: - - main - - /litellm_.*/ + # - e2e_ui_testing: + # name: e2e_ui_testing_chromium + # browser: chromium + # context: e2e_ui_tests + # requires: + # - ui_build + # - build_docker_database_image + # - prisma_schema_sync + # filters: + # branches: + # only: + # - main + # - /litellm_.*/ + # - e2e_ui_testing: + # name: e2e_ui_testing_firefox + # browser: firefox + # context: e2e_ui_tests + # requires: + # - ui_build + # - build_docker_database_image + # - prisma_schema_sync + # filters: + # branches: + # only: + # - main + # - /litellm_.*/ - build_and_test: requires: - build_docker_database_image @@ -4352,34 +3683,14 @@ workflows: only: - main - /litellm_.*/ - - litellm_mapped_tests_llms: - filters: - branches: - only: - main - /litellm_.*/ - - litellm_mapped_tests_core: - filters: - branches: - only: - main - /litellm_.*/ - - litellm_mapped_tests_mcps: - filters: - branches: - only: - main - /litellm_.*/ - - litellm_mapped_tests_integrations: - filters: - branches: - only: - main - /litellm_.*/ - - litellm_mapped_tests_litellm_core_utils: - filters: - branches: - only: - main - /litellm_.*/ - batches_testing: @@ -4429,11 +3740,6 @@ workflows: - search_testing - litellm_mapped_tests_proxy_part1 - litellm_mapped_tests_proxy_part2 - - litellm_mapped_tests_llms - - litellm_mapped_tests_core - - litellm_mapped_tests_mcps - - litellm_mapped_tests_integrations - - litellm_mapped_tests_litellm_core_utils - litellm_mapped_enterprise_tests - batches_testing - litellm_utils_testing @@ -4441,8 +3747,6 @@ workflows: - image_gen_testing - logging_testing - audio_testing - - caching_unit_tests - - litellm_proxy_unit_testing_key_generation - langfuse_logging_unit_tests - local_testing_part1 - local_testing_part2 @@ -4489,59 +3793,3 @@ workflows: only: - main - /litellm_release_day_.*/ - - publish_to_pypi: - requires: - - mypy_linting - - semgrep - - local_testing_part1 - - local_testing_part2 - - build_and_test - - e2e_openai_endpoints - - test_bad_database_url - - llm_translation_testing - - realtime_translation_testing - - mcp_testing - - agent_testing - - google_generate_content_endpoint_testing - - llm_responses_api_testing - - ocr_testing - - search_testing - - litellm_mapped_tests_proxy_part1 - - litellm_mapped_tests_proxy_part2 - - litellm_mapped_tests_llms - - litellm_mapped_tests_core - - litellm_mapped_tests_mcps - - litellm_mapped_tests_integrations - - litellm_mapped_tests_litellm_core_utils - - litellm_mapped_enterprise_tests - - batches_testing - - litellm_utils_testing - - pass_through_unit_testing - - image_gen_testing - - logging_testing - - audio_testing - - litellm_router_testing - - litellm_router_unit_testing - - caching_unit_tests - - langfuse_logging_unit_tests - - litellm_assistants_api_testing - - auth_ui_unit_tests - - ui_unit_tests - - db_migration_disable_update_check - - e2e_ui_testing_chromium - - e2e_ui_testing_firefox - - litellm_proxy_unit_testing_key_generation - - litellm_proxy_unit_testing_part1 - - litellm_proxy_unit_testing_part2 - - litellm_security_tests - - installing_litellm_on_python - - installing_litellm_on_python_3_13 - - proxy_logging_guardrails_model_info_tests - - proxy_spend_accuracy_tests - - proxy_multi_instance_tests - - proxy_store_model_in_db_tests - - proxy_build_from_pip_tests - - proxy_pass_through_endpoint_tests - - check_code_and_doc_quality - - publish_proxy_extras - - guardrails_testing diff --git a/.circleci/requirements.txt b/.circleci/requirements.txt index ab4c3995772..be12ab2d0f1 100644 --- a/.circleci/requirements.txt +++ b/.circleci/requirements.txt @@ -7,15 +7,15 @@ cohere redis==5.2.1 redisvl==0.4.1 anthropic -orjson==3.10.12 # fast /embedding responses -pydantic==2.11.0 -google-cloud-aiplatform==1.43.0 +orjson==3.10.15 # fast /embedding responses +pydantic==2.12.5 +google-cloud-aiplatform==1.133.0 google-cloud-iam==2.19.1 fastapi-sso==0.16.0 uvloop==0.21.0 -mcp==1.25.0 # for MCP server +mcp==1.26.0 # for MCP server semantic_router==0.1.10 # for auto-routing with litellm -fastuuid==0.12.0 +fastuuid==0.14.0 responses==0.25.7 # for proxy client tests pytest-retry==1.6.3 # for automatic test retries litellm-proxy-extras # for prisma migrations \ No newline at end of file diff --git a/.claude/settings.json b/.claude/settings.json deleted file mode 100644 index 8c1d85f96e0..00000000000 --- a/.claude/settings.json +++ /dev/null @@ -1,36 +0,0 @@ -{ - "permissions": { - "allow": [ - "Bash(git show:*)", - "Bash(git worktree add:*)", - "Read(//Users/krrishdholakia/Documents/litellm/**)", - "Read(//Users/krrishdholakia/Documents/litellm-claude-code-guardrails/litellm/types/**)", - "Read(//Users/krrishdholakia/Documents/litellm-claude-code-guardrails/**)", - "Read(//Users/krrishdholakia/Documents/litellm-claude-code-guardrails/litellm/**)", - "Bash(python:*)", - "Bash(python -c \"\nimport sys; sys.path.insert\\(0, ''.''\\)\nfrom litellm.proxy.guardrails.guardrail_hooks.claude_code.guardrail import ClaudeCodeGuardrail, HOSTED_TOOL_PREFIXES\nprint\\(''HOSTED_TOOL_PREFIXES:'', HOSTED_TOOL_PREFIXES\\)\nprint\\(''ClaudeCodeGuardrail imported OK''\\)\n\")", - "Read(//Users/krrishdholakia/Documents/litellm-mcp-jwt-groups/litellm/proxy/**)", - "Read(//Users/krrishdholakia/Documents/litellm-mcp-jwt-groups/**)", - "Bash(poetry run pytest:*)", - "Bash(git add:*)", - "Bash(git commit:*)", - "Bash(poetry run python:*)", - "Bash(poetry run pip:*)", - "Bash(git reset:*)", - "Bash(git cherry-pick:*)", - "Bash(git checkout:*)", - "Read(//Users/krrishdholakia/Documents/litellm/litellm/proxy/guardrails/guardrail_hooks/**)", - "Read(//Users/krrishdholakia/Documents/**)", - "Bash(git -C /Users/krrishdholakia/Documents/litellm-mcp-user-permissions worktree list)", - "Bash(ls:*)" - ], - "additionalDirectories": [ - "/Users/krrishdholakia/Documents/litellm-mcp-group-plan/plan", - "/Users/krrishdholakia/Documents/litellm-claude-code-guardrails/litellm/proxy/guardrails/guardrail_hooks/claude_code", - "/Users/krrishdholakia/Documents/litellm-claude-code-guardrails/litellm/types", - "/Users/krrishdholakia/Documents/litellm-claude-code-guardrails", - "/Users/krrishdholakia/Documents/litellm-mcp-jwt-groups/litellm/proxy", - "/Users/krrishdholakia/Documents/litellm-mcp-jwt-groups/tests/test_litellm/proxy/auth" - ] - } -} diff --git a/.devcontainer/post-create.sh b/.devcontainer/post-create.sh index bd72e91a20f..484baa9041d 100644 --- a/.devcontainer/post-create.sh +++ b/.devcontainer/post-create.sh @@ -12,6 +12,6 @@ echo "[post-create] Generating Prisma client" poetry run prisma generate echo "[post-create] Installing npm dependencies" -cd ui/litellm-dashboard && npm install --no-audit --no-fund +cd ui/litellm-dashboard && npm ci echo "[post-create] Done" \ No newline at end of file diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml index 4744ab048c7..cbf380bac01 100644 --- a/.github/ISSUE_TEMPLATE/config.yml +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -1,7 +1,7 @@ blank_issues_enabled: true contact_links: - name: Schedule Demo - url: https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions + url: https://enterprise.litellm.ai/demo about: Speak directly with Krrish and Ishaan, the founders, to discuss issues, share feedback, or explore improvements for LiteLLM - name: Discord url: https://discord.com/invite/wuPM9dRgDw diff --git a/.github/actions/helm-oci-chart-releaser/action.yml b/.github/actions/helm-oci-chart-releaser/action.yml index 1823e262832..454c591d436 100644 --- a/.github/actions/helm-oci-chart-releaser/action.yml +++ b/.github/actions/helm-oci-chart-releaser/action.yml @@ -41,32 +41,54 @@ runs: using: composite steps: - name: Helm | Setup - uses: azure/setup-helm@v4 + uses: azure/setup-helm@1a275c3b69536ee54be43f2070a358922e12c8d4 # v4.3.1 with: version: v3.20.0 - name: Helm | Login shell: bash - run: echo ${{ inputs.registry_password }} | helm registry login -u ${{ inputs.registry_username }} --password-stdin ${{ inputs.registry }} + env: + REGISTRY_PASSWORD: ${{ inputs.registry_password }} + REGISTRY_USERNAME: ${{ inputs.registry_username }} + REGISTRY: ${{ inputs.registry }} + run: echo "$REGISTRY_PASSWORD" | helm registry login -u "$REGISTRY_USERNAME" --password-stdin "$REGISTRY" - name: Helm | Dependency if: inputs.update_dependencies == 'true' shell: bash - run: helm dependency update ${{ inputs.path == null && format('{0}/{1}', 'charts', inputs.name) || inputs.path }} + env: + CHART_PATH: ${{ inputs.path == null && format('{0}/{1}', 'charts', inputs.name) || inputs.path }} + run: helm dependency update "$CHART_PATH" - name: Helm | Package shell: bash - run: helm package ${{ inputs.path == null && format('{0}/{1}', 'charts', inputs.name) || inputs.path }} --version ${{ inputs.tag }} --app-version ${{ inputs.app_version }} + env: + CHART_PATH: ${{ inputs.path == null && format('{0}/{1}', 'charts', inputs.name) || inputs.path }} + TAG: ${{ inputs.tag }} + APP_VERSION: ${{ inputs.app_version }} + run: helm package "$CHART_PATH" --version "$TAG" --app-version "$APP_VERSION" - name: Helm | Push shell: bash - run: helm push ${{ inputs.name }}-${{ inputs.tag }}.tgz oci://${{ inputs.registry }}/${{ inputs.repository }} + env: + NAME: ${{ inputs.name }} + TAG: ${{ inputs.tag }} + REGISTRY: ${{ inputs.registry }} + REPOSITORY: ${{ inputs.repository }} + run: helm push "${NAME}-${TAG}.tgz" "oci://${REGISTRY}/${REPOSITORY}" - name: Helm | Logout shell: bash - run: helm registry logout ${{ inputs.registry }} + env: + REGISTRY: ${{ inputs.registry }} + run: helm registry logout "$REGISTRY" - name: Helm | Output id: output shell: bash - run: echo "image=${{ inputs.registry }}/${{ inputs.repository }}/${{ inputs.name }}:${{ inputs.tag }}" >> $GITHUB_OUTPUT + env: + REGISTRY: ${{ inputs.registry }} + REPOSITORY: ${{ inputs.repository }} + NAME: ${{ inputs.name }} + TAG: ${{ inputs.tag }} + run: echo "image=${REGISTRY}/${REPOSITORY}/${NAME}:${TAG}" >> $GITHUB_OUTPUT diff --git a/.github/codeql/codeql-config.yml b/.github/codeql/codeql-config.yml index 20807685e12..36d70c1d746 100644 --- a/.github/codeql/codeql-config.yml +++ b/.github/codeql/codeql-config.yml @@ -1,22 +1,21 @@ name: "LiteLLM CodeQL config" -# Use security-extended suite instead of security-and-quality to avoid -# result sets > 2 GiB on this codebase that cause fatal OOM failures. queries: - - uses: security-extended + - uses: security-and-quality -# These two queries are security queries included in security-extended that -# individually produce result sets > 2 GiB on this codebase, causing fatal -# OOM failures. Exclude them as a safety net until CI confirms they no longer -# OOM; drop these exclusions in a follow-up once verified. +# Known OOM queries on large Python codebases: +# CodeQL builds a full data flow graph in memory. These two queries trace +# sensitive data through every log call / regex pattern, causing combinatorial +# path explosion on codebases with extensive logging like LiteLLM (>2 GiB +# result sets). This is a known CodeQL scaling limitation, not a code issue. +# Re-test periodically as CodeQL improves or the codebase refactors logging. query-filters: - exclude: - id: py/clear-text-logging-sensitive-data # CWE-312 — > 2 GiB result set + id: py/clear-text-logging-sensitive-data # CWE-312 - exclude: - id: py/polynomial-redos # CWE-730 — > 2 GiB result set + id: py/polynomial-redos # CWE-730 paths-ignore: - tests - docs - "**/*.md" - - litellm/proxy/_experimental/out diff --git a/.github/dependabot.yaml b/.github/dependabot.yaml index 58e7cfe10da..c49882a8d62 100644 --- a/.github/dependabot.yaml +++ b/.github/dependabot.yaml @@ -4,6 +4,9 @@ updates: directory: "/" schedule: interval: "daily" + cooldown: + default-days: 7 + semver-major-days: 14 groups: github-actions: patterns: diff --git a/.github/workflows/_test-unit-base.yml b/.github/workflows/_test-unit-base.yml new file mode 100644 index 00000000000..f1ae30e67d7 --- /dev/null +++ b/.github/workflows/_test-unit-base.yml @@ -0,0 +1,96 @@ +name: _Unit Test Base (Reusable) + +on: + workflow_call: + inputs: + test-path: + description: "Pytest path(s) to run" + required: true + type: string + workers: + description: "Number of pytest-xdist workers" + required: false + type: number + default: 2 + reruns: + description: "Number of reruns for flaky tests" + required: false + type: number + default: 2 + timeout-minutes: + description: "Job timeout in minutes" + required: false + type: number + default: 20 + max-failures: + description: "Stop after this many failures" + required: false + type: number + default: 10 + +permissions: + contents: read + +jobs: + run: + name: Run tests + runs-on: ubuntu-latest + timeout-minutes: ${{ inputs.timeout-minutes }} + + steps: + - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + with: + persist-credentials: false + + - name: Set up Python + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: "3.12" + + - name: Install Poetry + run: pip install 'poetry==2.3.2' + + - name: Cache Poetry dependencies + uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 + with: + path: | + ~/.cache/pypoetry + ~/.cache/pip + .venv + key: ${{ runner.os }}-poetry-${{ hashFiles('poetry.lock') }} + restore-keys: | + ${{ runner.os }}-poetry- + + - name: Install dependencies + run: | + poetry config virtualenvs.in-project true + poetry install --with dev,proxy-dev --extras "proxy semantic-router" + poetry run pip install google-genai==1.22.0 \ + google-cloud-aiplatform==1.115.0 fastapi-offline==1.7.3 python-multipart==0.0.22 openapi-core==0.23.0 + + - name: Setup litellm-enterprise + run: | + poetry run pip install --force-reinstall --no-deps -e enterprise/ + + - name: Generate Prisma client + env: + PRISMA_BINARY_CACHE_DIR: ${{ runner.temp }}/prisma-cache + run: | + poetry run pip install nodejs-wheel-binaries==24.13.1 + poetry run prisma generate --schema litellm/proxy/schema.prisma + + - name: Run tests + env: + TEST_PATH: ${{ inputs.test-path }} + MAX_FAILURES: ${{ inputs.max-failures }} + WORKERS: ${{ inputs.workers }} + RERUNS: ${{ inputs.reruns }} + run: | + poetry run pytest ${TEST_PATH:?} \ + --tb=short -vv \ + --maxfail="${MAX_FAILURES}" \ + -n "${WORKERS}" \ + --reruns "${RERUNS}" \ + --reruns-delay 1 \ + --dist=loadscope \ + --durations=20 diff --git a/.github/workflows/_test-unit-services-base.yml b/.github/workflows/_test-unit-services-base.yml new file mode 100644 index 00000000000..d53a9e8822a --- /dev/null +++ b/.github/workflows/_test-unit-services-base.yml @@ -0,0 +1,164 @@ +name: _Unit Test Services Base (Reusable) + +on: + workflow_call: + inputs: + test-path: + description: "Pytest path(s) to run" + required: true + type: string + workers: + description: "Number of pytest-xdist workers (0 = no parallelism)" + required: false + type: number + default: 2 + reruns: + description: "Number of reruns for flaky tests" + required: false + type: number + default: 2 + timeout-minutes: + description: "Job timeout in minutes" + required: false + type: number + default: 20 + max-failures: + description: "Stop after this many failures" + required: false + type: number + default: 10 + enable-redis: + description: "Pass Redis Cloud credentials to tests via REDIS_HOST/PORT/PASSWORD env vars" + required: false + type: boolean + default: false + enable-postgres: + description: "Start a local Postgres service container and run Prisma migrations" + required: false + type: boolean + default: false + secrets: + REDIS_HOST: + required: false + REDIS_PORT: + required: false + REDIS_PASSWORD: + required: false + DATABASE_URL: + required: false + POSTGRES_USER: + required: false + POSTGRES_PASSWORD: + required: false + +permissions: + contents: read + +jobs: + run: + name: Run tests + runs-on: ubuntu-latest + timeout-minutes: ${{ inputs.timeout-minutes }} + # Environment is derived from the enable-* flags, not caller-controllable. + # This prevents callers from passing arbitrary environment names to bypass secret scoping. + # Note: Postgres service container always starts (GHA limitation), so any Redis job + # also needs Postgres secrets → uses integration-redis-postgres, not integration-redis. + environment: >- + ${{ + inputs.enable-redis && 'integration-redis-postgres' || + inputs.enable-postgres && 'integration-postgres' || + '' + }} + + services: + postgres: + image: postgres@sha256:705a5d5b5836f3fcba0d02c4d281e6a7dd9ed2dd4078640f08a1e1e9896e097d # postgres:14 + env: + POSTGRES_USER: ${{ secrets.POSTGRES_USER }} + POSTGRES_PASSWORD: ${{ secrets.POSTGRES_PASSWORD }} + POSTGRES_DB: litellm_test + ports: + - 5432:5432 + options: >- + --health-cmd "pg_isready" + --health-interval 10s + --health-timeout 5s + --health-retries 5 + + steps: + - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + with: + persist-credentials: false + + - name: Set up Python + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: "3.12" + + - name: Install Poetry + run: pip install 'poetry==2.3.2' + + - name: Cache Poetry dependencies + uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 + with: + path: | + ~/.cache/pypoetry + ~/.cache/pip + .venv + key: ${{ runner.os }}-poetry-services-${{ hashFiles('poetry.lock') }} + restore-keys: | + ${{ runner.os }}-poetry-services- + + - name: Install dependencies + run: | + poetry config virtualenvs.in-project true + poetry install --with dev,proxy-dev --extras "proxy semantic-router" + poetry run pip install google-genai==1.22.0 \ + google-cloud-aiplatform==1.115.0 fastapi-offline==1.7.3 python-multipart==0.0.22 openapi-core==0.23.0 + + - name: Setup litellm-enterprise + run: | + poetry run pip install --force-reinstall --no-deps -e enterprise/ + + - name: Generate Prisma client + env: + PRISMA_BINARY_CACHE_DIR: ${{ runner.temp }}/prisma-cache + run: | + poetry run pip install nodejs-wheel-binaries==24.13.1 + poetry run prisma generate --schema litellm/proxy/schema.prisma + + - name: Run Prisma migrations + if: ${{ inputs.enable-postgres }} + env: + DATABASE_URL: ${{ secrets.DATABASE_URL }} + run: | + poetry run prisma db push --schema litellm/proxy/schema.prisma --accept-data-loss + + - name: Run tests + env: + TEST_PATH: ${{ inputs.test-path }} + MAX_FAILURES: ${{ inputs.max-failures }} + WORKERS: ${{ inputs.workers }} + RERUNS: ${{ inputs.reruns }} + DATABASE_URL: ${{ inputs.enable-postgres && secrets.DATABASE_URL || '' }} + REDIS_HOST: ${{ inputs.enable-redis && secrets.REDIS_HOST || '' }} + REDIS_PORT: ${{ inputs.enable-redis && secrets.REDIS_PORT || '' }} + REDIS_PASSWORD: ${{ inputs.enable-redis && secrets.REDIS_PASSWORD || '' }} + run: | + if [ "${WORKERS}" = "0" ]; then + poetry run pytest ${TEST_PATH:?} \ + --tb=short -vv \ + --maxfail="${MAX_FAILURES}" \ + --reruns "${RERUNS}" \ + --reruns-delay 1 \ + --durations=20 + else + poetry run pytest ${TEST_PATH:?} \ + --tb=short -vv \ + --maxfail="${MAX_FAILURES}" \ + -n "${WORKERS}" \ + --reruns "${RERUNS}" \ + --reruns-delay 1 \ + --dist=loadscope \ + --durations=20 + fi diff --git a/.github/workflows/auto_update_price_and_context_window.yml b/.github/workflows/auto_update_price_and_context_window.yml index 98b9d868e68..60e89936219 100644 --- a/.github/workflows/auto_update_price_and_context_window.yml +++ b/.github/workflows/auto_update_price_and_context_window.yml @@ -2,18 +2,24 @@ name: Updates model_prices_and_context_window.json and Create Pull Request on: schedule: - - cron: "0 0 * * 0" # Run every Sundays at midnight + - cron: "0 0 * * 0" # Run every Sundays at midnight #- cron: "0 0 * * *" # Run daily at midnight +permissions: + contents: write + pull-requests: write + jobs: auto_update_price_and_context_window: if: github.repository == 'BerriAI/litellm' runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + with: + persist-credentials: false - name: Install Dependencies run: | - pip install aiohttp + pip install 'aiohttp==3.13.3' - name: Update JSON Data run: | python ".github/workflows/auto_update_price_and_context_window_file.py" @@ -26,4 +32,4 @@ jobs: --head auto-update-price-and-context-window-$(date +'%Y-%m-%d') \ --base main env: - GH_TOKEN: ${{ secrets.GH_TOKEN }} \ No newline at end of file + GH_TOKEN: ${{ secrets.GH_TOKEN }} diff --git a/.github/workflows/check-schema-sync.yml b/.github/workflows/check-schema-sync.yml new file mode 100644 index 00000000000..0e5e2804e60 --- /dev/null +++ b/.github/workflows/check-schema-sync.yml @@ -0,0 +1,58 @@ +name: Check Schema Sync + +on: + pull_request: + paths: + - 'schema.prisma' + - 'litellm/proxy/schema.prisma' + - 'litellm-proxy-extras/litellm_proxy_extras/schema.prisma' + +permissions: + contents: read + +jobs: + check-sync: + name: Verify schema.prisma copies match root + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - name: Checkout PR + uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + with: + persist-credentials: false + + - name: Reject symlinked schema files + run: | + for f in schema.prisma litellm/proxy/schema.prisma litellm-proxy-extras/litellm_proxy_extras/schema.prisma; do + if [ -L "$f" ]; then + echo "::error file=$f::$f is a symlink, which is not allowed" + exit 1 + fi + done + + - name: Check all schemas match root + run: | + EXIT=0 + + diff schema.prisma litellm/proxy/schema.prisma || { + echo "::error file=litellm/proxy/schema.prisma::litellm/proxy/schema.prisma differs from root schema.prisma" + EXIT=1 + } + + diff schema.prisma litellm-proxy-extras/litellm_proxy_extras/schema.prisma || { + echo "::error file=litellm-proxy-extras/litellm_proxy_extras/schema.prisma::litellm-proxy-extras/litellm_proxy_extras/schema.prisma differs from root schema.prisma" + EXIT=1 + } + + if [ "$EXIT" -ne 0 ]; then + echo "" + echo "Schema files are out of sync." + echo "The root schema.prisma is the source of truth." + echo "" + echo "To fix, run from the repo root:" + echo " cp schema.prisma litellm/proxy/schema.prisma" + echo " cp schema.prisma litellm-proxy-extras/litellm_proxy_extras/schema.prisma" + exit 1 + fi + + echo "All schema copies are in sync with root." diff --git a/.github/workflows/check_duplicate_issues.yml b/.github/workflows/check_duplicate_issues.yml index 6d11ce573eb..289d78880ad 100644 --- a/.github/workflows/check_duplicate_issues.yml +++ b/.github/workflows/check_duplicate_issues.yml @@ -12,7 +12,7 @@ jobs: contents: read steps: - name: Check for potential duplicates - uses: wow-actions/potential-duplicates@v1 + uses: wow-actions/potential-duplicates@4d4ea0352e0383859279938e255179dd1dbb67b5 # v1.1.0 with: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} label: potential-duplicate @@ -30,13 +30,14 @@ jobs: - name: Checkout close script if: github.event.action == 'opened' - uses: actions/checkout@v4 + uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 with: sparse-checkout: .github/scripts + persist-credentials: false - name: Set up Python if: github.event.action == 'opened' - uses: actions/setup-python@v5 + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version: "3.11" diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 0b7cce2e4be..e86fca17c7a 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -6,8 +6,8 @@ on: pull_request: branches: [main] schedule: - # Run weekly on Sundays at 04:00 UTC - - cron: "0 4 * * 0" + # Run daily at 04:00 UTC + - cron: "0 4 * * *" concurrency: group: ${{ github.workflow }}-${{ github.ref }} @@ -15,6 +15,7 @@ concurrency: jobs: analyze: + if: github.event_name != 'schedule' || github.repository == 'BerriAI/litellm' name: Analyze (${{ matrix.language }}) runs-on: ubuntu-latest timeout-minutes: 30 @@ -37,16 +38,18 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v4 + uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + with: + persist-credentials: false - name: Initialize CodeQL - uses: github/codeql-action/init@v3 + uses: github/codeql-action/init@ebcb5b36ded6beda4ceefea6a8bc4cc885255bb3 # v3 with: languages: ${{ matrix.language }} build-mode: ${{ matrix.build-mode }} config-file: ./.github/codeql/codeql-config.yml - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v3 + uses: github/codeql-action/analyze@ebcb5b36ded6beda4ceefea6a8bc4cc885255bb3 # v3 with: category: "/language:${{ matrix.language }}" diff --git a/.github/workflows/codspeed.yml b/.github/workflows/codspeed.yml index 385b95fdaf5..52d64addea9 100644 --- a/.github/workflows/codspeed.yml +++ b/.github/workflows/codspeed.yml @@ -25,10 +25,12 @@ jobs: timeout-minutes: 15 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + with: + persist-credentials: false - name: Set up Python - uses: actions/setup-python@v5 + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version: "3.12" @@ -38,7 +40,7 @@ jobs: pip install pytest pytest-codspeed==4.3.0 - name: Run benchmarks - uses: CodSpeedHQ/action@v4 + uses: CodSpeedHQ/action@1c8ae4843586d3ba879736b7f6b7b0c990757fab # v4.12.1 with: mode: simulation run: pytest tests/benchmarks/ --codspeed diff --git a/.github/workflows/create-release.yml b/.github/workflows/create-release.yml new file mode 100644 index 00000000000..2ae01823a96 --- /dev/null +++ b/.github/workflows/create-release.yml @@ -0,0 +1,93 @@ +name: Create Release + +on: + workflow_dispatch: + inputs: + tag: + description: "Release tag (e.g. v1.83.0-stable)" + required: true + type: string + commit_hash: + description: "Full 40-char commit SHA to target" + required: true + type: string + +permissions: {} + +jobs: + release: + name: Create Release + runs-on: ubuntu-latest + permissions: + contents: write + steps: + - name: Validate inputs + env: + TAG: ${{ inputs.tag }} + COMMIT_HASH: ${{ inputs.commit_hash }} + run: | + if ! echo "${COMMIT_HASH}" | grep -qE '^[0-9a-f]{40}$'; then + echo "::error::commit_hash must be a full 40-character commit SHA" + exit 1 + fi + if ! echo "${TAG}" | grep -qE '^v[0-9]+\.[0-9]+\.[0-9]+'; then + echo "::error::tag must start with vX.Y.Z" + exit 1 + fi + + - name: Create release + env: + TAG: ${{ inputs.tag }} + COMMIT_HASH: ${{ inputs.commit_hash }} + uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1 + with: + script: | + const tag = process.env.TAG; + const commitHash = process.env.COMMIT_HASH; + + const cosignSection = [ + `## Verify Docker Image Signature`, + ``, + `All LiteLLM Docker images are signed with [cosign](https://docs.sigstore.dev/cosign/overview/). To verify the integrity of an image before deploying:`, + ``, + '```bash', + `cosign verify \\`, + ` --key https://raw.githubusercontent.com/BerriAI/litellm/${tag}/cosign.pub \\`, + ` ghcr.io/berriai/litellm:${tag}`, + '```', + ``, + `Expected output:`, + ``, + '```', + `The following checks were performed on each of these signatures:`, + ` - The cosign claims were validated`, + ` - The signatures were verified against the specified public key`, + '```', + ``, + `---`, + ``, + ].join('\n'); + + try { + const response = await github.rest.repos.createRelease({ + draft: true, + generate_release_notes: true, + target_commitish: commitHash, + name: tag, + owner: context.repo.owner, + prerelease: false, + repo: context.repo.repo, + tag_name: tag, + }); + + const updatedBody = cosignSection + (response.data.body ?? ''); + await github.rest.repos.updateRelease({ + owner: context.repo.owner, + repo: context.repo.repo, + release_id: response.data.id, + body: updatedBody, + draft: false, + }); + } catch (error) { + core.setFailed(error.message); + } diff --git a/.github/workflows/create_daily_staging_branch.yml b/.github/workflows/create_daily_staging_branch.yml index 08aebd7d04c..424d8de0a41 100644 --- a/.github/workflows/create_daily_staging_branch.yml +++ b/.github/workflows/create_daily_staging_branch.yml @@ -2,18 +2,22 @@ name: Create Daily Staging Branch on: schedule: - - cron: '0 0,12 * * *' # Runs every 12 hours at midnight and noon UTC - workflow_dispatch: # Allow manual trigger + - cron: "0 0,12 * * *" # Runs every 12 hours at midnight and noon UTC + workflow_dispatch: # Allow manual trigger jobs: create-staging-branch: + if: github.repository == 'BerriAI/litellm' runs-on: ubuntu-latest + permissions: + contents: write steps: - name: Checkout repository - uses: actions/checkout@v3 + uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 with: fetch-depth: 0 + persist-credentials: false - name: Create daily staging branch env: @@ -43,13 +47,17 @@ jobs: fi create-internal-dev-branch: + if: github.repository == 'BerriAI/litellm' runs-on: ubuntu-latest + permissions: + contents: write steps: - name: Checkout repository - uses: actions/checkout@v3 + uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 with: fetch-depth: 0 + persist-credentials: false - name: Create internal dev branch env: diff --git a/.github/workflows/ghcr_deploy.yml b/.github/workflows/ghcr_deploy.yml deleted file mode 100644 index 344b0ec48ee..00000000000 --- a/.github/workflows/ghcr_deploy.yml +++ /dev/null @@ -1,444 +0,0 @@ -# this workflow is triggered by an API call when there is a new PyPI release of LiteLLM -name: Build, Publish LiteLLM Docker Image. New Release -on: - workflow_dispatch: - inputs: - tag: - description: "The tag version you want to build" - required: true - release_type: - description: "The release type you want to build. Can be 'latest', 'stable', 'dev', 'rc'" - type: string - default: "latest" - commit_hash: - description: "Commit hash" - required: true - -# Defines two custom environment variables for the workflow. Used for the Container registry domain, and a name for the Docker image that this workflow builds. -env: - REGISTRY: ghcr.io - IMAGE_NAME: ${{ github.repository }} - CHART_NAME: litellm-helm - -# There is a single job in this workflow. It's configured to run on the latest available version of Ubuntu. -jobs: - # print commit hash, tag, and release type - print: - runs-on: ubuntu-latest - steps: - - run: | - echo "Commit hash: ${{ github.event.inputs.commit_hash }}" - echo "Tag: ${{ github.event.inputs.tag }}" - echo "Release type: ${{ github.event.inputs.release_type }}" - docker-hub-deploy: - if: github.repository == 'BerriAI/litellm' - runs-on: ubuntu-latest - steps: - - - name: Checkout - uses: actions/checkout@v4 - with: - ref: ${{ github.event.inputs.commit_hash }} - - - name: Set up QEMU - uses: docker/setup-qemu-action@v3 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - - name: Login to Docker Hub - uses: docker/login-action@v3 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - - name: Build and push - uses: docker/build-push-action@v5 - with: - context: . - push: true - tags: litellm/litellm:${{ github.event.inputs.tag || 'latest' }} - - - name: Build and push litellm-database image - uses: docker/build-push-action@v5 - with: - context: . - push: true - file: ./docker/Dockerfile.database - tags: litellm/litellm-database:${{ github.event.inputs.tag || 'latest' }} - - - name: Build and push litellm-spend-logs image - uses: docker/build-push-action@v5 - with: - context: . - push: true - file: ./litellm-js/spend-logs/Dockerfile - tags: litellm/litellm-spend_logs:${{ github.event.inputs.tag || 'latest' }} - - - name: Build and push litellm-non_root image - uses: docker/build-push-action@v5 - with: - context: . - push: true - file: ./docker/Dockerfile.non_root - tags: litellm/litellm-non_root:${{ github.event.inputs.tag || 'latest' }} - build-and-push-image: - runs-on: ubuntu-latest - # Sets the permissions granted to the `GITHUB_TOKEN` for the actions in this job. - permissions: - contents: read - packages: write - steps: - - name: Checkout repository - uses: actions/checkout@v4 - with: - ref: ${{ github.event.inputs.commit_hash }} - # Uses the `docker/login-action` action to log in to the Container registry registry using the account and password that will publish the packages. Once published, the packages are scoped to the account defined here. - - name: Log in to the Container registry - uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1 - with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - # This step uses [docker/metadata-action](https://github.com/docker/metadata-action#about) to extract tags and labels that will be applied to the specified image. The `id` "meta" allows the output of this step to be referenced in a subsequent step. The `images` value provides the base name for the tags and labels. - - name: Extract metadata (tags, labels) for Docker - id: meta - uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7 - with: - images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} - # Configure multi platform Docker builds - - name: Set up QEMU - uses: docker/setup-qemu-action@e0e4588fad221d38ee467c0bffd91115366dc0c5 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@edfb0fe6204400c56fbfd3feba3fe9ad1adfa345 - # This step uses the `docker/build-push-action` action to build the image, based on your repository's `Dockerfile`. If the build succeeds, it pushes the image to GitHub Packages. - # It uses the `context` parameter to define the build's context as the set of files located in the specified path. For more information, see "[Usage](https://github.com/docker/build-push-action#usage)" in the README of the `docker/build-push-action` repository. - # It uses the `tags` and `labels` parameters to tag and label the image with the output from the "meta" step. - - name: Build and push Docker image - uses: docker/build-push-action@4976231911ebf5f32aad765192d35f942aa48cb8 - with: - context: . - push: true - tags: | - ${{ steps.meta.outputs.tags }}-${{ github.event.inputs.tag || 'latest' }}, - ${{ steps.meta.outputs.tags }}-${{ github.event.inputs.release_type }} - ${{ (github.event.inputs.release_type == 'stable' || github.event.inputs.release_type == 'rc') && format('{0}/berriai/litellm:main-{1}', env.REGISTRY, github.event.inputs.tag) || '' }}, - ${{ github.event.inputs.release_type == 'stable' && format('{0}/berriai/litellm:main-stable', env.REGISTRY) || '' }}, - ${{ (github.event.inputs.release_type == 'stable' || github.event.inputs.release_type == 'rc') && format('{0}/berriai/litellm:{1}', env.REGISTRY, github.event.inputs.tag) || '' }}, - labels: ${{ steps.meta.outputs.labels }} - platforms: local,linux/amd64,linux/arm64,linux/arm64/v8 - - build-and-push-image-ee: - runs-on: ubuntu-latest - permissions: - contents: read - packages: write - steps: - - name: Checkout repository - uses: actions/checkout@v4 - with: - ref: ${{ github.event.inputs.commit_hash }} - - - name: Log in to the Container registry - uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1 - with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Extract metadata (tags, labels) for EE Dockerfile - id: meta-ee - uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7 - with: - images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee - # Configure multi platform Docker builds - - name: Set up QEMU - uses: docker/setup-qemu-action@e0e4588fad221d38ee467c0bffd91115366dc0c5 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@edfb0fe6204400c56fbfd3feba3fe9ad1adfa345 - - - name: Build and push EE Docker image - uses: docker/build-push-action@f2a1d5e99d037542a71f64918e516c093c6f3fc4 - with: - context: . - file: Dockerfile - push: true - tags: | - ${{ steps.meta-ee.outputs.tags }}-${{ github.event.inputs.tag || 'latest' }}, - ${{ steps.meta-ee.outputs.tags }}-${{ github.event.inputs.release_type }} - ${{ (github.event.inputs.release_type == 'stable' || github.event.inputs.release_type == 'rc') && format('{0}/berriai/litellm-ee:main-{1}', env.REGISTRY, github.event.inputs.tag) || '' }}, - ${{ github.event.inputs.release_type == 'stable' && format('{0}/berriai/litellm-ee:main-stable', env.REGISTRY) || '' }} - labels: ${{ steps.meta-ee.outputs.labels }} - platforms: local,linux/amd64,linux/arm64,linux/arm64/v8 - - build-and-push-image-database: - runs-on: ubuntu-latest - permissions: - contents: read - packages: write - steps: - - name: Checkout repository - uses: actions/checkout@v4 - with: - ref: ${{ github.event.inputs.commit_hash }} - - - name: Log in to the Container registry - uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1 - with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Extract metadata (tags, labels) for database Dockerfile - id: meta-database - uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7 - with: - images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-database - # Configure multi platform Docker builds - - name: Set up QEMU - uses: docker/setup-qemu-action@e0e4588fad221d38ee467c0bffd91115366dc0c5 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@edfb0fe6204400c56fbfd3feba3fe9ad1adfa345 - - - name: Build and push Database Docker image - uses: docker/build-push-action@f2a1d5e99d037542a71f64918e516c093c6f3fc4 - with: - context: . - file: ./docker/Dockerfile.database - push: true - tags: | - ${{ steps.meta-database.outputs.tags }}-${{ github.event.inputs.tag || 'latest' }}, - ${{ steps.meta-database.outputs.tags }}-${{ github.event.inputs.release_type }} - ${{ (github.event.inputs.release_type == 'stable' || github.event.inputs.release_type == 'rc') && format('{0}/berriai/litellm-database:main-{1}', env.REGISTRY, github.event.inputs.tag) || '' }}, - ${{ github.event.inputs.release_type == 'stable' && format('{0}/berriai/litellm-database:main-stable', env.REGISTRY) || '' }} - labels: ${{ steps.meta-database.outputs.labels }} - platforms: local,linux/amd64,linux/arm64,linux/arm64/v8 - - build-and-push-image-non_root: - runs-on: ubuntu-latest - permissions: - contents: read - packages: write - steps: - - name: Checkout repository - uses: actions/checkout@v4 - with: - ref: ${{ github.event.inputs.commit_hash }} - - - name: Log in to the Container registry - uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1 - with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Extract metadata (tags, labels) for non_root Dockerfile - id: meta-non_root - uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7 - with: - images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-non_root - # Configure multi platform Docker builds - - name: Set up QEMU - uses: docker/setup-qemu-action@e0e4588fad221d38ee467c0bffd91115366dc0c5 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@edfb0fe6204400c56fbfd3feba3fe9ad1adfa345 - - - name: Build and push non_root Docker image - uses: docker/build-push-action@f2a1d5e99d037542a71f64918e516c093c6f3fc4 - with: - context: . - file: ./docker/Dockerfile.non_root - push: true - tags: | - ${{ steps.meta-non_root.outputs.tags }}-${{ github.event.inputs.tag || 'latest' }}, - ${{ steps.meta-non_root.outputs.tags }}-${{ github.event.inputs.release_type }} - ${{ (github.event.inputs.release_type == 'stable' || github.event.inputs.release_type == 'rc') && format('{0}/berriai/litellm-non_root:main-{1}', env.REGISTRY, github.event.inputs.tag) || '' }}, - ${{ github.event.inputs.release_type == 'stable' && format('{0}/berriai/litellm-non_root:main-stable', env.REGISTRY) || '' }} - labels: ${{ steps.meta-non_root.outputs.labels }} - platforms: local,linux/amd64,linux/arm64,linux/arm64/v8 - - build-and-push-image-spend-logs: - runs-on: ubuntu-latest - permissions: - contents: read - packages: write - steps: - - name: Checkout repository - uses: actions/checkout@v4 - with: - ref: ${{ github.event.inputs.commit_hash }} - - - name: Log in to the Container registry - uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1 - with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Extract metadata (tags, labels) for spend-logs Dockerfile - id: meta-spend-logs - uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7 - with: - images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-spend_logs - # Configure multi platform Docker builds - - name: Set up QEMU - uses: docker/setup-qemu-action@e0e4588fad221d38ee467c0bffd91115366dc0c5 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@edfb0fe6204400c56fbfd3feba3fe9ad1adfa345 - - - name: Build and push Database Docker image - uses: docker/build-push-action@f2a1d5e99d037542a71f64918e516c093c6f3fc4 - with: - context: . - file: ./litellm-js/spend-logs/Dockerfile - push: true - tags: | - ${{ steps.meta-spend-logs.outputs.tags }}-${{ github.event.inputs.tag || 'latest' }}, - ${{ steps.meta-spend-logs.outputs.tags }}-${{ github.event.inputs.release_type }} - ${{ (github.event.inputs.release_type == 'stable' || github.event.inputs.release_type == 'rc') && format('{0}/berriai/litellm-spend_logs:main-{1}', env.REGISTRY, github.event.inputs.tag) || '' }}, - ${{ github.event.inputs.release_type == 'stable' && format('{0}/berriai/litellm-spend_logs:main-stable', env.REGISTRY) || '' }} - platforms: local,linux/amd64,linux/arm64,linux/arm64/v8 - - run-observatory-tests: - if: github.event.inputs.release_type == 'rc' || github.event.inputs.release_type == 'stable' - needs: [docker-hub-deploy] - uses: ./.github/workflows/run_observatory_tests.yml - with: - tag: ${{ github.event.inputs.tag }} - commit_hash: ${{ github.event.inputs.commit_hash }} - secrets: inherit - - build-and-push-helm-chart: - if: github.event.inputs.release_type != 'dev' - needs: [docker-hub-deploy, build-and-push-image, build-and-push-image-database] - runs-on: ubuntu-latest - steps: - - name: Checkout repository - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - name: Log in to the Container registry - uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1 - with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: lowercase github.repository_owner - run: | - echo "REPO_OWNER=`echo ${{github.repository_owner}} | tr '[:upper:]' '[:lower:]'`" >>${GITHUB_ENV} - - # Sync Helm chart version with LiteLLM release version (1-1 versioning) - # This allows users to easily map Helm chart versions to LiteLLM versions - # See: https://codefresh.io/docs/docs/ci-cd-guides/helm-best-practices/ - - name: Calculate chart and app versions - id: chart_version - shell: bash - run: | - INPUT_TAG="${{ github.event.inputs.tag }}" - RELEASE_TYPE="${{ github.event.inputs.release_type }}" - - # Chart version = LiteLLM version without 'v' prefix (Helm semver convention) - # v1.81.0 -> 1.81.0, v1.81.0.rc.1 -> 1.81.0.rc.1 - CHART_VERSION="${INPUT_TAG#v}" - - # Add suffix for 'latest' releases (rc already has suffix in tag) - if [ "$RELEASE_TYPE" = "latest" ]; then - CHART_VERSION="${CHART_VERSION}-latest" - fi - - # App version = Docker tag (keeps 'v' prefix to match Docker image tags) - APP_VERSION="${INPUT_TAG}" - - echo "version=${CHART_VERSION}" | tee -a $GITHUB_OUTPUT - echo "app_version=${APP_VERSION}" | tee -a $GITHUB_OUTPUT - - - uses: ./.github/actions/helm-oci-chart-releaser - with: - name: ${{ env.CHART_NAME }} - repository: ${{ env.REPO_OWNER }} - tag: ${{ steps.chart_version.outputs.version }} - app_version: ${{ steps.chart_version.outputs.app_version }} - path: deploy/charts/${{ env.CHART_NAME }} - registry: ${{ env.REGISTRY }} - registry_username: ${{ github.actor }} - registry_password: ${{ secrets.GITHUB_TOKEN }} - update_dependencies: true - - release: - name: "New LiteLLM Release" - needs: [docker-hub-deploy, build-and-push-image, build-and-push-image-database] - permissions: - contents: write - runs-on: "ubuntu-latest" - - steps: - - name: Display version - run: echo "Current version is ${{ github.event.inputs.tag }}" - - name: "Set Release Tag" - run: echo "RELEASE_TAG=${{ github.event.inputs.tag }}" >> $GITHUB_ENV - - name: Display release tag - run: echo "RELEASE_TAG is $RELEASE_TAG" - - name: "Create release" - uses: "actions/github-script@v6" - with: - github-token: "${{ secrets.GITHUB_TOKEN }}" - script: | - const commitHash = "${{ github.event.inputs.commit_hash}}"; - console.log("Commit Hash:", commitHash); // Add this line for debugging - try { - const response = await github.rest.repos.createRelease({ - draft: false, - generate_release_notes: true, - target_commitish: commitHash, - name: process.env.RELEASE_TAG, - owner: context.repo.owner, - prerelease: false, - repo: context.repo.repo, - tag_name: process.env.RELEASE_TAG, - }); - - core.exportVariable('RELEASE_ID', response.data.id); - core.exportVariable('RELEASE_UPLOAD_URL', response.data.upload_url); - } catch (error) { - core.setFailed(error.message); - } - - name: Fetch Release Notes - id: release-notes - uses: actions/github-script@v6 - with: - github-token: "${{ secrets.GITHUB_TOKEN }}" - script: | - try { - const response = await github.rest.repos.getRelease({ - owner: context.repo.owner, - repo: context.repo.repo, - release_id: process.env.RELEASE_ID, - }); - const formattedBody = JSON.stringify(response.data.body).slice(1, -1); - return formattedBody; - } catch (error) { - core.setFailed(error.message); - } - env: - RELEASE_ID: ${{ env.RELEASE_ID }} - - name: Github Releases To Discord - env: - WEBHOOK_URL: ${{ secrets.WEBHOOK_URL }} - REALEASE_TAG: ${{ env.RELEASE_TAG }} - RELEASE_NOTES: ${{ steps.release-notes.outputs.result }} - run: | - curl -H "Content-Type: application/json" -X POST -d '{ - "content": "New LiteLLM release '"${RELEASE_TAG}"'", - "username": "Release Changelog", - "avatar_url": "https://cdn.discordapp.com/avatars/487431320314576937/bd64361e4ba6313d561d54e78c9e7171.png", - "embeds": [ - { - "title": "Changelog for LiteLLM '"${RELEASE_TAG}"'", - "description": "'"${RELEASE_NOTES}"'", - "color": 2105893 - } - ] - }' $WEBHOOK_URL - diff --git a/.github/workflows/ghcr_helm_deploy.yml b/.github/workflows/ghcr_helm_deploy.yml deleted file mode 100644 index 21b2eaafe19..00000000000 --- a/.github/workflows/ghcr_helm_deploy.yml +++ /dev/null @@ -1,67 +0,0 @@ -# Standalone workflow to publish LiteLLM Helm Chart -# Note: The main ghcr_deploy.yml workflow also publishes the Helm chart as part of a full release -name: Build, Publish LiteLLM Helm Chart. New Release -on: - workflow_dispatch: - inputs: - tag: - description: "LiteLLM version tag (e.g., v1.81.0)" - required: true - -# Defines two custom environment variables for the workflow. Used for the Container registry domain, and a name for the Docker image that this workflow builds. -env: - REGISTRY: ghcr.io - IMAGE_NAME: ${{ github.repository }} - REPO_OWNER: ${{github.repository_owner}} - -# There is a single job in this workflow. It's configured to run on the latest available version of Ubuntu. -jobs: - build-and-push-helm-chart: - runs-on: ubuntu-latest - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - - name: Log in to the Container registry - uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1 - with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: lowercase github.repository_owner - run: | - echo "REPO_OWNER=`echo ${{github.repository_owner}} | tr '[:upper:]' '[:lower:]'`" >>${GITHUB_ENV} - - # Sync Helm chart version with LiteLLM release version (1-1 versioning) - - name: Calculate chart and app versions - id: chart_version - shell: bash - run: | - INPUT_TAG="${{ github.event.inputs.tag }}" - - # Chart version = LiteLLM version without 'v' prefix - # v1.81.0 -> 1.81.0 - CHART_VERSION="${INPUT_TAG#v}" - - # App version = Docker tag (keeps 'v' prefix) - APP_VERSION="${INPUT_TAG}" - - echo "version=${CHART_VERSION}" | tee -a $GITHUB_OUTPUT - echo "app_version=${APP_VERSION}" | tee -a $GITHUB_OUTPUT - - - name: Lint helm chart - run: helm lint deploy/charts/litellm-helm - - - uses: ./.github/actions/helm-oci-chart-releaser - with: - name: litellm-helm - repository: ${{ env.REPO_OWNER }} - tag: ${{ steps.chart_version.outputs.version }} - app_version: ${{ steps.chart_version.outputs.app_version }} - path: deploy/charts/litellm-helm - registry: ${{ env.REGISTRY }} - registry_username: ${{ github.actor }} - registry_password: ${{ secrets.GITHUB_TOKEN }} - update_dependencies: true - diff --git a/.github/workflows/helm_unit_test.yml b/.github/workflows/helm_unit_test.yml index c4b83af70a1..06836b1d1cd 100644 --- a/.github/workflows/helm_unit_test.yml +++ b/.github/workflows/helm_unit_test.yml @@ -6,22 +6,36 @@ on: branches: - main +permissions: + contents: read + jobs: unit-test: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + with: + persist-credentials: false - name: Set up Helm 3.11.1 - uses: azure/setup-helm@v1 + uses: azure/setup-helm@1a275c3b69536ee54be43f2070a358922e12c8d4 # v4.3.1 with: - version: '3.11.1' + version: "3.11.1" - name: Install Helm Unit Test Plugin run: | helm plugin install https://github.com/helm-unittest/helm-unittest --version v0.4.4 + - name: Verify Helm Unit Test Plugin integrity + run: | + EXPECTED_SHA="e251ba198448629678ff2168e1a469249d998155" + PLUGIN_DIR="$(helm env HELM_PLUGINS)/helm-unittest" + ACTUAL_SHA="$(git -C "$PLUGIN_DIR" rev-parse HEAD)" + if [ "$ACTUAL_SHA" != "$EXPECTED_SHA" ]; then + echo "::error::Helm unittest plugin checksum mismatch! Expected $EXPECTED_SHA but got $ACTUAL_SHA" + exit 1 + fi + echo "Helm unittest plugin integrity verified: $ACTUAL_SHA" - name: Run unit tests - run: - helm unittest -f 'tests/*.yaml' deploy/charts/litellm-helm \ No newline at end of file + run: helm unittest -f 'tests/*.yaml' deploy/charts/litellm-helm diff --git a/.github/workflows/interpret_load_test.py b/.github/workflows/interpret_load_test.py deleted file mode 100644 index 348ff300fff..00000000000 --- a/.github/workflows/interpret_load_test.py +++ /dev/null @@ -1,139 +0,0 @@ -import csv -import os -from github import Github - - -def interpret_results(csv_file): - with open(csv_file, newline="") as csvfile: - csvreader = csv.DictReader(csvfile) - rows = list(csvreader) - """ - in this csv reader - - Create 1 new column "Status" - - if a row has a median response time < 300 and an average response time < 300, Status = "Passed ✅" - - if a row has a median response time >= 300 or an average response time >= 300, Status = "Failed ❌" - - Order the table in this order Name, Status, Median Response Time, Average Response Time, Requests/s,Failures/s, Min Response Time, Max Response Time, all other columns - """ - - # Add a new column "Status" - for row in rows: - median_response_time = float( - row["Median Response Time"].strip().rstrip("ms") - ) - average_response_time = float( - row["Average Response Time"].strip().rstrip("s") - ) - - request_count = int(row["Request Count"]) - failure_count = int(row["Failure Count"]) - - failure_percent = round((failure_count / request_count) * 100, 2) - - # Determine status based on conditions - if ( - median_response_time < 300 - and average_response_time < 300 - and failure_percent < 5 - ): - row["Status"] = "Passed ✅" - else: - row["Status"] = "Failed ❌" - - # Construct Markdown table header - markdown_table = "| Name | Status | Median Response Time (ms) | Average Response Time (ms) | Requests/s | Failures/s | Request Count | Failure Count | Min Response Time (ms) | Max Response Time (ms) |" - markdown_table += ( - "\n| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |" - ) - - # Construct Markdown table rows - for row in rows: - markdown_table += f"\n| {row['Name']} | {row['Status']} | {row['Median Response Time']} | {row['Average Response Time']} | {row['Requests/s']} | {row['Failures/s']} | {row['Request Count']} | {row['Failure Count']} | {row['Min Response Time']} | {row['Max Response Time']} |" - print("markdown table: ", markdown_table) - return markdown_table - - -def _get_docker_run_command_stable_release(release_version): - return f""" -\n\n -## Docker Run LiteLLM Proxy - -``` -docker run \\ --e STORE_MODEL_IN_DB=True \\ --p 4000:4000 \\ -ghcr.io/berriai/litellm:litellm_stable_release_branch-{release_version} -``` - """ - - -def _get_docker_run_command(release_version): - return f""" -\n\n -## Docker Run LiteLLM Proxy - -``` -docker run \\ --e STORE_MODEL_IN_DB=True \\ --p 4000:4000 \\ -ghcr.io/berriai/litellm:main-{release_version} -``` - """ - - -def get_docker_run_command(release_version): - if "stable" in release_version: - return _get_docker_run_command_stable_release(release_version) - else: - return _get_docker_run_command(release_version) - - -if __name__ == "__main__": - return - csv_file = "load_test_stats.csv" # Change this to the path of your CSV file - markdown_table = interpret_results(csv_file) - - # Update release body with interpreted results - github_token = os.getenv("GITHUB_TOKEN") - g = Github(github_token) - repo = g.get_repo( - "BerriAI/litellm" - ) # Replace with your repository's username and name - latest_release = repo.get_latest_release() - print("got latest release: ", latest_release) - print(latest_release.title) - print(latest_release.tag_name) - - release_version = latest_release.title - - print("latest release body: ", latest_release.body) - print("markdown table: ", markdown_table) - - # check if "Load Test LiteLLM Proxy Results" exists - existing_release_body = latest_release.body - if "Load Test LiteLLM Proxy Results" in latest_release.body: - # find the "Load Test LiteLLM Proxy Results" section and delete it - start_index = latest_release.body.find("Load Test LiteLLM Proxy Results") - existing_release_body = latest_release.body[:start_index] - - docker_run_command = get_docker_run_command(release_version) - print("docker run command: ", docker_run_command) - - new_release_body = ( - existing_release_body - + docker_run_command - + "\n\n" - + "### Don't want to maintain your internal proxy? get in touch 🎉" - + "\nHosted Proxy Alpha: https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions" - + "\n\n" - + "## Load Test LiteLLM Proxy Results" - + "\n\n" - + markdown_table - ) - print("new release body: ", new_release_body) - try: - latest_release.update_release( - name=latest_release.tag_name, - message=new_release_body, - ) - except Exception as e: - print(e) diff --git a/.github/workflows/issue-keyword-labeler.yml b/.github/workflows/issue-keyword-labeler.yml index 936f90f747f..7e2693209b6 100644 --- a/.github/workflows/issue-keyword-labeler.yml +++ b/.github/workflows/issue-keyword-labeler.yml @@ -2,8 +2,8 @@ name: Issue Keyword Labeler on: issues: - types: - - opened + types: + - opened jobs: scan-and-label: @@ -13,7 +13,9 @@ jobs: contents: read steps: - name: Checkout code - uses: actions/checkout@v4 + uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + with: + persist-credentials: false - name: Scan for provider keywords id: scan @@ -24,7 +26,7 @@ jobs: - name: Ensure label exists if: steps.scan.outputs.found == 'true' - uses: actions/github-script@v7 + uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7.1.0 with: github-token: ${{ secrets.GITHUB_TOKEN }} script: | @@ -51,7 +53,7 @@ jobs: - name: Add label to the issue if: steps.scan.outputs.found == 'true' - uses: actions/github-script@v7 + uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7.1.0 with: github-token: ${{ secrets.GITHUB_TOKEN }} script: | @@ -61,4 +63,3 @@ jobs: issue_number: context.issue.number, labels: ['llm translation'] }); - diff --git a/.github/workflows/label-component.yml b/.github/workflows/label-component.yml index fd079fce6c1..e0c2fa94d8c 100644 --- a/.github/workflows/label-component.yml +++ b/.github/workflows/label-component.yml @@ -12,7 +12,7 @@ jobs: issues: write steps: - name: Add component labels - uses: actions/github-script@v7 + uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7.1.0 with: github-token: ${{ secrets.GITHUB_TOKEN }} script: | diff --git a/.github/workflows/llm-translation-testing.yml b/.github/workflows/llm-translation-testing.yml index 7fda37a66dc..922013c4b54 100644 --- a/.github/workflows/llm-translation-testing.yml +++ b/.github/workflows/llm-translation-testing.yml @@ -4,38 +4,41 @@ on: workflow_dispatch: inputs: release_candidate_tag: - description: 'Release candidate tag/version' + description: "Release candidate tag/version" required: true type: string push: tags: - - 'v*-rc*' # Triggers on release candidate tags like v1.0.0-rc1 - + - "v*-rc*" # Triggers on release candidate tags like v1.0.0-rc1 + +permissions: + contents: read + jobs: run-llm-translation-tests: runs-on: ubuntu-latest timeout-minutes: 90 - + steps: - name: Checkout code - uses: actions/checkout@v4 + uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 with: + persist-credentials: false ref: ${{ github.event.inputs.release_candidate_tag || github.ref }} - + - name: Set up Python - uses: actions/setup-python@v5 + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: - python-version: '3.11' - + python-version: "3.11" + - name: Install Poetry - uses: snok/install-poetry@v1 - with: - version: latest - virtualenvs-create: true - virtualenvs-in-project: true - - - name: Cache Poetry dependencies - uses: actions/cache@v3 + run: | + pip install 'poetry==2.3.2' + poetry config virtualenvs.create true + poetry config virtualenvs.in-project true + + - name: Restore Poetry dependencies cache + uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.0.0 with: path: | ~/.cache/pypoetry @@ -43,15 +46,15 @@ jobs: key: ${{ runner.os }}-poetry-${{ hashFiles('**/poetry.lock') }} restore-keys: | ${{ runner.os }}-poetry- - + - name: Install dependencies run: | poetry install --with dev - poetry run pip install pytest-xdist pytest-timeout - + poetry run pip install 'pytest-xdist==3.8.0' 'pytest-timeout==2.4.0' + - name: Create test results directory run: mkdir -p test-results - + - name: Run LLM Translation Tests env: OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} @@ -61,13 +64,14 @@ jobs: AZURE_API_KEY: ${{ secrets.AZURE_API_KEY }} AZURE_API_BASE: ${{ secrets.AZURE_API_BASE }} AZURE_API_VERSION: ${{ secrets.AZURE_API_VERSION }} - # Add other API keys as needed + RC_TAG: ${{ github.event.inputs.release_candidate_tag || github.ref_name }} + COMMIT_SHA: ${{ github.sha }} run: | python .github/workflows/run_llm_translation_tests.py \ - --tag "${{ github.event.inputs.release_candidate_tag || github.ref_name }}" \ - --commit "${{ github.sha }}" \ + --tag "$RC_TAG" \ + --commit "$COMMIT_SHA" \ || true # Continue even if tests fail - + - name: Display test summary if: always() run: | @@ -79,9 +83,9 @@ jobs: else echo "Warning: Test report was not generated" fi - + - name: Upload test artifacts - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 if: always() with: name: LLM-Translation-Artifact-${{ github.event.inputs.release_candidate_tag || github.ref_name }} diff --git a/.github/workflows/load_test.yml b/.github/workflows/load_test.yml deleted file mode 100644 index cdaffa328c9..00000000000 --- a/.github/workflows/load_test.yml +++ /dev/null @@ -1,59 +0,0 @@ -name: Test Locust Load Test - -on: - workflow_run: - workflows: ["Build, Publish LiteLLM Docker Image. New Release"] - types: - - completed - workflow_dispatch: - -jobs: - build: - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v1 - - name: Setup Python - uses: actions/setup-python@v2 - with: - python-version: '3.x' - - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install PyGithub - - name: re-deploy proxy - run: | - echo "Current working directory: $PWD" - ls - python ".github/workflows/redeploy_proxy.py" - env: - LOAD_TEST_REDEPLOY_URL1: ${{ secrets.LOAD_TEST_REDEPLOY_URL1 }} - LOAD_TEST_REDEPLOY_URL2: ${{ secrets.LOAD_TEST_REDEPLOY_URL2 }} - working-directory: ${{ github.workspace }} - - name: Run Load Test - id: locust_run - uses: BerriAI/locust-github-action@master - with: - LOCUSTFILE: ".github/workflows/locustfile.py" - URL: "https://post-release-load-test-proxy.onrender.com/" - USERS: "20" - RATE: "20" - RUNTIME: "300s" - - name: Process Load Test Stats - run: | - echo "Current working directory: $PWD" - ls - python ".github/workflows/interpret_load_test.py" - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - working-directory: ${{ github.workspace }} - - name: Upload CSV as Asset to Latest Release - uses: xresloader/upload-to-github-release@v1 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - file: "load_test_stats.csv;load_test.html" - update_latest_release: true - tag_name: "load-test" - overwrite: true \ No newline at end of file diff --git a/.github/workflows/locustfile.py b/.github/workflows/locustfile.py deleted file mode 100644 index 36dbeee9c48..00000000000 --- a/.github/workflows/locustfile.py +++ /dev/null @@ -1,28 +0,0 @@ -from locust import HttpUser, task, between - - -class MyUser(HttpUser): - wait_time = between(1, 5) - - @task - def chat_completion(self): - headers = { - "Content-Type": "application/json", - "Authorization": "Bearer sk-8N1tLOOyH8TIxwOLahhIVg", - # Include any additional headers you may need for authentication, etc. - } - - # Customize the payload with "model" and "messages" keys - payload = { - "model": "fake-openai-endpoint", - "messages": [ - {"role": "system", "content": "You are a chat bot."}, - {"role": "user", "content": "Hello, how are you?"}, - ], - # Add more data as necessary - } - - # Make a POST request to the "chat/completions" endpoint - response = self.client.post("chat/completions", json=payload, headers=headers) - - # Print or log the response if needed diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml deleted file mode 100644 index 23e4a06da9e..00000000000 --- a/.github/workflows/main.yml +++ /dev/null @@ -1,34 +0,0 @@ -name: Publish Dev Release to PyPI - -on: - workflow_dispatch: - -jobs: - publish-dev-release: - runs-on: ubuntu-latest - - steps: - - name: Checkout code - uses: actions/checkout@v2 - - - name: Set up Python - uses: actions/setup-python@v2 - with: - python-version: 3.8 # Adjust the Python version as needed - - - name: Install dependencies - run: pip install toml twine - - - name: Read version from pyproject.toml - id: read-version - run: | - version=$(python -c 'import toml; print(toml.load("pyproject.toml")["tool"]["commitizen"]["version"])') - printf "LITELLM_VERSION=%s" "$version" >> $GITHUB_ENV - - - name: Check if version exists on PyPI - id: check-version - run: | - set -e - if twine check --repository-url https://pypi.org/simple/ "litellm==$LITELLM_VERSION" >/dev/null 2>&1; then - echo "Version $LITELLM_VERSION already exists on PyPI. Skipping publish." - diff --git a/.github/workflows/publish-migrations.yml b/.github/workflows/publish-migrations.yml deleted file mode 100644 index a5187cb2f55..00000000000 --- a/.github/workflows/publish-migrations.yml +++ /dev/null @@ -1,207 +0,0 @@ -name: Publish Prisma Migrations - -permissions: - contents: write - pull-requests: write - -on: - push: - paths: - - 'schema.prisma' # Check root schema.prisma - branches: - - main - -jobs: - publish-migrations: - if: github.repository == 'BerriAI/litellm' - runs-on: ubuntu-latest - services: - postgres: - image: postgres:14 - env: - POSTGRES_DB: temp_db - POSTGRES_USER: postgres - POSTGRES_PASSWORD: postgres - ports: - - 5432:5432 - options: >- - --health-cmd pg_isready - --health-interval 10s - --health-timeout 5s - --health-retries 5 - - # Add shadow database service - postgres_shadow: - image: postgres:14 - env: - POSTGRES_DB: shadow_db - POSTGRES_USER: postgres - POSTGRES_PASSWORD: postgres - ports: - - 5433:5432 - options: >- - --health-cmd pg_isready - --health-interval 10s - --health-timeout 5s - --health-retries 5 - - steps: - - uses: actions/checkout@v3 - - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: '3.x' - - - name: Install Dependencies - run: | - pip install prisma - pip install python-dotenv - - - name: Generate Initial Migration if None Exists - env: - DATABASE_URL: "postgresql://postgres:postgres@localhost:5432/temp_db" - DIRECT_URL: "postgresql://postgres:postgres@localhost:5432/temp_db" - SHADOW_DATABASE_URL: "postgresql://postgres:postgres@localhost:5433/shadow_db" - run: | - mkdir -p deploy/migrations - echo 'provider = "postgresql"' > deploy/migrations/migration_lock.toml - - if [ -z "$(ls -A deploy/migrations/2* 2>/dev/null)" ]; then - echo "No existing migrations found, creating baseline..." - VERSION=$(date +%Y%m%d%H%M%S) - mkdir -p deploy/migrations/${VERSION}_initial - - echo "Generating initial migration..." - # Save raw output for debugging - prisma migrate diff \ - --from-empty \ - --to-schema-datamodel schema.prisma \ - --shadow-database-url "${SHADOW_DATABASE_URL}" \ - --script > deploy/migrations/${VERSION}_initial/raw_migration.sql - - echo "Raw migration file content:" - cat deploy/migrations/${VERSION}_initial/raw_migration.sql - - echo "Cleaning migration file..." - # Clean the file - sed '/^Installing/d' deploy/migrations/${VERSION}_initial/raw_migration.sql > deploy/migrations/${VERSION}_initial/migration.sql - - # Verify the migration file - if [ ! -s deploy/migrations/${VERSION}_initial/migration.sql ]; then - echo "ERROR: Migration file is empty after cleaning" - echo "Original content was:" - cat deploy/migrations/${VERSION}_initial/raw_migration.sql - exit 1 - fi - - echo "Final migration file content:" - cat deploy/migrations/${VERSION}_initial/migration.sql - - # Verify it starts with SQL - if ! head -n 1 deploy/migrations/${VERSION}_initial/migration.sql | grep -q "^--\|^CREATE\|^ALTER"; then - echo "ERROR: Migration file does not start with SQL command or comment" - echo "First line is:" - head -n 1 deploy/migrations/${VERSION}_initial/migration.sql - echo "Full content is:" - cat deploy/migrations/${VERSION}_initial/migration.sql - exit 1 - fi - - echo "Initial migration generated at $(date -u)" > deploy/migrations/${VERSION}_initial/README.md - fi - - - name: Compare and Generate Migration - if: success() - env: - DATABASE_URL: "postgresql://postgres:postgres@localhost:5432/temp_db" - DIRECT_URL: "postgresql://postgres:postgres@localhost:5432/temp_db" - SHADOW_DATABASE_URL: "postgresql://postgres:postgres@localhost:5433/shadow_db" - run: | - # Create temporary migration workspace - mkdir -p temp_migrations - - # Copy existing migrations (will not fail if directory is empty) - cp -r deploy/migrations/* temp_migrations/ 2>/dev/null || true - - VERSION=$(date +%Y%m%d%H%M%S) - - # Generate diff against existing migrations or empty state - prisma migrate diff \ - --from-migrations temp_migrations \ - --to-schema-datamodel schema.prisma \ - --shadow-database-url "${SHADOW_DATABASE_URL}" \ - --script > temp_migrations/migration_${VERSION}.sql - - # Check if there are actual changes - if [ -s temp_migrations/migration_${VERSION}.sql ]; then - echo "Changes detected, creating new migration" - mkdir -p deploy/migrations/${VERSION}_schema_update - mv temp_migrations/migration_${VERSION}.sql deploy/migrations/${VERSION}_schema_update/migration.sql - echo "Migration generated at $(date -u)" > deploy/migrations/${VERSION}_schema_update/README.md - else - echo "No schema changes detected" - exit 0 - fi - - - name: Verify Migration - if: success() - env: - DATABASE_URL: "postgresql://postgres:postgres@localhost:5432/temp_db" - DIRECT_URL: "postgresql://postgres:postgres@localhost:5432/temp_db" - SHADOW_DATABASE_URL: "postgresql://postgres:postgres@localhost:5433/shadow_db" - run: | - # Create test database - psql "${SHADOW_DATABASE_URL}" -c 'CREATE DATABASE migration_test;' - - # Apply all migrations in order to verify - for migration in deploy/migrations/*/migration.sql; do - echo "Applying migration: $migration" - psql "${SHADOW_DATABASE_URL}" -f $migration - done - - # Add this step before create-pull-request to debug permissions - - name: Check Token Permissions - run: | - echo "Checking token permissions..." - curl -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \ - -H "Accept: application/vnd.github.v3+json" \ - https://api.github.com/repos/BerriAI/litellm/collaborators - - echo "\nChecking if token can create PRs..." - curl -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \ - -H "Accept: application/vnd.github.v3+json" \ - https://api.github.com/repos/BerriAI/litellm - - # Add this debug step before git push - - name: Debug Changed Files - run: | - echo "Files staged for commit:" - git diff --name-status --staged - - echo "\nAll changed files:" - git status - - - name: Create Pull Request - if: success() - uses: peter-evans/create-pull-request@v5 - with: - token: ${{ secrets.GITHUB_TOKEN }} - commit-message: "chore: update prisma migrations" - title: "Update Prisma Migrations" - body: | - Auto-generated migration based on schema.prisma changes. - - Generated files: - - deploy/migrations/${VERSION}_schema_update/migration.sql - - deploy/migrations/${VERSION}_schema_update/README.md - branch: feat/prisma-migration-${{ env.VERSION }} - base: main - delete-branch: true - - - name: Generate and Save Migrations - run: | - # Only add migration files - git add deploy/migrations/ - git status # Debug what's being committed - git commit -m "chore: update prisma migrations" diff --git a/.github/workflows/publish_enterprise.yml b/.github/workflows/publish_enterprise.yml deleted file mode 100644 index 459a233cb71..00000000000 --- a/.github/workflows/publish_enterprise.yml +++ /dev/null @@ -1,94 +0,0 @@ -name: Publish litellm-enterprise to PyPI - -on: - workflow_dispatch: - inputs: - bump: - description: "Version bump type" - required: true - default: "patch" - type: choice - options: - - patch - - minor - - major - -jobs: - publish: - runs-on: ubuntu-latest - if: github.repository == 'BerriAI/litellm' - permissions: - contents: write - pull-requests: write - defaults: - run: - working-directory: enterprise - - steps: - - uses: actions/checkout@v4 - - - uses: actions/setup-python@v5 - with: - python-version: "3.11" - - - name: Install Poetry - run: pip install poetry - - - name: Bump version - id: bump - run: | - OLD=$(poetry version -s) - poetry version ${{ github.event.inputs.bump }} - NEW=$(poetry version -s) - echo "old=$OLD" >> $GITHUB_OUTPUT - echo "new=$NEW" >> $GITHUB_OUTPUT - - - name: Update version refs in root pyproject.toml and requirements.txt - run: | - OLD=${{ steps.bump.outputs.old }} - NEW=${{ steps.bump.outputs.new }} - sed -i "s/litellm-enterprise = {version = \"${OLD}\"/litellm-enterprise = {version = \"${NEW}\"/" ../pyproject.toml - sed -i "s/litellm-enterprise==${OLD}/litellm-enterprise==${NEW}/" ../requirements.txt - - - name: Update poetry.lock - working-directory: . - run: poetry lock - - - name: Build - run: poetry build - - - name: Commit version bump and create PR - id: create-pr - run: | - git config user.name "github-actions[bot]" - git config user.email "github-actions[bot]@users.noreply.github.com" - cd .. - BRANCH="bump/enterprise-${{ steps.bump.outputs.new }}" - git checkout -b "$BRANCH" - git add enterprise/pyproject.toml pyproject.toml requirements.txt poetry.lock - git commit -m "bump: litellm-enterprise ${{ steps.bump.outputs.old }} → ${{ steps.bump.outputs.new }}" - git push origin "$BRANCH" --force - gh pr create \ - --title "bump: litellm-enterprise ${{ steps.bump.outputs.old }} → ${{ steps.bump.outputs.new }}" \ - --body "Version bump for litellm-enterprise. Merge to update main." \ - --head "$BRANCH" \ - --base main \ - || true - PR_URL=$(gh pr list --head "$BRANCH" --json url -q '.[0].url') - echo "pr_url=$PR_URL" >> $GITHUB_OUTPUT - env: - GH_TOKEN: ${{ github.token }} - - - name: Enable auto-merge - run: | - gh pr merge "${{ steps.create-pr.outputs.pr_url }}" --auto --squash - env: - GH_TOKEN: ${{ github.token }} - - - name: Publish to PyPI - env: - TWINE_USERNAME: __token__ - TWINE_PASSWORD: ${{ secrets.PYPI_ENTERPRISE }} - run: | - pip install twine - twine upload dist/litellm_enterprise-${{ steps.bump.outputs.new }}* diff --git a/.github/workflows/publish_proxy_extras.yml b/.github/workflows/publish_proxy_extras.yml deleted file mode 100644 index fa30b153163..00000000000 --- a/.github/workflows/publish_proxy_extras.yml +++ /dev/null @@ -1,74 +0,0 @@ -name: Publish litellm-proxy-extras to PyPI - -on: - workflow_dispatch: - inputs: - bump: - description: "Version bump type" - required: true - default: "patch" - type: choice - options: - - patch - - minor - - major - -jobs: - publish: - runs-on: ubuntu-latest - if: github.repository == 'BerriAI/litellm' - permissions: - contents: write - defaults: - run: - working-directory: litellm-proxy-extras - - steps: - - uses: actions/checkout@v4 - - - uses: actions/setup-python@v5 - with: - python-version: "3.11" - - - name: Install Poetry - run: pip install poetry - - - name: Bump version - id: bump - run: | - OLD=$(poetry version -s) - poetry version ${{ github.event.inputs.bump }} - NEW=$(poetry version -s) - echo "old=$OLD" >> $GITHUB_OUTPUT - echo "new=$NEW" >> $GITHUB_OUTPUT - - - name: Update version refs in root pyproject.toml and requirements.txt - run: | - OLD=${{ steps.bump.outputs.old }} - NEW=${{ steps.bump.outputs.new }} - sed -i "s/litellm-proxy-extras = {version = \"${OLD}\"/litellm-proxy-extras = {version = \"${NEW}\"/" ../pyproject.toml - sed -i "s/litellm-proxy-extras==${OLD}/litellm-proxy-extras==${NEW}/" ../requirements.txt - - - name: Update poetry.lock - working-directory: . - run: poetry lock - - - name: Build - run: poetry build - - - name: Commit version bump - run: | - git config user.name "github-actions[bot]" - git config user.email "github-actions[bot]@users.noreply.github.com" - cd .. - git add litellm-proxy-extras/pyproject.toml pyproject.toml requirements.txt poetry.lock - git commit -m "bump: litellm-proxy-extras ${{ steps.bump.outputs.old }} → ${{ steps.bump.outputs.new }}" - git push - - - name: Publish to PyPI - env: - TWINE_USERNAME: __token__ - TWINE_PASSWORD: ${{ secrets.PYPI_PUBLISH_PASSWORD }} - run: | - pip install twine - twine upload dist/litellm_proxy_extras-${{ steps.bump.outputs.new }}* diff --git a/.github/workflows/publish_to_pypi.yml b/.github/workflows/publish_to_pypi.yml new file mode 100644 index 00000000000..8f675bb3075 --- /dev/null +++ b/.github/workflows/publish_to_pypi.yml @@ -0,0 +1,136 @@ +name: Publish to PyPI + +on: + workflow_dispatch: + +jobs: + preflight-checks: + name: Preflight Checks + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + contents: read + # No environment — read-only checks, no approval needed + outputs: + needs_publish: ${{ steps.check-litellm.outputs.needs_publish }} + version: ${{ steps.check-litellm.outputs.version }} + + steps: + - name: Checkout repo + uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + + - name: Set up Python + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: "3.12" + + - name: Check litellm version on PyPI + id: check-litellm + run: | + VERSION=$(grep -m1 '^version' pyproject.toml | sed 's/version = "\(.*\)"/\1/') + echo "version=$VERSION" >> "$GITHUB_OUTPUT" + echo "Checking if litellm $VERSION exists on PyPI..." + + HTTP_STATUS=$(curl -s -o /dev/null -w "%{http_code}" "https://pypi.org/pypi/litellm/$VERSION/json") + if [ "$HTTP_STATUS" = "200" ]; then + echo "litellm $VERSION already exists on PyPI. Skipping publish." + echo "needs_publish=false" >> "$GITHUB_OUTPUT" + else + echo "litellm $VERSION not found on PyPI. Publish needed." + echo "needs_publish=true" >> "$GITHUB_OUTPUT" + fi + + - name: Sanity check proxy-extras version + run: | + # Read pinned version from requirements.txt + REQ_VERSION=$(grep -oP 'litellm-proxy-extras==\K[0-9.]+' requirements.txt) + if [ -z "$REQ_VERSION" ]; then + echo "::error::Could not find litellm-proxy-extras version in requirements.txt" + exit 1 + fi + echo "requirements.txt pins litellm-proxy-extras==$REQ_VERSION" + + # Read pinned version from pyproject.toml dependency + PYPROJECT_VERSION=$(python3 -c " + import re + with open('pyproject.toml') as f: + content = f.read() + match = re.search(r'litellm-proxy-extras\s*=\s*\{version\s*=\s*\"([^\"]+)\"', content) + if match: + print(match.group(1).lstrip('^~>=')) + else: + import sys + print('::error::Could not find litellm-proxy-extras dependency in pyproject.toml', file=sys.stderr) + sys.exit(1) + ") + echo "pyproject.toml pins litellm-proxy-extras version: $PYPROJECT_VERSION" + + # Check that both pinned versions match + if [ "$REQ_VERSION" != "$PYPROJECT_VERSION" ]; then + echo "::error::Version mismatch: requirements.txt has $REQ_VERSION but pyproject.toml has $PYPROJECT_VERSION" + exit 1 + fi + + # Check that the pinned version exists on PyPI + echo "Checking if litellm-proxy-extras $REQ_VERSION exists on PyPI..." + HTTP_STATUS=$(curl -s -o /dev/null -w "%{http_code}" "https://pypi.org/pypi/litellm-proxy-extras/$REQ_VERSION/json") + if [ "$HTTP_STATUS" != "200" ]; then + echo "::error::litellm-proxy-extras $REQ_VERSION is not published on PyPI yet. Publish it before releasing litellm." + exit 1 + fi + echo "litellm-proxy-extras $REQ_VERSION exists on PyPI. Sanity check passed." + + publish-litellm: + name: Publish litellm to PyPI + needs: preflight-checks + if: needs.preflight-checks.outputs.needs_publish == 'true' + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + id-token: write + contents: read + environment: pypi-publish + + steps: + - name: Checkout repo + uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + + - name: Set up Python + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: "3.12" + + - name: Copy model prices backup + run: cp model_prices_and_context_window.json litellm/model_prices_and_context_window_backup.json + + - name: Install build tools + run: python -m pip install --upgrade pip build==1.4.2 + + - name: Build package + run: | + rm -rf build dist + python -m build + + - name: Verify build artifacts + env: + EXPECTED_VERSION: ${{ needs.preflight-checks.outputs.version }} + run: | + echo "Contents of dist/:" + ls -la dist/ + # Ensure we have both sdist and wheel + ls dist/*.tar.gz + ls dist/*.whl + # Verify built version matches expected + ls dist/ | grep -q "litellm-${EXPECTED_VERSION}" || { + echo "::error::Built artifacts do not match expected version $EXPECTED_VERSION" + ls dist/ + exit 1 + } + + - name: Validate package metadata + run: | + pip install twine==6.2.0 + twine check dist/* + + - name: Publish to PyPI + uses: pypa/gh-action-pypi-publish@ed0c53931b1dc9bd32cbe73a98c7f6766f8a527e # v1.13.0 diff --git a/.github/workflows/read_pyproject_version.yml b/.github/workflows/read_pyproject_version.yml index 8f6310f935b..04b4a38ce19 100644 --- a/.github/workflows/read_pyproject_version.yml +++ b/.github/workflows/read_pyproject_version.yml @@ -3,7 +3,10 @@ name: Read Version from pyproject.toml on: push: branches: - - main # Change this to the default branch of your repository + - main # Change this to the default branch of your repository + +permissions: + contents: read jobs: read-version: @@ -11,20 +14,14 @@ jobs: steps: - name: Checkout code - uses: actions/checkout@v2 - - - name: Set up Python - uses: actions/setup-python@v2 + uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 with: - python-version: 3.8 # Adjust the Python version as needed - - - name: Install dependencies - run: pip install toml + persist-credentials: false - name: Read version from pyproject.toml id: read-version run: | - version=$(python -c 'import toml; print(toml.load("pyproject.toml")["tool"]["commitizen"]["version"])') + version=$(grep -m1 '^version' pyproject.toml | sed 's/version = "\(.*\)"/\1/') printf "LITELLM_VERSION=%s" "$version" >> $GITHUB_ENV - name: Display version diff --git a/.github/workflows/redeploy_proxy.py b/.github/workflows/redeploy_proxy.py deleted file mode 100644 index ed46bef73a2..00000000000 --- a/.github/workflows/redeploy_proxy.py +++ /dev/null @@ -1,20 +0,0 @@ -""" - -redeploy_proxy.py -""" - -import os -import requests -import time - -# send a get request to this endpoint -deploy_hook1 = os.getenv("LOAD_TEST_REDEPLOY_URL1") -response = requests.get(deploy_hook1, timeout=20) - - -deploy_hook2 = os.getenv("LOAD_TEST_REDEPLOY_URL2") -response = requests.get(deploy_hook2, timeout=20) - -print("SENT GET REQUESTS to re-deploy proxy") -print("sleeeping.... for 60s") -time.sleep(60) diff --git a/.github/workflows/regenerate-poetry-lock.yml b/.github/workflows/regenerate-poetry-lock.yml deleted file mode 100644 index c0844f1c705..00000000000 --- a/.github/workflows/regenerate-poetry-lock.yml +++ /dev/null @@ -1,80 +0,0 @@ -name: Regenerate poetry.lock - -# Runs whenever pyproject.toml is merged into main (the most common cause of -# the "pyproject.toml changed significantly since poetry.lock was last generated" -# CI failure). Can also be triggered manually. -on: - push: - branches: - - main - paths: - - pyproject.toml - workflow_dispatch: - -permissions: - contents: write # needed to push the auto/regenerate-poetry-lock-* branch - pull-requests: write # needed to open the PR and enable auto-merge - -jobs: - regenerate-lock: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: "3.11" - - - name: Install Poetry - run: pip install poetry - - - name: Regenerate poetry.lock - run: poetry lock - - - name: Check whether poetry.lock actually changed - id: diff - run: | - if git diff --quiet poetry.lock; then - echo "changed=false" >> "$GITHUB_OUTPUT" - else - echo "changed=true" >> "$GITHUB_OUTPUT" - fi - - - name: Open PR with the refreshed lock file - if: steps.diff.outputs.changed == 'true' - id: open-pr - run: | - BRANCH="auto/regenerate-poetry-lock-$(date +'%Y%m%d%H%M%S')" - git config user.name "github-actions[bot]" - git config user.email "github-actions[bot]@users.noreply.github.com" - git checkout -b "$BRANCH" - git add poetry.lock - git commit -m "chore: regenerate poetry.lock to match pyproject.toml" - git push -f origin "$BRANCH" - - cat > /tmp/pr-body.md << 'BODY' - Automated regeneration of `poetry.lock` after `pyproject.toml` was updated on `main`. - - Fixes the recurring CI failure: - ``` - pyproject.toml changed significantly since poetry.lock was last generated. - Run `poetry lock` to fix the lock file. - ``` - BODY - - PR_URL=$(gh pr create \ - --title "chore: regenerate poetry.lock to match pyproject.toml" \ - --body-file /tmp/pr-body.md \ - --head "$BRANCH" \ - --base main) - echo "pr_url=$PR_URL" >> "$GITHUB_OUTPUT" - env: - GH_TOKEN: ${{ github.token }} - - - name: Enable auto-merge - if: steps.diff.outputs.changed == 'true' - run: | - gh pr merge "${{ steps.open-pr.outputs.pr_url }}" --auto --squash - env: - GH_TOKEN: ${{ github.token }} diff --git a/.github/workflows/reset_stable.yml b/.github/workflows/reset_stable.yml deleted file mode 100644 index f6fed672d47..00000000000 --- a/.github/workflows/reset_stable.yml +++ /dev/null @@ -1,39 +0,0 @@ -name: Reset litellm_stable branch - -on: - release: - types: [published, created] -jobs: - update-stable-branch: - if: ${{ startsWith(github.event.release.tag_name, 'v') && !endsWith(github.event.release.tag_name, '-stable') }} - runs-on: ubuntu-latest - - steps: - - name: Checkout repository - uses: actions/checkout@v3 - - - name: Reset litellm_stable_release_branch branch to the release commit - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - # Configure Git user - git config user.name "github-actions[bot]" - git config user.email "github-actions[bot]@users.noreply.github.com" - - # Fetch all branches and tags - git fetch --all - - # Check if the litellm_stable_release_branch branch exists - if git show-ref --verify --quiet refs/remotes/origin/litellm_stable_release_branch; then - echo "litellm_stable_release_branch branch exists." - git checkout litellm_stable_release_branch - else - echo "litellm_stable_release_branch branch does not exist. Creating it." - git checkout -b litellm_stable_release_branch - fi - - # Reset litellm_stable_release_branch branch to the release commit - git reset --hard $GITHUB_SHA - - # Push the updated litellm_stable_release_branch branch - git push origin litellm_stable_release_branch --force diff --git a/.github/workflows/run_observatory_tests.yml b/.github/workflows/run_observatory_tests.yml index d343098ed32..a25b96766d7 100644 --- a/.github/workflows/run_observatory_tests.yml +++ b/.github/workflows/run_observatory_tests.yml @@ -33,7 +33,9 @@ jobs: timeout-minutes: 30 steps: - name: Checkout repository - uses: actions/checkout@v4 + uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + with: + persist-credentials: false - name: Validate tag input env: @@ -49,11 +51,12 @@ jobs: TAG: ${{ inputs.tag }} AZURE_API_KEY: ${{ secrets.AZURE_API_KEY }} AZURE_API_BASE: ${{ secrets.AZURE_API_BASE }} + WORKSPACE: ${{ github.workspace }} run: | docker run -d \ --name litellm-rc \ -p 4000:4000 \ - -v "${{ github.workspace }}/.github/observatory/litellm_config.yaml:/app/config.yaml" \ + -v "${WORKSPACE}/.github/observatory/litellm_config.yaml:/app/config.yaml" \ -e LITELLM_MASTER_KEY="${LITELLM_MASTER_KEY}" \ -e AZURE_API_KEY="${AZURE_API_KEY}" \ -e AZURE_API_BASE="${AZURE_API_BASE}" \ @@ -77,8 +80,9 @@ jobs: - name: Start cloudflared tunnel run: | - # Install cloudflared + # Install cloudflared (pinned version + checksum) curl -sL https://github.com/cloudflare/cloudflared/releases/download/2025.2.1/cloudflared-linux-amd64 -o /usr/local/bin/cloudflared + echo "afdfadd1ef552e66bffc35246fe30a9bd578356d2d386de95585ccfc432472b8 /usr/local/bin/cloudflared" | sha256sum -c - chmod +x /usr/local/bin/cloudflared # Start a quick tunnel (no account needed) and capture the URL @@ -103,11 +107,11 @@ jobs: - name: Verify tunnel connectivity run: | - echo "Testing tunnel at ${{ env.TUNNEL_URL }}..." + echo "Testing tunnel at ${TUNNEL_URL}..." # Quick tunnels need time for DNS propagation; retry to avoid # transient NXDOMAIN (curl exit code 6) on first attempt. for i in $(seq 1 10); do - if curl -sf "${{ env.TUNNEL_URL }}/health/liveliness" > /dev/null 2>&1; then + if curl -sf "${TUNNEL_URL}/health/liveliness" > /dev/null 2>&1; then echo "Tunnel is working (attempt $i)" exit 0 fi @@ -221,5 +225,5 @@ jobs: - name: Cleanup if: always() run: | - kill "${{ env.CLOUDFLARED_PID }}" 2>/dev/null || true + kill "$CLOUDFLARED_PID" 2>/dev/null || true docker rm -f litellm-rc 2>/dev/null || true diff --git a/.github/workflows/scan_duplicate_issues.yml b/.github/workflows/scan_duplicate_issues.yml index 06e8f453a8c..222ff11f304 100644 --- a/.github/workflows/scan_duplicate_issues.yml +++ b/.github/workflows/scan_duplicate_issues.yml @@ -21,14 +21,15 @@ jobs: contents: read steps: - name: Checkout scripts - uses: actions/checkout@v4 + uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 with: sparse-checkout: .github/scripts + persist-credentials: false - name: Set up Python - uses: actions/setup-python@v5 + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: - python-version: "3.11" + python-version: "3.13" - name: Scan for duplicate issues env: diff --git a/.github/workflows/scorecard.yml b/.github/workflows/scorecard.yml new file mode 100644 index 00000000000..7cd12bb219c --- /dev/null +++ b/.github/workflows/scorecard.yml @@ -0,0 +1,47 @@ +name: Scorecard supply-chain security + +on: + branch_protection_rule: + schedule: + - cron: '27 12 * * 4' + push: + branches: ["main"] + +permissions: read-all + +jobs: + analysis: + name: Scorecard analysis + runs-on: ubuntu-latest + if: github.event.repository.default_branch == github.ref_name + permissions: + security-events: write + id-token: write + # Uncomment for private repos if needed: + # contents: read + # actions: read + + steps: + - name: Checkout code + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + persist-credentials: false + + - name: Run analysis + uses: ossf/scorecard-action@f49aabe0b5af0936a0987cfb85d86b75731b0186 # v2.4.1 + with: + results_file: results.sarif + results_format: sarif + publish_results: true + + - name: Upload artifact + uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1 + with: + name: SARIF file + path: results.sarif + retention-days: 5 + + - name: Upload to code scanning + uses: github/codeql-action/upload-sarif@c10b806170c8ee63ea24152429041b5624f0baf5 # v4.35.1 + with: + sarif_file: results.sarif diff --git a/.github/workflows/simple_pypi_publish.yml b/.github/workflows/simple_pypi_publish.yml deleted file mode 100644 index e1830556819..00000000000 --- a/.github/workflows/simple_pypi_publish.yml +++ /dev/null @@ -1,67 +0,0 @@ -name: Simple PyPI Publish - -on: - workflow_dispatch: - inputs: - version: - description: 'Version to publish (e.g., 1.74.10)' - required: true - type: string - -env: - TWINE_USERNAME: __token__ - -jobs: - publish: - runs-on: ubuntu-latest - if: github.repository == 'BerriAI/litellm' - - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: '3.8' - - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install toml build wheel twine - - - name: Update version in pyproject.toml - run: | - python -c " - import toml - - with open('pyproject.toml', 'r') as f: - data = toml.load(f) - - data['tool']['poetry']['version'] = '${{ github.event.inputs.version }}' - - with open('pyproject.toml', 'w') as f: - toml.dump(data, f) - - print(f'Updated version to ${{ github.event.inputs.version }}') - " - - - name: Copy model prices file - run: | - cp model_prices_and_context_window.json litellm/model_prices_and_context_window_backup.json - - - name: Build package - run: | - rm -rf build dist - python -m build - - - name: Publish to PyPI - env: - TWINE_PASSWORD: ${{ secrets.PYPI_PUBLISH_PASSWORD }} - run: | - twine upload dist/* - - - name: Output success - run: | - echo "✅ Successfully published litellm v${{ github.event.inputs.version }} to PyPI" - echo "📦 Package: https://pypi.org/project/litellm/${{ github.event.inputs.version }}/" \ No newline at end of file diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index 5a9b19fc9ca..c905bb12312 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -2,19 +2,24 @@ name: "Stale Issue Management" on: schedule: - - cron: '0 0 * * *' # Runs daily at midnight UTC + - cron: "0 0 * * *" # Runs daily at midnight UTC workflow_dispatch: +permissions: + issues: write + pull-requests: write + jobs: stale: + if: github.repository == 'BerriAI/litellm' runs-on: ubuntu-latest steps: - - uses: actions/stale@v8 + - uses: actions/stale@1160a2240286f5da8ec72b1c0816ce2481aabf84 # v8 with: repo-token: "${{ secrets.GITHUB_TOKEN }}" stale-issue-message: "This issue has been automatically marked as stale because it has not had recent activity. It will be closed if no further activity occurs." stale-pr-message: "This pull request has been automatically marked as stale because it has not had recent activity. It will be closed if no further activity occurs." - days-before-stale: 90 # Revert to 60 days - days-before-close: 7 # Revert to 7 days + days-before-stale: 90 # Revert to 60 days + days-before-close: 7 # Revert to 7 days stale-issue-label: "stale" - operations-per-run: 1000 \ No newline at end of file + operations-per-run: 1000 diff --git a/.github/workflows/sync-schema.yml b/.github/workflows/sync-schema.yml new file mode 100644 index 00000000000..72a5c56293e --- /dev/null +++ b/.github/workflows/sync-schema.yml @@ -0,0 +1,73 @@ +name: Sync schema.prisma copies + +on: + pull_request: + paths: + - 'schema.prisma' + +# Scoped to ONLY the permissions needed: +# - contents:write to push the sync commit to the PR branch +# - pull-requests:read is implicit (needed to check out the PR) +permissions: + contents: write + +jobs: + sync: + name: Copy root schema to proxy and proxy-extras + runs-on: ubuntu-latest + timeout-minutes: 5 + # Only run on PRs from branches in THIS repo (not forks). + # Fork PRs cannot push back to the head branch with GITHUB_TOKEN, + # and pull_request events from forks have read-only tokens anyway. + # Also reject PRs from branches named after protected branches to + # prevent pushing directly to main/master. + if: >- + github.event.pull_request.head.repo.full_name == github.repository + && github.head_ref != 'main' + && github.head_ref != 'master' + steps: + - name: Checkout PR branch by SHA + uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + with: + # Use the merge commit SHA for safety — github.head_ref is an + # attacker-controlled string (the branch name) and could contain + # unusual characters that cause unexpected git behavior. + ref: ${{ github.event.pull_request.head.sha }} + persist-credentials: true # needed for git push + + - name: Reject symlinked schema files + run: | + for f in schema.prisma litellm/proxy/schema.prisma litellm-proxy-extras/litellm_proxy_extras/schema.prisma; do + if [ -L "$f" ]; then + echo "::error file=$f::$f is a symlink, which is not allowed" + exit 1 + fi + done + + - name: Copy root schema to other locations + run: | + cp schema.prisma litellm/proxy/schema.prisma + cp schema.prisma litellm-proxy-extras/litellm_proxy_extras/schema.prisma + + - name: Check for changes + id: diff + run: | + if git diff --quiet -- litellm/proxy/schema.prisma litellm-proxy-extras/litellm_proxy_extras/schema.prisma; then + echo "changed=false" >> "$GITHUB_OUTPUT" + echo "Schemas already in sync. Nothing to do." + else + echo "changed=true" >> "$GITHUB_OUTPUT" + echo "Schema copies need updating." + fi + + - name: Commit synced schemas + if: steps.diff.outputs.changed == 'true' + run: | + # Push to the PR's head branch (need the branch name for git push). + # We checked out by SHA above for safety, so configure the push target explicitly. + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git checkout -B "$GITHUB_HEAD_REF" + git add -- litellm/proxy/schema.prisma litellm-proxy-extras/litellm_proxy_extras/schema.prisma + git commit -m "chore: sync schema.prisma copies from root" + git push origin "HEAD:$GITHUB_HEAD_REF" diff --git a/.github/workflows/test-linting.yml b/.github/workflows/test-linting.yml index 4cedb8b5bae..5bb85716a17 100644 --- a/.github/workflows/test-linting.yml +++ b/.github/workflows/test-linting.yml @@ -2,7 +2,10 @@ name: LiteLLM Linting on: pull_request: - branches: [ main ] + branches: [main] + +permissions: + contents: read jobs: lint: @@ -10,72 +13,73 @@ jobs: timeout-minutes: 5 steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - clean: true + - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + with: + fetch-depth: 0 + clean: true + persist-credentials: false - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: '3.12' + - name: Set up Python + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: "3.12" - - name: Install Poetry - uses: snok/install-poetry@v1 + - name: Install Poetry + run: pip install 'poetry==2.3.2' - - name: Clean Python cache - run: | - find . -type d -name "__pycache__" -exec rm -rf {} + || true - find . -name "*.pyc" -delete || true + - name: Clean Python cache + run: | + find . -type d -name "__pycache__" -exec rm -rf {} + || true + find . -name "*.pyc" -delete || true - - name: Check poetry.lock is up to date - run: | - poetry check --lock || (echo "❌ poetry.lock is out of sync with pyproject.toml. Run 'poetry lock' locally and commit the result." && exit 1) + - name: Check poetry.lock is up to date + run: | + poetry check --lock || (echo "❌ poetry.lock is out of sync with pyproject.toml. Run 'poetry lock' locally and commit the result." && exit 1) - - name: Install dependencies - run: | - poetry install --with dev + - name: Install dependencies + run: | + poetry install --with dev - - name: Check Black formatting - run: | - cd litellm - poetry run black --check --exclude '/enterprise/' . - cd .. + - name: Check Black formatting + run: | + cd litellm + poetry run black --check --exclude '/enterprise/' . + cd .. - - name: Debug - Check file state - run: | - echo "Current branch:" - git branch --show-current - echo "Last 3 commits:" - git log --oneline -3 - echo "File content around line 43:" - head -50 litellm/litellm_core_utils/custom_logger_registry.py | tail -10 - - - name: Run Ruff linting - run: | - cd litellm - poetry run ruff check . - cd .. + - name: Debug - Check file state + run: | + echo "Current branch:" + git branch --show-current + echo "Last 3 commits:" + git log --oneline -3 + echo "File content around line 43:" + head -50 litellm/litellm_core_utils/custom_logger_registry.py | tail -10 - - name: Print OpenAI version - run: | - poetry run python -c "import openai; print(f'OpenAI version: {openai.__version__}')" + - name: Run Ruff linting + run: | + cd litellm + poetry run ruff check . + cd .. - - name: Run MyPy type checking - run: | - cd litellm - poetry run mypy . - cd .. + - name: Print OpenAI version + run: | + poetry run python -c "import openai; print(f'OpenAI version: {openai.__version__}')" - - name: Check for circular imports - run: | - cd litellm - poetry run python ../tests/documentation_tests/test_circular_imports.py - cd .. + - name: Run MyPy type checking + run: | + cd litellm + poetry run mypy . + cd .. - - name: Check import safety - run: | - poetry run python -c "from litellm import *" || (echo '🚨 import failed, this means you introduced unprotected imports! 🚨'; exit 1) + - name: Check for circular imports + run: | + cd litellm + poetry run python ../tests/documentation_tests/test_circular_imports.py + cd .. + + - name: Check import safety + run: | + poetry run python -c "from litellm import *" || (echo '🚨 import failed, this means you introduced unprotected imports! 🚨'; exit 1) secret-scan: runs-on: ubuntu-latest @@ -84,27 +88,28 @@ jobs: contents: read steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 + - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + with: + fetch-depth: 0 + persist-credentials: false - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: '3.12' + - name: Set up Python + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: "3.12" - - name: Run secret scan test - run: | - pip install pytest - pytest tests/litellm/test_no_hardcoded_secrets.py -v + - name: Run secret scan test + run: | + pip install 'pytest==9.0.2' + pytest tests/litellm/test_no_hardcoded_secrets.py -v - - name: Run ggshield secret scan - env: - GITGUARDIAN_API_KEY: ${{ secrets.GITGUARDIAN_API_KEY }} - run: | - if [ -n "$GITGUARDIAN_API_KEY" ]; then - pip install ggshield - ggshield secret scan repo . - else - echo "GITGUARDIAN_API_KEY not set, skipping ggshield scan" - fi + - name: Run ggshield secret scan + env: + GITGUARDIAN_API_KEY: ${{ secrets.GITGUARDIAN_API_KEY }} + run: | + if [ -n "$GITGUARDIAN_API_KEY" ]; then + pip install 'ggshield==1.48.0' + ggshield secret scan repo . + else + echo "GITGUARDIAN_API_KEY not set, skipping ggshield scan" + fi diff --git a/.github/workflows/test-litellm-matrix.yml b/.github/workflows/test-litellm-matrix.yml index d0ac28ab41a..dafabe4d83e 100644 --- a/.github/workflows/test-litellm-matrix.yml +++ b/.github/workflows/test-litellm-matrix.yml @@ -4,6 +4,9 @@ on: pull_request: branches: [main] +permissions: + contents: read + # Cancel in-progress runs for the same PR concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} @@ -12,7 +15,7 @@ concurrency: jobs: test: runs-on: ubuntu-latest - timeout-minutes: 20 # Increased from 15 to 20 + timeout-minutes: 20 # Increased from 15 to 20 strategy: fail-fast: false matrix: @@ -43,7 +46,7 @@ jobs: - name: "integrations" path: "tests/test_litellm/integrations" workers: 2 - reruns: 3 # Integration tests tend to be flakier + reruns: 3 # Integration tests tend to be flakier - name: "core-utils" path: "tests/test_litellm/litellm_core_utils" workers: 2 @@ -117,18 +120,20 @@ jobs: name: test (${{ matrix.test-group.name }}) steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + with: + persist-credentials: false - name: Set up Python - uses: actions/setup-python@v5 + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version: "3.12" - name: Install Poetry - uses: snok/install-poetry@v1 + run: pip install 'poetry==2.3.2' - name: Cache Poetry dependencies - uses: actions/cache@v4 + uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.0.0 with: path: | ~/.cache/pypoetry @@ -144,14 +149,17 @@ jobs: poetry install --with dev,proxy-dev --extras "proxy semantic-router" # pytest-rerunfailures and pytest-xdist are in pyproject.toml dev dependencies poetry run pip install google-genai==1.22.0 \ - google-cloud-aiplatform>=1.38 fastapi-offline==1.7.3 python-multipart==0.0.22 openapi-core + google-cloud-aiplatform==1.115.0 fastapi-offline==1.7.3 python-multipart==0.0.22 openapi-core==0.23.0 - name: Setup litellm-enterprise run: | poetry run pip install --force-reinstall --no-deps -e enterprise/ - name: Generate Prisma client + env: + PRISMA_BINARY_CACHE_DIR: ${{ runner.temp }}/prisma-cache run: | + poetry run pip install nodejs-wheel-binaries==24.13.1 poetry run prisma generate --schema litellm/proxy/schema.prisma - name: Run tests - ${{ matrix.test-group.name }} @@ -163,4 +171,44 @@ jobs: --reruns ${{ matrix.test-group.reruns }} \ --reruns-delay 1 \ --dist=loadscope \ - --durations=20 + --durations=20 \ + --cov=litellm \ + --cov-report=xml:coverage-${{ matrix.test-group.name }}.xml \ + --cov-config=pyproject.toml + + - name: Save coverage report + if: always() + uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1 + with: + name: coverage-${{ matrix.test-group.name }} + path: coverage-${{ matrix.test-group.name }}.xml + retention-days: 1 + + upload-coverage: + name: Upload coverage to Codecov + needs: test + if: always() + runs-on: ubuntu-latest + permissions: + contents: read + id-token: write # Required for OIDC tokenless upload + pull-requests: write # Required for Codecov PR comments + + steps: + - name: Checkout code + uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + + - name: Download all coverage reports + uses: actions/download-artifact@95815c38cf2ff2164869cbab79da8d1f422bc89e # v4.2.1 + with: + pattern: coverage-* + path: coverage-reports + merge-multiple: true + + - name: Upload to Codecov + uses: codecov/codecov-action@aa56896cf108bd10b5eb883cd1d24196da57f695 # v5.5.4 + with: + use_oidc: true + directory: coverage-reports + root_dir: ${{ github.workspace }} + fail_ci_if_error: false diff --git a/.github/workflows/test-litellm-ui-build.yml b/.github/workflows/test-litellm-ui-build.yml index b0a8b648a44..bef568298e0 100644 --- a/.github/workflows/test-litellm-ui-build.yml +++ b/.github/workflows/test-litellm-ui-build.yml @@ -16,17 +16,19 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v4 + uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + with: + persist-credentials: false - name: Setup Node.js - uses: actions/setup-node@v4 + uses: actions/setup-node@a0853c24544627f65ddf259abe73b1d18a591444 # v5.0 with: node-version: "20" cache: "npm" cache-dependency-path: ui/litellm-dashboard/package-lock.json - name: Install dependencies - run: npm install + run: npm ci - name: Build run: npm run build diff --git a/.github/workflows/test-litellm.yml b/.github/workflows/test-litellm.yml index 3f8369df926..0c040b3ebe7 100644 --- a/.github/workflows/test-litellm.yml +++ b/.github/workflows/test-litellm.yml @@ -4,45 +4,50 @@ name: LiteLLM Mock Tests (folder - tests/test_litellm) # the same tests in parallel across 10 jobs for faster CI times. # Kept for manual debugging only. on: - workflow_dispatch: # Manual trigger only + workflow_dispatch: # Manual trigger only # pull_request: # branches: [ main ] +permissions: + contents: read + jobs: test: runs-on: ubuntu-latest timeout-minutes: 25 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + with: + persist-credentials: false - - name: Thank You Message - run: | - echo "### 🙏 Thank you for contributing to LiteLLM!" >> $GITHUB_STEP_SUMMARY - echo "Your PR is being tested now. We appreciate your help in making LiteLLM better!" >> $GITHUB_STEP_SUMMARY + - name: Thank You Message + run: | + echo "### 🙏 Thank you for contributing to LiteLLM!" >> $GITHUB_STEP_SUMMARY + echo "Your PR is being tested now. We appreciate your help in making LiteLLM better!" >> $GITHUB_STEP_SUMMARY - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: '3.12' + - name: Set up Python + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: "3.12" - - name: Install Poetry - uses: snok/install-poetry@v1 + - name: Install Poetry + run: pip install 'poetry==2.3.2' - - name: Install dependencies - run: | - poetry lock - poetry install --with dev,proxy-dev --extras "proxy semantic-router" - poetry run pip install "pytest-retry==1.6.3" - poetry run pip install pytest-xdist - poetry run pip install "google-genai==1.22.0" - poetry run pip install "google-cloud-aiplatform>=1.38" - poetry run pip install "fastapi-offline==1.7.3" - poetry run pip install "python-multipart>=0.0.20" - poetry run pip install "openapi-core" - - name: Setup litellm-enterprise as local package - run: | - poetry run pip install --force-reinstall --no-deps -e enterprise/ - - name: Run tests - run: | - poetry run pytest tests/test_litellm --tb=short -vv --maxfail=10 -n 4 --durations=50 + - name: Install dependencies + run: | + poetry lock + poetry install --with dev,proxy-dev --extras "proxy semantic-router" + poetry run pip install "pytest-retry==1.6.3" + poetry run pip install 'pytest-xdist==3.8.0' + poetry run pip install "google-genai==1.22.0" + poetry run pip install "google-cloud-aiplatform==1.115.0" + poetry run pip install "fastapi-offline==1.7.3" + poetry run pip install "python-multipart==0.0.22" + poetry run pip install "openapi-core==0.23.0" + - name: Setup litellm-enterprise as local package + run: | + poetry run pip install --force-reinstall --no-deps -e enterprise/ + - name: Run tests + run: | + poetry run pytest tests/test_litellm --tb=short -vv --maxfail=10 -n 4 --durations=50 diff --git a/.github/workflows/test-mcp.yml b/.github/workflows/test-mcp.yml index 2e32aae7680..1b228ab76bb 100644 --- a/.github/workflows/test-mcp.yml +++ b/.github/workflows/test-mcp.yml @@ -2,7 +2,10 @@ name: LiteLLM MCP Tests (folder - tests/mcp_tests) on: pull_request: - branches: [ main ] + branches: [main] + +permissions: + contents: read jobs: test: @@ -10,38 +13,40 @@ jobs: timeout-minutes: 25 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + with: + persist-credentials: false - - name: Thank You Message - run: | - echo "### 🙏 Thank you for contributing to LiteLLM!" >> $GITHUB_STEP_SUMMARY - echo "Your PR is being tested now. We appreciate your help in making LiteLLM better!" >> $GITHUB_STEP_SUMMARY + - name: Thank You Message + run: | + echo "### 🙏 Thank you for contributing to LiteLLM!" >> $GITHUB_STEP_SUMMARY + echo "Your PR is being tested now. We appreciate your help in making LiteLLM better!" >> $GITHUB_STEP_SUMMARY - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: '3.12' + - name: Set up Python + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: "3.12" - - name: Install Poetry - uses: snok/install-poetry@v1 + - name: Install Poetry + run: pip install 'poetry==2.3.2' - - name: Install dependencies - run: | - poetry lock - poetry install --with dev,proxy-dev --extras "proxy semantic-router" - poetry run pip install "pytest==7.3.1" - poetry run pip install "pytest-retry==1.6.3" - poetry run pip install "pytest-cov==5.0.0" - poetry run pip install "pytest-asyncio==0.21.1" - poetry run pip install "respx==0.22.0" - poetry run pip install "pydantic==2.11.0" - poetry run pip install "mcp==1.25.0" - poetry run pip install pytest-xdist + - name: Install dependencies + run: | + poetry lock + poetry install --with dev,proxy-dev --extras "proxy semantic-router" + poetry run pip install "pytest==7.3.1" + poetry run pip install "pytest-retry==1.6.3" + poetry run pip install "pytest-cov==5.0.0" + poetry run pip install "pytest-asyncio==0.21.1" + poetry run pip install "respx==0.22.0" + poetry run pip install "pydantic==2.11.0" + poetry run pip install "mcp==1.25.0" + poetry run pip install 'pytest-xdist==3.8.0' - - name: Setup litellm-enterprise as local package - run: | - poetry run pip install --force-reinstall --no-deps -e enterprise/ + - name: Setup litellm-enterprise as local package + run: | + poetry run pip install --force-reinstall --no-deps -e enterprise/ - - name: Run MCP tests - run: | - poetry run pytest tests/mcp_tests -x -vv -n 4 --cov=litellm --cov-report=xml --durations=5 + - name: Run MCP tests + run: | + poetry run pytest tests/mcp_tests -x -vv -n 4 --cov=litellm --cov-report=xml --durations=5 diff --git a/.github/workflows/test-model-map.yaml b/.github/workflows/test-model-map.yaml index ae5ac402e23..429f9e1ce0a 100644 --- a/.github/workflows/test-model-map.yaml +++ b/.github/workflows/test-model-map.yaml @@ -2,13 +2,18 @@ name: Validate model_prices_and_context_window.json on: pull_request: - branches: [ main ] + branches: [main] + +permissions: + contents: read jobs: validate-model-prices-json: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + with: + persist-credentials: false - name: Validate model_prices_and_context_window.json run: | diff --git a/.github/workflows/test-proxy-e2e-azure-batches.yml b/.github/workflows/test-proxy-e2e-azure-batches.yml index 4d74f3db0ac..7cbbe0b338f 100644 --- a/.github/workflows/test-proxy-e2e-azure-batches.yml +++ b/.github/workflows/test-proxy-e2e-azure-batches.yml @@ -9,6 +9,9 @@ concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} cancel-in-progress: true +permissions: + contents: read + jobs: proxy_e2e_azure_batches_tests: runs-on: ubuntu-latest @@ -30,18 +33,20 @@ jobs: --health-retries 5 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + with: + persist-credentials: false - name: Set up Python - uses: actions/setup-python@v5 + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version: "3.12" - name: Install Poetry - uses: snok/install-poetry@v1 + run: pip install 'poetry==2.3.2' - name: Cache Poetry dependencies - uses: actions/cache@v4 + uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.0.0 with: path: | ~/.cache/pypoetry @@ -56,14 +61,17 @@ jobs: run: | poetry config virtualenvs.in-project true poetry install --with dev,proxy-dev --extras "proxy" - poetry run pip install psycopg2-binary uvicorn fastapi httpx tenacity + poetry run pip install psycopg2-binary==2.9.11 uvicorn==0.42.0 fastapi==0.135.2 httpx==0.28.1 tenacity==9.1.4 - name: Setup litellm-enterprise run: | poetry run pip install --force-reinstall --no-deps -e enterprise/ - name: Generate Prisma client + env: + PRISMA_BINARY_CACHE_DIR: ${{ runner.temp }}/prisma-cache run: | + poetry run pip install nodejs-wheel-binaries==24.13.1 poetry run prisma generate --schema litellm/proxy/schema.prisma - name: Run Prisma migrations @@ -87,4 +95,3 @@ jobs: --tb=short \ --maxfail=3 \ --durations=10 - diff --git a/.github/workflows/test-unit-caching-redis.yml b/.github/workflows/test-unit-caching-redis.yml new file mode 100644 index 00000000000..ca274324f2f --- /dev/null +++ b/.github/workflows/test-unit-caching-redis.yml @@ -0,0 +1,38 @@ +name: "Unit Tests: Caching (Redis)" + +# Uses cloud Redis credentials — only runs on trusted branches, not PRs. +# This prevents external PRs from accessing Redis credentials. +on: + push: + branches: [main, "litellm_*"] + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + caching-redis: + uses: ./.github/workflows/_test-unit-services-base.yml + with: + # Redis-only tests that do NOT require provider API keys. + # Tests needing API keys (test_caching.py, test_caching_ssl.py, test_prometheus_service.py, + # test_router_caching.py) are in Phase 3 integration workflows. + test-path: >- + tests/local_testing/test_dual_cache.py + tests/local_testing/test_redis_batch_optimizations.py + tests/local_testing/test_router_utils.py + workers: 2 + reruns: 2 + timeout-minutes: 20 + enable-redis: true + enable-postgres: false + secrets: + REDIS_HOST: ${{ secrets.REDIS_HOST }} + REDIS_PORT: ${{ secrets.REDIS_PORT }} + REDIS_PASSWORD: ${{ secrets.REDIS_PASSWORD }} + DATABASE_URL: ${{ secrets.DATABASE_URL }} + POSTGRES_USER: ${{ secrets.POSTGRES_USER }} + POSTGRES_PASSWORD: ${{ secrets.POSTGRES_PASSWORD }} diff --git a/.github/workflows/test-unit-core-utils.yml b/.github/workflows/test-unit-core-utils.yml new file mode 100644 index 00000000000..2f3698fdf60 --- /dev/null +++ b/.github/workflows/test-unit-core-utils.yml @@ -0,0 +1,20 @@ +name: "Unit Tests: Core Utilities" + +on: + pull_request: + branches: [main] + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + core-utils: + uses: ./.github/workflows/_test-unit-base.yml + with: + test-path: "tests/test_litellm/litellm_core_utils" + workers: 2 + reruns: 1 diff --git a/.github/workflows/test-unit-documentation.yml b/.github/workflows/test-unit-documentation.yml new file mode 100644 index 00000000000..d8b30de6844 --- /dev/null +++ b/.github/workflows/test-unit-documentation.yml @@ -0,0 +1,67 @@ +name: "Unit Tests: Documentation Validation" + +on: + pull_request: + branches: [main] + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + documentation: + runs-on: ubuntu-latest + timeout-minutes: 10 + + steps: + - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + with: + persist-credentials: false + + - name: Set up Python + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: "3.12" + + - name: Install Poetry + run: pip install 'poetry==2.3.2' + + - name: Cache Poetry dependencies + uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 + with: + path: | + ~/.cache/pypoetry + ~/.cache/pip + .venv + key: ${{ runner.os }}-poetry-${{ hashFiles('poetry.lock') }} + restore-keys: | + ${{ runner.os }}-poetry- + + - name: Install dependencies + run: | + poetry config virtualenvs.in-project true + poetry install --with dev,proxy-dev --extras "proxy semantic-router" + poetry run pip install google-genai==1.22.0 \ + google-cloud-aiplatform==1.115.0 fastapi-offline==1.7.3 python-multipart==0.0.22 openapi-core==0.23.0 + + - name: Setup litellm-enterprise + run: | + poetry run pip install --force-reinstall --no-deps -e enterprise/ + + - name: Generate Prisma client + env: + PRISMA_BINARY_CACHE_DIR: ${{ runner.temp }}/prisma-cache + run: | + poetry run pip install nodejs-wheel-binaries==24.13.1 + poetry run prisma generate --schema litellm/proxy/schema.prisma + + # Run the same documentation tests that CircleCI ran (as direct Python scripts) + - name: Run documentation validation tests + run: | + poetry run python ./tests/documentation_tests/test_env_keys.py + poetry run python ./tests/documentation_tests/test_router_settings.py + poetry run python ./tests/documentation_tests/test_api_docs.py + poetry run python ./tests/documentation_tests/test_circular_imports.py diff --git a/.github/workflows/test-unit-enterprise-routing.yml b/.github/workflows/test-unit-enterprise-routing.yml new file mode 100644 index 00000000000..13ae3efedba --- /dev/null +++ b/.github/workflows/test-unit-enterprise-routing.yml @@ -0,0 +1,24 @@ +name: "Unit Tests: Enterprise, Google GenAI & Routing" + +on: + pull_request: + branches: [main] + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + enterprise-routing: + uses: ./.github/workflows/_test-unit-base.yml + with: + test-path: >- + tests/test_litellm/enterprise + tests/test_litellm/google_genai + tests/test_litellm/router_utils + tests/test_litellm/router_strategy + workers: 2 + reruns: 2 diff --git a/.github/workflows/test-unit-integrations.yml b/.github/workflows/test-unit-integrations.yml new file mode 100644 index 00000000000..2789f99d81c --- /dev/null +++ b/.github/workflows/test-unit-integrations.yml @@ -0,0 +1,20 @@ +name: "Unit Tests: Integrations (Callbacks & Logging)" + +on: + pull_request: + branches: [main] + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + integrations: + uses: ./.github/workflows/_test-unit-base.yml + with: + test-path: "tests/test_litellm/integrations" + workers: 2 + reruns: 3 diff --git a/.github/workflows/test-unit-llm-providers.yml b/.github/workflows/test-unit-llm-providers.yml new file mode 100644 index 00000000000..6c00272b0c8 --- /dev/null +++ b/.github/workflows/test-unit-llm-providers.yml @@ -0,0 +1,29 @@ +name: "Unit Tests: LLM Provider Transformations" + +on: + pull_request: + branches: [main] + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + vertex-ai: + name: Vertex AI + uses: ./.github/workflows/_test-unit-base.yml + with: + test-path: "tests/test_litellm/llms/vertex_ai" + workers: 1 + reruns: 2 + + other-providers: + name: All Other Providers + uses: ./.github/workflows/_test-unit-base.yml + with: + test-path: "tests/test_litellm/llms --ignore=tests/test_litellm/llms/vertex_ai" + workers: 2 + reruns: 2 diff --git a/.github/workflows/test-unit-misc.yml b/.github/workflows/test-unit-misc.yml new file mode 100644 index 00000000000..9228decd7cc --- /dev/null +++ b/.github/workflows/test-unit-misc.yml @@ -0,0 +1,31 @@ +name: "Unit Tests: MCP, Secrets, Containers & Misc" + +on: + pull_request: + branches: [main] + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + misc: + uses: ./.github/workflows/_test-unit-base.yml + with: + test-path: >- + tests/test_litellm/secret_managers + tests/test_litellm/a2a_protocol + tests/test_litellm/anthropic_interface + tests/test_litellm/completion_extras + tests/test_litellm/containers + tests/test_litellm/experimental_mcp_client + tests/test_litellm/images + tests/test_litellm/interactions + tests/test_litellm/passthrough + tests/test_litellm/vector_stores + tests/test_litellm/test_*.py + workers: 2 + reruns: 2 diff --git a/.github/workflows/test-unit-proxy-auth.yml b/.github/workflows/test-unit-proxy-auth.yml new file mode 100644 index 00000000000..e71821db701 --- /dev/null +++ b/.github/workflows/test-unit-proxy-auth.yml @@ -0,0 +1,20 @@ +name: "Unit Tests: Proxy Auth & Key Management" + +on: + pull_request: + branches: [main] + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + proxy-auth: + uses: ./.github/workflows/_test-unit-base.yml + with: + test-path: "tests/test_litellm/proxy/auth tests/test_litellm/proxy/hooks tests/test_litellm/proxy/policy_engine tests/test_litellm/proxy/client" + workers: 2 + reruns: 2 diff --git a/.github/workflows/test-unit-proxy-db.yml b/.github/workflows/test-unit-proxy-db.yml new file mode 100644 index 00000000000..bdfb6efeef1 --- /dev/null +++ b/.github/workflows/test-unit-proxy-db.yml @@ -0,0 +1,45 @@ +name: "Unit Tests: Proxy DB Operations" + +# Uses DATABASE_URL secret — only runs on trusted branches, not PRs. +on: + push: + branches: [main, "litellm_*"] + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + proxy-db: + strategy: + fail-fast: false + matrix: + include: + # Key generation tests must NOT run in parallel (event loop conflicts with logging worker) + - test-group: key-generation + test-path: "tests/proxy_unit_tests/test_key_generate_prisma.py" + workers: 0 + timeout: 30 + - test-group: auth-checks + test-path: "tests/proxy_unit_tests/test_auth_checks.py tests/proxy_unit_tests/test_user_api_key_auth.py" + workers: 8 + timeout: 20 + - test-group: remaining + test-path: "tests/proxy_unit_tests --ignore=tests/proxy_unit_tests/test_key_generate_prisma.py --ignore=tests/proxy_unit_tests/test_auth_checks.py --ignore=tests/proxy_unit_tests/test_user_api_key_auth.py" + workers: 8 + timeout: 20 + uses: ./.github/workflows/_test-unit-services-base.yml + with: + test-path: ${{ matrix.test-path }} + workers: ${{ matrix.workers }} + reruns: 2 + timeout-minutes: ${{ matrix.timeout }} + enable-redis: false + enable-postgres: true + secrets: + DATABASE_URL: ${{ secrets.DATABASE_URL }} + POSTGRES_USER: ${{ secrets.POSTGRES_USER }} + POSTGRES_PASSWORD: ${{ secrets.POSTGRES_PASSWORD }} diff --git a/.github/workflows/test-unit-proxy-endpoints.yml b/.github/workflows/test-unit-proxy-endpoints.yml new file mode 100644 index 00000000000..caff3b3ae06 --- /dev/null +++ b/.github/workflows/test-unit-proxy-endpoints.yml @@ -0,0 +1,35 @@ +name: "Unit Tests: Proxy API Endpoints" + +on: + pull_request: + branches: [main] + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + proxy-endpoints: + uses: ./.github/workflows/_test-unit-base.yml + with: + test-path: >- + tests/test_litellm/proxy/management_endpoints + tests/test_litellm/proxy/guardrails + tests/test_litellm/proxy/management_helpers + tests/test_litellm/proxy/anthropic_endpoints + tests/test_litellm/proxy/google_endpoints + tests/test_litellm/proxy/openai_files_endpoint + tests/test_litellm/proxy/response_api_endpoints + tests/test_litellm/proxy/image_endpoints + tests/test_litellm/proxy/vector_store_endpoints + tests/test_litellm/proxy/agent_endpoints + tests/test_litellm/proxy/discovery_endpoints + tests/test_litellm/proxy/health_endpoints + tests/test_litellm/proxy/public_endpoints + tests/test_litellm/proxy/prompts + tests/test_litellm/proxy/ui_crud_endpoints + workers: 2 + reruns: 2 diff --git a/.github/workflows/test-unit-proxy-infra.yml b/.github/workflows/test-unit-proxy-infra.yml new file mode 100644 index 00000000000..4dfbbe317ed --- /dev/null +++ b/.github/workflows/test-unit-proxy-infra.yml @@ -0,0 +1,28 @@ +name: "Unit Tests: Proxy Infrastructure" + +on: + pull_request: + branches: [main] + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + proxy-infra: + uses: ./.github/workflows/_test-unit-base.yml + with: + test-path: >- + tests/test_litellm/proxy/db + tests/test_litellm/proxy/middleware + tests/test_litellm/proxy/spend_tracking + tests/test_litellm/proxy/pass_through_endpoints + tests/test_litellm/proxy/_experimental + tests/test_litellm/proxy/experimental + tests/test_litellm/proxy/common_utils + tests/test_litellm/proxy/test_*.py + workers: 2 + reruns: 2 diff --git a/.github/workflows/test-unit-proxy-legacy.yml b/.github/workflows/test-unit-proxy-legacy.yml new file mode 100644 index 00000000000..a9391137263 --- /dev/null +++ b/.github/workflows/test-unit-proxy-legacy.yml @@ -0,0 +1,96 @@ +name: "Unit Tests: Proxy Legacy Tests" + +on: + pull_request: + branches: [main] + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + test: + runs-on: ubuntu-latest + timeout-minutes: 20 + strategy: + fail-fast: false + matrix: + test-group: + - name: "auth-and-jwt" + path: "tests/proxy_unit_tests/test_[a-j]*.py" + - name: "key-generation" + path: "tests/proxy_unit_tests/test_[k-o]*.py" + - name: "proxy-config" + path: "tests/proxy_unit_tests/test_prisma*.py tests/proxy_unit_tests/test_project*.py tests/proxy_unit_tests/test_prompt*.py tests/proxy_unit_tests/test_proxy_[c-r]*.py" + - name: "proxy-server" + path: "tests/proxy_unit_tests/test_proxy_server.py" + - name: "proxy-server-extras" + path: "tests/proxy_unit_tests/test_proxy_server_*.py tests/proxy_unit_tests/test_proxy_setting_guardrails.py" + - name: "proxy-utils" + path: "tests/proxy_unit_tests/test_proxy_utils.py" + - name: "proxy-token-counter" + path: "tests/proxy_unit_tests/test_proxy_token_counter.py" + - name: "proxy-response-and-misc" + path: "tests/proxy_unit_tests/test_[r-t]*.py" + - name: "proxy-user-auth-and-spend" + path: "tests/proxy_unit_tests/test_[u-z]*.py" + + name: ${{ matrix.test-group.name }} + + steps: + - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + with: + persist-credentials: false + + - name: Set up Python + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: "3.12" + + - name: Install Poetry + run: pip install 'poetry==2.3.2' + + - name: Cache Poetry dependencies + uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 + with: + path: | + ~/.cache/pypoetry + ~/.cache/pip + .venv + key: ${{ runner.os }}-poetry-${{ hashFiles('poetry.lock') }} + restore-keys: | + ${{ runner.os }}-poetry- + + - name: Install dependencies + run: | + poetry config virtualenvs.in-project true + poetry install --with dev,proxy-dev --extras "proxy semantic-router" + poetry run pip install google-genai==1.22.0 \ + google-cloud-aiplatform==1.115.0 fastapi-offline==1.7.3 python-multipart==0.0.22 openapi-core==0.23.0 + + - name: Setup litellm-enterprise + run: | + poetry run pip install --force-reinstall --no-deps -e enterprise/ + + - name: Generate Prisma client + env: + PRISMA_BINARY_CACHE_DIR: ${{ runner.temp }}/prisma-cache + run: | + poetry run pip install nodejs-wheel-binaries==24.13.1 + poetry run prisma generate --schema litellm/proxy/schema.prisma + + - name: Run tests - ${{ matrix.test-group.name }} + env: + TEST_PATH: ${{ matrix.test-group.path }} + run: | + poetry run pytest ${TEST_PATH} \ + --tb=short -vv \ + --maxfail=10 \ + -n 2 \ + --reruns 1 \ + --reruns-delay 1 \ + --dist=loadscope \ + --durations=20 diff --git a/.github/workflows/test-unit-responses-caching-types.yml b/.github/workflows/test-unit-responses-caching-types.yml new file mode 100644 index 00000000000..7f3acac2803 --- /dev/null +++ b/.github/workflows/test-unit-responses-caching-types.yml @@ -0,0 +1,20 @@ +name: "Unit Tests: Responses, Caching & Types" + +on: + pull_request: + branches: [main] + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + responses-caching-types: + uses: ./.github/workflows/_test-unit-base.yml + with: + test-path: "tests/test_litellm/responses tests/test_litellm/caching tests/test_litellm/types" + workers: 2 + reruns: 2 diff --git a/.github/workflows/test-unit-security.yml b/.github/workflows/test-unit-security.yml new file mode 100644 index 00000000000..b38c82b1c24 --- /dev/null +++ b/.github/workflows/test-unit-security.yml @@ -0,0 +1,28 @@ +name: "Unit Tests: Security" + +# Uses DATABASE_URL secret — only runs on trusted branches, not PRs. +on: + push: + branches: [main, "litellm_*"] + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + security: + uses: ./.github/workflows/_test-unit-services-base.yml + with: + test-path: "tests/proxy_security_tests/" + workers: 1 + reruns: 2 + timeout-minutes: 20 + enable-redis: false + enable-postgres: true + secrets: + DATABASE_URL: ${{ secrets.DATABASE_URL }} + POSTGRES_USER: ${{ secrets.POSTGRES_USER }} + POSTGRES_PASSWORD: ${{ secrets.POSTGRES_PASSWORD }} diff --git a/.github/workflows/test_server_root_path.yml b/.github/workflows/test_server_root_path.yml index c359e38bff9..47636ce8e92 100644 --- a/.github/workflows/test_server_root_path.yml +++ b/.github/workflows/test_server_root_path.yml @@ -17,13 +17,15 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v4 + uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + with: + persist-credentials: false - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 + uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12 - name: Build Docker image - uses: docker/build-push-action@v5 + uses: docker/build-push-action@0adf9959216b96bec444f325f1e493d4aa344497 #v6.14 with: context: . file: ./docker/Dockerfile.non_root diff --git a/.github/workflows/zizmor.yml b/.github/workflows/zizmor.yml new file mode 100644 index 00000000000..9a1e899fed5 --- /dev/null +++ b/.github/workflows/zizmor.yml @@ -0,0 +1,31 @@ +name: GitHub Actions Security Analysis + +on: + push: + branches: [main] + pull_request: + branches: [main] + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +permissions: {} + +jobs: + zizmor: + name: zizmor + runs-on: ubuntu-latest + timeout-minutes: 5 + permissions: + security-events: write + contents: read + actions: read + steps: + - name: Checkout repository + uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + with: + persist-credentials: false + + - name: Run zizmor + uses: zizmorcore/zizmor-action@71321a20a9ded102f6e9ce5718a2fcec2c4f70d8 # v0.5.2 diff --git a/.gitignore b/.gitignore index 76cf6fdba2a..38bf9554b5b 100644 --- a/.gitignore +++ b/.gitignore @@ -72,8 +72,7 @@ tests/local_testing/log.txt .codegpt litellm/proxy/_new_new_secret_config.yaml litellm/proxy/custom_guardrail.py -.mypy_cache/* -.mypy_cache/* +**/.mypy_cache/ litellm/proxy/application.log tests/llm_translation/vertex_test_account.json tests/llm_translation/test_vertex_key.json diff --git a/.npmrc b/.npmrc new file mode 100644 index 00000000000..168e81a1c4e --- /dev/null +++ b/.npmrc @@ -0,0 +1,5 @@ +# Supply-chain hardening +# Packages needing lifecycle scripts: npm rebuild +ignore-scripts=true +# Protects local npm install only — npm ci (used in CI) ignores this +min-release-age=3d diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml deleted file mode 100644 index 2bc361bc48f..00000000000 --- a/.pre-commit-config.yaml +++ /dev/null @@ -1,40 +0,0 @@ -repos: -- repo: local - hooks: - - id: pyright - name: pyright - entry: pyright - language: system - types: [python] - files: ^(litellm/|litellm_proxy_extras/|enterprise/) - - id: isort - name: isort - entry: isort - language: system - types: [python] - files: (litellm/|litellm_proxy_extras/|enterprise/).*\.py - exclude: ^litellm/__init__.py$ - - id: black - name: black - entry: poetry run black - language: system - types: [python] - files: (litellm/|litellm_proxy_extras/).*\.py -- repo: https://github.com/pycqa/flake8 - rev: 7.0.0 # The version of flake8 to use - hooks: - - id: flake8 - exclude: ^litellm/tests/|^litellm/proxy/tests/|^litellm/tests/test_litellm/|^tests/test_litellm/|^tests/enterprise/ - additional_dependencies: [flake8-print] - files: (litellm/|litellm_proxy_extras/|enterprise/).*\.py -- repo: https://github.com/python-poetry/poetry - rev: 1.8.0 - hooks: - - id: poetry-check - files: ^(pyproject.toml|litellm-proxy-extras/pyproject.toml)$ -- repo: local - hooks: - - id: check-files-match - name: Check if files match - entry: python3 ci_cd/check_files_match.py - language: system \ No newline at end of file diff --git a/.semgrep/rules/security/no-claude-directory.yml b/.semgrep/rules/security/no-claude-directory.yml new file mode 100644 index 00000000000..7d120a7c23c --- /dev/null +++ b/.semgrep/rules/security/no-claude-directory.yml @@ -0,0 +1,18 @@ +rules: + - id: no-claude-directory-committed + message: > + .claude/ directory must not be committed to the repository. + It contains local Claude Code settings (permissions, worktree paths) that are + developer-machine-specific and may expose internal paths or credentials. + Add .claude/ to .gitignore instead. + severity: ERROR + languages: [generic] + paths: + include: + - "/.claude/**" + - "/.claude/*" + pattern-regex: '[\s\S]+' + metadata: + category: security + tags: [supply-chain, secrets] + confidence: HIGH diff --git a/Dockerfile b/Dockerfile index 7bda32acf27..f4cb501ad8b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,8 +1,8 @@ # Base image for building -ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base +ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:a5a619c1793039dcf92f02178f37c94bb3d6001403716da59d6092dfe8d9b502 # Runtime image -ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base +ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:a5a619c1793039dcf92f02178f37c94bb3d6001403716da59d6092dfe8d9b502 # Builder stage FROM $LITELLM_BUILD_IMAGE AS builder @@ -15,7 +15,7 @@ USER root # Install build dependencies RUN apk add --no-cache bash gcc py3-pip python3 python3-dev openssl openssl-dev -RUN python -m pip install build +RUN python -m pip install build==1.4.2 # Copy the current directory contents into the container at /app COPY . . @@ -49,7 +49,7 @@ USER root # Install runtime dependencies (libsndfile needed for audio processing on ARM64) RUN apk add --no-cache bash openssl tzdata nodejs npm python3 py3-pip libsndfile && \ - npm install -g npm@latest tar@7.5.11 glob@11.1.0 @isaacs/brace-expansion@5.0.1 minimatch@10.2.4 diff@8.0.3 && \ + npm install -g npm@11.12.1 tar@7.5.11 glob@11.1.0 @isaacs/brace-expansion@5.0.1 minimatch@10.2.4 diff@8.0.3 && \ # SECURITY FIX: npm bundles tar, glob, and brace-expansion at multiple nested # levels inside its dependency tree. `npm install -g ` only creates a # SEPARATE global package, it does NOT replace npm's internal copies. @@ -90,7 +90,7 @@ COPY --from=builder /app/dist/*.whl . COPY --from=builder /wheels/ /wheels/ # Install the built wheel using pip; again using a wildcard if it's the only file -RUN pip install *.whl /wheels/* --no-index --find-links=/wheels/ && rm -f *.whl && rm -rf /wheels +RUN pip install *.whl /wheels/* --no-index --find-links=/wheels/ --no-deps && rm -f *.whl && rm -rf /wheels # Replace the nodejs-wheel-binaries bundled node with the system node (fixes CVE-2025-55130) RUN NODEJS_WHEEL_NODE=$(find /usr/lib -path "*/nodejs_wheel/bin/node" 2>/dev/null) && \ diff --git a/README.md b/README.md index 67f2f3a2048..a6dc597574e 100644 --- a/README.md +++ b/README.md @@ -266,6 +266,7 @@ Support for more providers. Missing a provider or LLM Platform, raise a [feature + @@ -402,7 +403,7 @@ Support for more providers. Missing a provider or LLM Platform, raise a [feature # Enterprise For companies that need better security, user management and professional support -[Talk to founders](https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions) +[Talk to founders](https://enterprise.litellm.ai/demo) This covers: - ✅ **Features under the [LiteLLM Commercial License](https://docs.litellm.ai/docs/proxy/enterprise):** @@ -452,7 +453,6 @@ All these checks must pass before your PR can be merged. - [Schedule Demo 👋](https://calendly.com/d/4mp-gd3-k5k/berriai-1-1-onboarding-litellm-hosted-version) - [Community Discord 💭](https://discord.gg/wuPM9dRgDw) - [Community Slack 💭](https://www.litellm.ai/support) -- Our numbers 📞 +1 (770) 8783-106 / ‭+1 (412) 618-6238‬ - Our emails ✉️ ishaan@berri.ai / krrish@berri.ai # Why did we build this diff --git a/ci_cd/publish-proxy-extras.sh b/ci_cd/publish-proxy-extras.sh deleted file mode 100644 index 6c83d1f9212..00000000000 --- a/ci_cd/publish-proxy-extras.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/bin/bash - -# Exit on error -set -e - -echo "🚀 Building and publishing litellm-proxy-extras" - -# Navigate to litellm-proxy-extras directory -cd "$(dirname "$0")/../litellm-proxy-extras" - -# Build the package -echo "📦 Building package..." -poetry build - -# Publish to PyPI -echo "🌎 Publishing to PyPI..." -poetry publish - -echo "✅ Done! Package published successfully" \ No newline at end of file diff --git a/ci_cd/security_scans.sh b/ci_cd/security_scans.sh index 801b700f64f..2138fca6cd5 100755 --- a/ci_cd/security_scans.sh +++ b/ci_cd/security_scans.sh @@ -10,13 +10,13 @@ echo "Starting security scans for LiteLLM..." # Function to install Trivy and required tools install_trivy() { echo "Installing Trivy and required tools..." + TRIVY_VERSION="0.35.0" sudo apt-get update - sudo apt-get install -y wget apt-transport-https gnupg lsb-release jq curl bsdmainutils - wget -qO - https://aquasecurity.github.io/trivy-repo/deb/public.key | sudo apt-key add - - echo "deb https://aquasecurity.github.io/trivy-repo/deb $(lsb_release -sc) main" | sudo tee -a /etc/apt/sources.list.d/trivy.list - sudo apt-get update - sudo apt-get install trivy - echo "Trivy and required tools installed successfully" + sudo apt-get install -y wget jq curl bsdmainutils + wget -qO trivy.deb "https://github.com/aquasecurity/trivy/releases/download/v${TRIVY_VERSION}/trivy_${TRIVY_VERSION}_Linux-64bit.deb" + sudo dpkg -i trivy.deb + rm trivy.deb + echo "Trivy ${TRIVY_VERSION} installed successfully" } # Function to install Grype @@ -160,7 +160,6 @@ run_grype_scans() { "CVE-2026-0775" # npm cli incorrect permission assignment - no fix available yet, npm is only used at build/prisma-generate time "GHSA-3ppc-4f35-3m26" # minimatch ReDoS via repeated wildcards - from nodejs_wheel bundled npm, not used in application runtime code "GHSA-83g3-92jg-28cx" # tar arbitrary file read/write via hardlink - from nodejs_wheel bundled npm, not used in application runtime code - "CVE-2026-25639" # axios - full fix requires 1.x major version bump; pinned to >=0.30.2 to clear other axios CVEs, upgrade to 1.x in follow-up "CVE-2026-2297" # Python 3.13 SourcelessFileLoader audit hook bypass - no fix available in base image "GHSA-qffp-2rhf-9h96" # tar hardlink path traversal - from nodejs_wheel bundled npm, not used in application runtime code "CVE-2026-2673" # OpenSSL 3.6.1 TLS 1.3 key exchange group negotiation issue - no fix available yet diff --git a/codecov.yaml b/codecov.yaml index c25cf0fbae8..09fccc6b995 100644 --- a/codecov.yaml +++ b/codecov.yaml @@ -17,6 +17,9 @@ component_management: - component_id: "Proxy_Authentication" paths: - "*/proxy/auth/**" + - component_id: "Enterprise" + paths: + - "enterprise/**" comment: layout: "header, diff, flags, components" # show component info in the PR comment diff --git a/cookbook/ai_coding_tool_guides/claude_code_quickstart/guide.md b/cookbook/ai_coding_tool_guides/claude_code_quickstart/guide.md index 3d6c75498b1..b2d81be25bb 100644 --- a/cookbook/ai_coding_tool_guides/claude_code_quickstart/guide.md +++ b/cookbook/ai_coding_tool_guides/claude_code_quickstart/guide.md @@ -230,7 +230,7 @@ model_list: # AWS Bedrock - model_name: claude-bedrock litellm_params: - model: bedrock/anthropic.claude-3-5-sonnet-20241022-v2:0 + model: bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0 aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY aws_region_name: us-east-1 diff --git a/cookbook/benchmark/readme.md b/cookbook/benchmark/readme.md index 57115eb96a9..afa59aa91ee 100644 --- a/cookbook/benchmark/readme.md +++ b/cookbook/benchmark/readme.md @@ -178,4 +178,4 @@ Benchmark Results for 'When will BerriAI IPO?': ``` ## Support -**🤝 Schedule a 1-on-1 Session:** Book a [1-on-1 session](https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions) with Krrish and Ishaan, the founders, to discuss any issues, provide feedback, or explore how we can improve LiteLLM for you. +**🤝 Schedule a 1-on-1 Session:** Book a [1-on-1 session](https://enterprise.litellm.ai/demo) with Krrish and Ishaan, the founders, to discuss any issues, provide feedback, or explore how we can improve LiteLLM for you. diff --git a/cookbook/codellama-server/README.MD b/cookbook/codellama-server/README.MD index b158bb083f2..82a7e62f40a 100644 --- a/cookbook/codellama-server/README.MD +++ b/cookbook/codellama-server/README.MD @@ -143,7 +143,6 @@ All responses from the server are returned in the following format (for all LLM - [Our calendar 👋](https://calendly.com/d/4mp-gd3-k5k/berriai-1-1-onboarding-litellm-hosted-version) - [Community Discord 💭](https://discord.gg/wuPM9dRgDw) -- Our numbers 📞 +1 (770) 8783-106 / +1 (412) 618-6238 - Our emails ✉️ ishaan@berri.ai / krrish@berri.ai ## Roadmap diff --git a/cookbook/litellm_proxy_server/readme.md b/cookbook/litellm_proxy_server/readme.md index d0b0592c433..2c1eab72c24 100644 --- a/cookbook/litellm_proxy_server/readme.md +++ b/cookbook/litellm_proxy_server/readme.md @@ -164,7 +164,6 @@ All responses from the server are returned in the following format (for all LLM - [Our calendar 👋](https://calendly.com/d/4mp-gd3-k5k/berriai-1-1-onboarding-litellm-hosted-version) - [Community Discord 💭](https://discord.gg/wuPM9dRgDw) -- Our numbers 📞 +1 (770) 8783-106 / +1 (412) 618-6238 - Our emails ✉️ ishaan@berri.ai / krrish@berri.ai ## Roadmap diff --git a/cookbook/misc/test_responses_api.py b/cookbook/misc/test_responses_api.py index 5fd19c6f66f..62e4e2cf62e 100644 --- a/cookbook/misc/test_responses_api.py +++ b/cookbook/misc/test_responses_api.py @@ -20,7 +20,7 @@ base64_image = encode_image(image_path) response = client.responses.create( - model="bedrock/us.anthropic.claude-3-5-sonnet-20241022-v2:0", + model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", input=[ { "role": "user", @@ -43,7 +43,7 @@ print("sleeping for 20 seconds...") time.sleep(20) print("making follow up request for existing id") response2 = client.responses.create( - model="bedrock/us.anthropic.claude-3-5-sonnet-20241022-v2:0", + model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", previous_response_id=response.id, input="ok, and what objects are in the image?" ) diff --git a/cosign.pub b/cosign.pub new file mode 100644 index 00000000000..2c2a555ab09 --- /dev/null +++ b/cosign.pub @@ -0,0 +1,4 @@ +-----BEGIN PUBLIC KEY----- +MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEKi4ivqGpE231OGH50PKbqy1Y1Kkb +POJC8+i2Wko82gBOUCe3M0Vw86H/4rhUhfoYEti4gdJ9wZbYmK0I2EE96g== +-----END PUBLIC KEY----- diff --git a/deploy/Dockerfile.ghcr_base b/deploy/Dockerfile.ghcr_base index 69b08a5893c..66e64e5b774 100644 --- a/deploy/Dockerfile.ghcr_base +++ b/deploy/Dockerfile.ghcr_base @@ -1,5 +1,5 @@ # Use the provided base image -FROM ghcr.io/berriai/litellm:main-latest +FROM ghcr.io/berriai/litellm:main-latest@sha256:7c311546c25e7bb6e8cafede9fcd3d0d622ac636b5c9418befaa32e85dfb0186 # Set the working directory to /app WORKDIR /app diff --git a/docker/Dockerfile.alpine b/docker/Dockerfile.alpine index ef2bb98db6e..bbc1ef4562b 100644 --- a/docker/Dockerfile.alpine +++ b/docker/Dockerfile.alpine @@ -1,8 +1,8 @@ # Base image for building -ARG LITELLM_BUILD_IMAGE=python:3.11-alpine +ARG LITELLM_BUILD_IMAGE=python:3.11-alpine@sha256:f07e2ace46f560f09a6eeec7b4913b80ee99546e749ef82342a419a326620856 # Runtime image -ARG LITELLM_RUNTIME_IMAGE=python:3.11-alpine +ARG LITELLM_RUNTIME_IMAGE=python:3.11-alpine@sha256:f07e2ace46f560f09a6eeec7b4913b80ee99546e749ef82342a419a326620856 # Builder stage FROM $LITELLM_BUILD_IMAGE AS builder @@ -13,8 +13,8 @@ WORKDIR /app # Install build dependencies RUN apk add --no-cache gcc python3-dev musl-dev -RUN pip install --upgrade pip && \ - pip install build +RUN pip install --upgrade pip==26.0.1 && \ + pip install build==1.4.2 # Copy the current directory contents into the container at /app COPY . . @@ -44,7 +44,7 @@ COPY --from=builder /app/dist/*.whl . COPY --from=builder /wheels/ /wheels/ # Install the built wheel using pip; again using a wildcard if it's the only file -RUN pip install *.whl /wheels/* --no-index --find-links=/wheels/ && rm -f *.whl && rm -rf /wheels +RUN pip install *.whl /wheels/* --no-index --find-links=/wheels/ --no-deps && rm -f *.whl && rm -rf /wheels # Convert Windows line endings to Unix for entrypoint scripts RUN sed -i 's/\r$//' docker/entrypoint.sh && chmod +x docker/entrypoint.sh diff --git a/docker/Dockerfile.custom_ui b/docker/Dockerfile.custom_ui index c1bd9a383fa..f836190a49a 100644 --- a/docker/Dockerfile.custom_ui +++ b/docker/Dockerfile.custom_ui @@ -1,4 +1,5 @@ # Use the provided base image +# NOTE: This is a dev/branch-specific tag. Update digest when the base image is rebuilt. FROM ghcr.io/berriai/litellm:litellm_fwd_server_root_path-dev # Set the working directory to /app @@ -18,8 +19,8 @@ RUN apt-get update && apt-get upgrade -y \ libxslt1.1 \ libgnutls30 \ libc6 && \ - apt-get install -y nodejs npm && \ - npm install -g npm@latest tar@7.5.11 glob@11.1.0 @isaacs/brace-expansion@5.0.1 minimatch@10.2.4 diff@8.0.3 && \ + apt-get install -y --no-install-recommends nodejs npm && \ + npm install -g npm@11.12.1 tar@7.5.11 glob@11.1.0 @isaacs/brace-expansion@5.0.1 minimatch@10.2.4 diff@8.0.3 && \ GLOBAL="$(npm root -g)" && \ find "$GLOBAL/npm" -type d -name "tar" -path "*/node_modules/tar" | while read d; do \ rm -rf "$d" && cp -rL "$GLOBAL/tar" "$d"; \ @@ -51,7 +52,7 @@ ENV UI_BASE_PATH="/prod/ui" # Build the UI with the specified UI_BASE_PATH WORKDIR /app/ui/litellm-dashboard -RUN npm install +RUN npm ci RUN UI_BASE_PATH=$UI_BASE_PATH npm run build # Create the destination directory diff --git a/docker/Dockerfile.database b/docker/Dockerfile.database index 3e1c55a75a8..36dd5a78741 100644 --- a/docker/Dockerfile.database +++ b/docker/Dockerfile.database @@ -1,8 +1,8 @@ # Base image for building -ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base +ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:a5a619c1793039dcf92f02178f37c94bb3d6001403716da59d6092dfe8d9b502 # Runtime image -ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base +ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:a5a619c1793039dcf92f02178f37c94bb3d6001403716da59d6092dfe8d9b502 # Builder stage FROM $LITELLM_BUILD_IMAGE AS builder @@ -21,7 +21,7 @@ RUN apk add --no-cache \ openssl \ openssl-dev -RUN python -m pip install build +RUN python -m pip install build==1.4.2 # Copy the current directory contents into the container at /app COPY . . @@ -50,7 +50,7 @@ USER root # Install runtime dependencies RUN apk add --no-cache bash openssl tzdata nodejs npm python3 py3-pip libsndfile && \ - npm install -g npm@latest tar@7.5.11 glob@11.1.0 @isaacs/brace-expansion@5.0.1 minimatch@10.2.4 diff@8.0.3 && \ + npm install -g npm@11.12.1 tar@7.5.11 glob@11.1.0 @isaacs/brace-expansion@5.0.1 minimatch@10.2.4 diff@8.0.3 && \ GLOBAL="$(npm root -g)" && \ find "$GLOBAL/npm" -type d -name "tar" -path "*/node_modules/tar" | while read d; do \ rm -rf "$d" && cp -rL "$GLOBAL/tar" "$d"; \ @@ -82,7 +82,7 @@ COPY --from=builder /app/dist/*.whl . COPY --from=builder /wheels/ /wheels/ # Install the built wheel using pip; again using a wildcard if it's the only file -RUN pip install *.whl /wheels/* --no-index --find-links=/wheels/ && rm -f *.whl && rm -rf /wheels +RUN pip install *.whl /wheels/* --no-index --find-links=/wheels/ --no-deps && rm -f *.whl && rm -rf /wheels # SECURITY FIX: nodejs-wheel-binaries (pip package used by Prisma) bundles a complete # npm with old vulnerable deps at /usr/lib/python3.*/site-packages/nodejs_wheel/. diff --git a/docker/Dockerfile.dev b/docker/Dockerfile.dev index e3e7ac0e0d6..e0ecff31ca4 100644 --- a/docker/Dockerfile.dev +++ b/docker/Dockerfile.dev @@ -1,8 +1,8 @@ # Base image for building -ARG LITELLM_BUILD_IMAGE=python:3.11-slim +ARG LITELLM_BUILD_IMAGE=python:3.13-slim@sha256:739e7213785e88c0f702dcdc12c0973afcbd606dbf021a589cab77d6b00b579d # Runtime image -ARG LITELLM_RUNTIME_IMAGE=python:3.11-slim +ARG LITELLM_RUNTIME_IMAGE=python:3.13-slim@sha256:739e7213785e88c0f702dcdc12c0973afcbd606dbf021a589cab77d6b00b579d # Builder stage FROM $LITELLM_BUILD_IMAGE AS builder @@ -19,7 +19,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ libssl-dev \ pkg-config \ && rm -rf /var/lib/apt/lists/* \ - && pip install --upgrade pip build + && pip install --upgrade pip==26.0.1 build==1.4.2 # Copy requirements first for better layer caching COPY requirements.txt . @@ -75,7 +75,7 @@ RUN apt-get update && apt-get upgrade -y \ nodejs \ npm \ && rm -rf /var/lib/apt/lists/* \ - && npm install -g npm@latest tar@7.5.11 glob@11.1.0 @isaacs/brace-expansion@5.0.1 minimatch@10.2.4 diff@8.0.3 \ + && npm install -g npm@11.12.1 tar@7.5.11 glob@11.1.0 @isaacs/brace-expansion@5.0.1 minimatch@10.2.4 diff@8.0.3 \ && GLOBAL="$(npm root -g)" \ && find "$GLOBAL/npm" -type d -name "tar" -path "*/node_modules/tar" | while read d; do \ rm -rf "$d" && cp -rL "$GLOBAL/tar" "$d"; \ @@ -109,7 +109,7 @@ COPY --from=builder /wheels/ /wheels/ COPY --from=builder /app/dist/*.whl . # Install all dependencies in one step with no-cache for smaller image -RUN pip install --no-cache-dir *.whl /wheels/* --no-index --find-links=/wheels/ && \ +RUN pip install --no-cache-dir *.whl /wheels/* --no-index --find-links=/wheels/ --no-deps && \ rm -f *.whl && \ rm -rf /wheels diff --git a/docker/Dockerfile.health_check b/docker/Dockerfile.health_check index de62e4bd729..fb9cc201d2f 100644 --- a/docker/Dockerfile.health_check +++ b/docker/Dockerfile.health_check @@ -1,4 +1,4 @@ -FROM python:3.11-slim +FROM python:3.13-slim@sha256:739e7213785e88c0f702dcdc12c0973afcbd606dbf021a589cab77d6b00b579d WORKDIR /app @@ -12,5 +12,13 @@ RUN pip install --no-cache-dir -r requirements.txt # Make script executable RUN chmod +x /app/health_check_client.py +# Run as non-root user +RUN adduser --disabled-password --gecos "" --uid 1001 healthcheck +USER healthcheck + +# Health check +HEALTHCHECK --interval=30s --timeout=5s --retries=3 \ + CMD python /app/health_check_client.py --help || exit 1 + # Set entrypoint ENTRYPOINT ["python", "/app/health_check_client.py"] diff --git a/docker/Dockerfile.non_root b/docker/Dockerfile.non_root index db3981fb7e7..1cf6c0dfd15 100644 --- a/docker/Dockerfile.non_root +++ b/docker/Dockerfile.non_root @@ -1,6 +1,6 @@ # Base images -ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base -ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base +ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:a5a619c1793039dcf92f02178f37c94bb3d6001403716da59d6092dfe8d9b502 +ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:a5a619c1793039dcf92f02178f37c94bb3d6001403716da59d6092dfe8d9b502 ARG PROXY_EXTRAS_SOURCE=published # ----------------- @@ -27,7 +27,7 @@ RUN for i in 1 2 3; do \ nodejs \ npm && break || sleep 5; \ done \ - && pip install --no-cache-dir --upgrade pip build + && pip install --no-cache-dir --upgrade pip==26.0.1 build==1.4.2 # Cache Python dependencies COPY requirements.txt . @@ -41,13 +41,19 @@ COPY . . ENV LITELLM_NON_ROOT=true # Build Admin UI using the upstream command order while keeping a single RUN layer +# NOTE: .npmrc (which has ignore-scripts=true and min-release-age=3d) is temporarily +# renamed during npm install/ci. This is safe because npm ci installs from +# package-lock.json with pinned versions + integrity hashes. RUN mkdir -p /var/lib/litellm/ui && \ - npm install -g npm@latest && npm cache clean --force && \ + mv /app/.npmrc /app/.npmrc.bak && \ + npm install -g npm@11.12.1 && npm cache clean --force && \ cd /app/ui/litellm-dashboard && \ if [ -f "/app/enterprise/enterprise_ui/enterprise_colors.json" ]; then \ cp /app/enterprise/enterprise_ui/enterprise_colors.json ./ui_colors.json; \ fi && \ - npm install --legacy-peer-deps && \ + mv .npmrc .npmrc.bak && \ + npm ci && \ + mv .npmrc.bak .npmrc && mv /app/.npmrc.bak /app/.npmrc && \ npm run build && \ cp -r /app/ui/litellm-dashboard/out/* /var/lib/litellm/ui/ && \ mkdir -p /var/lib/litellm/assets && \ @@ -106,7 +112,7 @@ RUN for i in 1 2 3; do \ apk add --no-cache python3 py3-pip bash openssl tzdata nodejs npm supervisor && break || sleep 5; \ done \ && apk upgrade --no-cache nodejs \ - && npm install -g npm@latest tar@7.5.11 glob@11.1.0 @isaacs/brace-expansion@5.0.1 minimatch@10.2.4 diff@8.0.3 \ + && npm install -g npm@11.12.1 tar@7.5.11 glob@11.1.0 @isaacs/brace-expansion@5.0.1 minimatch@10.2.4 diff@8.0.3 \ && GLOBAL="$(npm root -g)" \ && find "$GLOBAL/npm" -type d -name "tar" -path "*/node_modules/tar" | while read d; do \ rm -rf "$d" && cp -rL "$GLOBAL/tar" "$d"; \ diff --git a/docker/build_admin_ui.sh b/docker/build_admin_ui.sh index 5373ad0e3d9..efb2bac3535 100755 --- a/docker/build_admin_ui.sh +++ b/docker/build_admin_ui.sh @@ -40,11 +40,22 @@ else exit 1 fi fi -curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.38.0/install.sh | bash +NVM_VERSION="v0.40.4" +NVM_CHECKSUM="4b7412c49960c7d31e8df72da90c1fb5b8cccb419ac99537b737028d497aba4f" +NVM_SCRIPT=$(mktemp) +trap 'rm -f "$NVM_SCRIPT"' EXIT +curl -fsSL "https://raw.githubusercontent.com/nvm-sh/nvm/${NVM_VERSION}/install.sh" -o "$NVM_SCRIPT" +if command -v sha256sum &>/dev/null; then + echo "${NVM_CHECKSUM} ${NVM_SCRIPT}" | sha256sum -c - +elif command -v shasum &>/dev/null; then + echo "${NVM_CHECKSUM} ${NVM_SCRIPT}" | shasum -a 256 -c - +else + echo "No sha256 tool found; cannot verify nvm checksum"; exit 1 +fi || { echo "nvm checksum verification failed"; exit 1; } +bash "$NVM_SCRIPT" source ~/.nvm/nvm.sh nvm install v18.17.0 nvm use v18.17.0 -npm install -g npm # copy _enterprise.json from this directory to /ui/litellm-dashboard, and rename it to ui_colors.json cp enterprise/enterprise_ui/enterprise_colors.json ui/litellm-dashboard/ui_colors.json diff --git a/docker/build_from_pip/Dockerfile.build_from_pip b/docker/build_from_pip/Dockerfile.build_from_pip index 05236008ded..f26b993cce5 100644 --- a/docker/build_from_pip/Dockerfile.build_from_pip +++ b/docker/build_from_pip/Dockerfile.build_from_pip @@ -1,19 +1,17 @@ -FROM python:3.13-alpine +FROM python:3.13-slim@sha256:739e7213785e88c0f702dcdc12c0973afcbd606dbf021a589cab77d6b00b579d WORKDIR /app ENV HOME=/home/litellm ENV PATH="${HOME}/venv/bin:$PATH" -# Install runtime dependencies -# Note: Using Python 3.13 for compatibility with ddtrace and other packages -# rust and cargo are required for building ddtrace from source -# musl-dev and libffi-dev are needed for some Python packages on Alpine -RUN apk update && \ - apk add --no-cache gcc musl-dev libffi-dev openssl openssl-dev rust cargo +# Install runtime dependencies needed for building native extensions +RUN apt-get update && \ + apt-get install -y --no-install-recommends gcc libffi-dev && \ + rm -rf /var/lib/apt/lists/* RUN python -m venv ${HOME}/venv -RUN ${HOME}/venv/bin/pip install --no-cache-dir --upgrade pip +RUN ${HOME}/venv/bin/pip install --no-cache-dir --upgrade pip==26.0.1 COPY docker/build_from_pip/requirements.txt . RUN --mount=type=cache,target=${HOME}/.cache/pip \ @@ -28,4 +26,4 @@ RUN prisma generate EXPOSE 4000/tcp ENTRYPOINT ["litellm"] -CMD ["--port", "4000"] \ No newline at end of file +CMD ["--port", "4000"] diff --git a/docker/build_from_pip/requirements.txt b/docker/build_from_pip/requirements.txt index cc14b99727f..ec6cf2438db 100644 --- a/docker/build_from_pip/requirements.txt +++ b/docker/build_from_pip/requirements.txt @@ -1,6 +1,6 @@ -litellm[proxy]==1.67.4.dev1 # Specify the litellm version you want to use -prometheus_client -langfuse -prisma -openai==1.99.9 +litellm[proxy]==1.83.0 +prometheus_client==0.20.0 +langfuse==2.59.7 +prisma==0.11.0 +openai==2.24.0 ddtrace==2.19.0 # for advanced DD tracing / profiling diff --git a/docker/install_auto_router.sh b/docker/install_auto_router.sh index 794f9a2bbce..057baa19f59 100755 --- a/docker/install_auto_router.sh +++ b/docker/install_auto_router.sh @@ -1,3 +1,3 @@ #!/bin/bash pip install semantic_router==0.1.11 --no-deps -pip install aurelio-sdk==0.0.19 \ No newline at end of file +pip install aurelio-sdk==0.0.19 --no-deps \ No newline at end of file diff --git a/docs/my-website/blog/april_townhall_announcement/index.md b/docs/my-website/blog/april_townhall_announcement/index.md new file mode 100644 index 00000000000..b81f61bae62 --- /dev/null +++ b/docs/my-website/blog/april_townhall_announcement/index.md @@ -0,0 +1,35 @@ +--- +slug: april-townhall-announcement +title: "April Townhall: Friday, 10 April at 7:30 AM" +date: 2026-04-02T07:30:00 +authors: + - krrish +description: "Join the LiteLLM April townhall on Friday, 10 April at 7:30 AM." +tags: [announcement, townhall] +hide_table_of_contents: true +--- + +import Image from '@theme/IdealImage'; + +We are hosting our April townhall on **Friday, 10 April at 7:30 AM PST**. + + + +{/* truncate */} + +## Agenda + +- Product updates and roadmap progress +- Reliability and security updates +- Open Q&A with the team + +## Register + +Register here: [LiteLLM April Townhall Form](https://forms.gle/hvyVXwbFjzJQE7dEA) + +We will hold the townhall from **7:30 AM to 8:30 AM PST on Zoom**. + +For security, attendance is restricted to corporate emails. If you register with a non-corporate email, we will share the townhall slides and accompanying blog post after the event. diff --git a/docs/my-website/blog/ci_cd_v2_improvements/index.md b/docs/my-website/blog/ci_cd_v2_improvements/index.md new file mode 100644 index 00000000000..84f8f7bda6b --- /dev/null +++ b/docs/my-website/blog/ci_cd_v2_improvements/index.md @@ -0,0 +1,55 @@ +--- +slug: ci-cd-v2-improvements +title: "Announcing CI/CD v2 for LiteLLM" +date: 2026-03-30T21:30:00 +authors: + - krrish +description: "CI/CD v2 introduces isolated environments, stronger security gates, and safer release separation for LiteLLM." +tags: [engineering, ci-cd, security] +hide_table_of_contents: false +--- + +import Image from '@theme/IdealImage'; + +The CI/CD v2 is now live for LiteLLM. + + + +
+Building on the roadmap from our [security incident](https://docs.litellm.ai/blog/security-townhall-updates#roadmap), CI/CD v2 introduces isolated environments, stronger security gates, and safer release separation for LiteLLM. + +## What changed + +- Security scans and unit tests run in isolated environments. +- Validation and release are separated into different repositories, making it harder for an attacker to reach release credentials. +- Trusted Publishing for PyPI releases - this means no long-lived credentials are used to publish releases. +- Immutable Docker release tags - this means no tampering of Docker release tags after they are published [Learn more](https://docs.docker.com/docker-hub/repos/manage/hub-images/immutable-tags/). Note: work for GHCR docker releases is planned as well. + +## What's next + +Moving forward, we plan on: +- Adopting OpenSSF (this is a set of security criteria that projects should meet to demonstrate a strong security posture - [Learn more](https://baseline.openssf.org/versions/2026-02-19.html)) + - We've added Scorecard and Allstar to our Github + +- Adding SLSA Build Provenance to our CI/CD pipeline - this means we allow users to independently verify that a release came from us and prevent silent modifications of releases after they are published. + + +We hope that this will mean you can be confident that the releases you are using are safe and from us. + + +## The principle + +The new CI/CD pipeline reflects the principles, outlined below, and is designed to be more secure and reliable: + +- **Limit** what each package can access +- **Reduce** the number of sensitive environment variables +- **Avoid** compromised packages +- **Prevent** release tampering + + +## How to help: + +Help us plan April's stability sprint - https://github.com/BerriAI/litellm/issues/24825 \ No newline at end of file diff --git a/docs/my-website/blog/security_townhall_updates/index.md b/docs/my-website/blog/security_townhall_updates/index.md new file mode 100644 index 00000000000..b997de9c185 --- /dev/null +++ b/docs/my-website/blog/security_townhall_updates/index.md @@ -0,0 +1,190 @@ +--- +slug: security-townhall-updates +title: "Security Townhall Updates" +date: 2026-03-27T12:00:00 +authors: + - krrish + - ishaan-alt +description: "What happened, what we've done, and what comes next for LiteLLM's release and security processes." +tags: [security, incident-report] +hide_table_of_contents: false +--- + +import Image from '@theme/IdealImage'; + +Thank you to everyone who joined our town hall. + +We wanted to use that time to walk through what we know, what we've done so far, and how we're improving LiteLLM's release and security processes going forward. This post is a written version of that update. [Slides available here](https://drive.google.com/file/d/17hsSG7nk-OYL7VRCTbTa7McrWREtS9OO/view?usp=sharing) + +{/* truncate */} + +## What happened + +On March 24, 2026 at 10:39 UTC, LiteLLM v1.82.7 was pushed to PyPI. Version v1.82.8 was published soon after. Those packages were live for about 40 minutes before being quarantined by PyPI. By 16:00 UTC, the LiteLLM team had worked with PyPI to delete the affected packages. + +At this point, our understanding is that this was a supply-chain incident affecting those two published versions. + +## How did this happen? + +Our understanding is that the issue came from the [compromised Trivy security scanner](https://www.aquasec.com/blog/trivy-supply-chain-attack-what-you-need-to-know/) dependency in our CI/CD pipeline. + + + +There were three major contributing factors: + +### 1. Shared CI/CD environment + +At the time, everything was running on CircleCI, and all steps shared a common environment. That increased blast radius: if one component was compromised, it could potentially access credentials or context intended for other parts of the pipeline. + +### 2. Static credentials in environment variables + +Release credentials, including credentials for PyPI, GHCR, and Docker publishing, were available as static secrets in the environment. That meant a compromised step could access long-lived release credentials. + +### 3. Unpinned Trivy dependency + +In our security scanning component, we had an unpinned Trivy dependency. Our present understanding is that a compromised Trivy package ran during the scan, had access to environment variables, and enabled attackers to obtain those credentials. + +**In summary:** a compromised package in CI had access to secrets it should not have had, and those secrets were then used in the release path. + +## What we've already done + + +In the last 3 days, we've taken the following steps: + +### 1. Minimize Scope of Impact + +#### Prevented further key abuse + +We deleted or rotated all impacted or adjacent secret keys, including PyPI, GitHub, Docker, and related credentials. Out of an abundance of caution, we've also rotated LiteLLM maintainer accounts. + +#### Prevent branch attacks + +We removed roughly 6,000 open branches and added an auto-deletion policy for branches merged into `main`. This reduces the surface area for branch-based abuse. + +#### Pinned CI/CD dependencies + +We've pinned all Github Actions, and are working on pinning all CircleCI dependencies as well. + +#### Paused releases + +We've paused new releases until we've confirmed codebase security and put stronger release controls in place. + +### 2. Secured LiteLLM + +#### Forensic analysis + +We are working with Google's Mandiant cybersecurity team to confirm the source of the attack and verify the security of the codebase. We also confirmed that no malicious code was pushed to `main`. + +#### Confirm Application Security + +In parallel, we are working with whitehat hackers at [Veria Labs](https://verialabs.com/) to verify application security and review improvements to our CI/CD process. + +We have also confirmed that the last 20 LiteLLM releases contain no indicators of compromise, and that no unauthenticated attacks can be made against LiteLLM Proxy based on our current investigation. [Check Security Blog for release verification.](https://docs.litellm.ai/blog/security-update-march-2026#verified-safe-versions) + +#### Created a security working group + +We created a new security working group inside LiteLLM focused on: + +- Building threat models +- Auditing the build process and dependencies + +If you're interested in joining the security working group, please file an issue [here](https://github.com/BerriAI/litellm-security-wg). + +### 3. Improved CI/CD + +We've already begun making structural changes to how releases are built and published. These align with our goals (covered in the next section) around isolated environments, ephemeral credentials, and release auditing. + +## Roadmap + +We plan on following 4 guiding principles for our new CI/CD pipeline: + +1. **Limit** what each package can access +2. **Reduce** the number of sensitive environment variables +3. **Avoid** compromised packages +4. **Prevent** release tampering + + +### Isolated environments + + + +We are breaking our CI/CD into 4 semantic concepts: + +1. Unit tests +2. Integration tests +3. Security scans +4. Release publishing + +And will be running each of these in isolated environments. + +This will limit the damage that any single compromised component can cause. + +### Ephemeral credentials + +We plan to move to ephemeral credentials for PyPI (Trusted Publisher) and GHCR (Token-based authentication) releases. This will reduce the risk of credentials being leaked or compromised. + +We have already begun doing this: + +- PyPI Trusted Publisher on GitHub Actions [PR](https://github.com/BerriAI/litellm/pull/24654) +- GHCR Token-based authentication on GitHub Actions [PR](https://github.com/BerriAI/litellm/pull/24683) + +### Release auditing + +Our goal is to allow users to independently verify that a release came from us and prevent silent modifications of releases after they are published. + +This will ensure, your releases are safe, even when: +- Stolen PyPI/GHCR credentials are used to publish malicious releases +- Tampered registry artifacts are published +- Tag mutations are made after the release is published + +We believe that [Cosign](https://github.com/sigstore/cosign) is a good fit for this, and have already begun working on it [PR](https://github.com/BerriAI/litellm/pull/24683). + + +### Avoid Compromised Packages + +- Move to pinned, verified SHAs for packages and actions used in CI/CD, avoiding `latest` wherever possible. +- Add a cooldown period before upgrading to a new version of a package - allows more time to investigate and verify the new version. + +We've added zizmor to help us catch issues such as unpinned dependencies and credential leakage. [commit](https://github.com/BerriAI/litellm/commit/a671275f5c5b0e1fb1adacdf3b6ef779aaa5d56c). + + +## Frequently Asked Questions + +**Q: Did you observe any lateral movement into your corporate environment during this incident?** + +A: No. Our investigation to date, conducted in coordination with external security experts, has found no evidence of lateral movement into our internal corporate systems. The incident was isolated to the CI/CD pipeline and the release path for specific versions (v1.82.7 and v1.82.8). As a proactive measure, we have rotated all potentially impacted or adjacent secrets—including PyPI, GitHub, and Docker credentials—and updated maintainer account security to ensure continued isolation. + +**Q: Do you expect delays in future product releases due to these new security measures?** + +A: We are committed to balancing security with speed. While we have temporarily paused releases to implement stronger controls, we are moving quickly to automate our new security protocols. We are currently implementing isolated CI/CD environments, ephemeral credentials (via Trusted Publishers), and release auditing with Cosign. These improvements are designed to be integrated into our automated pipeline, allowing us to maintain a fast release cadence while ensuring every package is verified and secure. + +**Q: Were older packages impacted?** + +Our current findings show no indicators of compromise in the last 20 versions of LiteLLM. This was manually verified by our team and independently reviewed by Veria Labs. + +We have also published the verified versions for users to use. [Check Security Blog for release verification.](https://docs.litellm.ai/blog/security-update-march-2026#verified-safe-versions) + + + +## Questions & Support + +If you believe your systems may be affected, contact us immediately: + +- **Security:** security@berri.ai +- **Support:** support@berri.ai +- **Slack:** Reach out to the LiteLLM team directly [here](https://join.slack.com/t/litellmossslack/shared_invite/zt-3o7nkuyfr-p_kbNJj8taRfXGgQI1~YyA) + +## Hiring + +We are currently hiring for: + +- DevOps Engineer - to keep ci/cd secure and running smoothly +- Security Engineer - to keep the application secure + +If you're interest in joining, please apply [here](https://jobs.ashbyhq.com/litellm) \ No newline at end of file diff --git a/docs/my-website/blog/security_townhall_updates/shared_ci_cd_environment.png b/docs/my-website/blog/security_townhall_updates/shared_ci_cd_environment.png new file mode 100644 index 00000000000..29ec195b7fb Binary files /dev/null and b/docs/my-website/blog/security_townhall_updates/shared_ci_cd_environment.png differ diff --git a/docs/my-website/blog/security_update_march_2026/index.md b/docs/my-website/blog/security_update_march_2026/index.md new file mode 100644 index 00000000000..1c298fe372f --- /dev/null +++ b/docs/my-website/blog/security_update_march_2026/index.md @@ -0,0 +1,786 @@ +--- +slug: security-update-march-2026 +title: "Security Update: Suspected Supply Chain Incident" +date: 2026-03-24T14:00:00 +authors: + - krrish + - ishaan-alt +description: "As of 2:00 PM ET on March 24, 2026" +tags: [security, incident-report] +hide_table_of_contents: false +--- + +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; +import VersionVerificationTable from '@site/src/components/VersionVerificationTable'; + +> **Status:** Active investigation +> **Last updated:** March 27, 2026 + +> **Update (March 30):** A new **clean** version of LiteLLM is now available (v1.83.0). This was released by our new [CI/CD v2](https://docs.litellm.ai/blog/ci-cd-v2-improvements) pipeline which added isolated environments, stronger security gates, and safer release separation for LiteLLM. + +> **Update (March 27):** Review Townhall updates, including explanation of the incident, what we've done, and what comes next. [Learn more](https://docs.litellm.ai/blog/security-townhall-updates) + +> **Update (March 27):** Added [Verified safe versions](#verified-safe-versions) section with SHA-256 checksums for all audited PyPI and Docker releases. + +> **Update (March 26):** Added `checkmarx[.]zone` to [Indicators of compromise](#indicators-of-compromise-iocs) + +> **Update (March 25):** Added community-contributed scripts for scanning GitHub Actions and GitLab CI pipelines for the compromised versions. See [How to check if you are affected](#how-to-check-if-you-are-affected). s/o [@Zach Fury](https://www.linkedin.com/in/fryware/) for these scripts. + + +## TLDR; +- The compromised PyPI packages were **litellm==1.82.7** and **litellm==1.82.8**. Those packages were live on March 24, 2026 from 10:39 UTC for about 40 minutes before being quarantined by PyPI. +- We believe that the compromise originated from the [Trivy dependency](https://www.aquasec.com/blog/trivy-supply-chain-attack-what-you-need-to-know/) used in our CI/CD security scanning workflow. +- Customers running the official LiteLLM Proxy Docker image were not impacted. That deployment path pins dependencies in requirements.txt and does not rely on the compromised PyPI packages. +- ~~We have paused all new LiteLLM releases until we complete a broader supply-chain review and confirm the release path is safe.~~ **Updated:** We have now released a new **safe** version of LiteLLM (v1.83.0) by our new [CI/CD v2](https://docs.litellm.ai/blog/ci-cd-v2-improvements) pipeline which added isolated environments, stronger security gates, and safer release separation for LiteLLM. We have also verified the codebase is safe and no malicious code was pushed to `main`. + + +## Overview + +LiteLLM AI Gateway is investigating a suspected supply chain attack involving unauthorized PyPI package publishes. Current evidence suggests a maintainer's PyPI account may have been compromised and used to distribute malicious code. + +At this time, we believe this incident may be linked to the broader [Trivy security compromise](https://www.aquasec.com/blog/trivy-supply-chain-attack-what-you-need-to-know/), in which stolen credentials were reportedly used to gain unauthorized access to the LiteLLM publishing pipeline. + +This investigation is ongoing. Details below may change as we confirm additional findings. + +## Confirmed affected versions + +The following LiteLLM versions published to PyPI were impacted: + +- **v1.82.7**: contained a malicious payload in the LiteLLM AI Gateway `proxy_server.py` +- **v1.82.8**: contained `litellm_init.pth` and a malicious payload in the LiteLLM AI Gateway `proxy_server.py` + +If you installed or ran either of these versions, review the recommendations below immediately. + +Note: These versions have already been removed from PyPI. + +## What happened + +Initial evidence suggests the attacker bypassed official CI/CD workflows and uploaded malicious packages directly to PyPI. + +These compromised versions appear to have included a credential stealer designed to: + +- Harvest secrets by scanning for: + - environment variables + - SSH keys + - cloud provider credentials (AWS, GCP, Azure) + - Kubernetes tokens + - database passwords +- Encrypt and exfiltrate data via a `POST` request to `models.litellm.cloud`, which is **not** an official BerriAI / LiteLLM domain + +## Who is affected + +You may be affected if **any** of the following are true: + +- You installed or upgraded LiteLLM via `pip` on **March 24, 2026**, between **10:39 UTC and 16:00 UTC** +- You ran `pip install litellm` without pinning a version and received **v1.82.7** or **v1.82.8** +- You built a Docker image during this window that included `pip install litellm` without a pinned version +- A dependency in your project pulled in LiteLLM as a transitive, unpinned dependency + (for example through AI agent frameworks, MCP servers, or LLM orchestration tools) + +You are **not** affected if any of the following are true: + +**LiteLLM AI Gateway/Proxy users:** Customers running the official LiteLLM Proxy Docker image were not impacted. That deployment path pins dependencies in requirements.txt and does not rely on the compromised PyPI packages. + +- You are using **LiteLLM Cloud** +- You are using the official LiteLLM AI Gateway Docker image: `ghcr.io/berriai/litellm` +- You are on **v1.82.6 or earlier** and did not upgrade during the affected window +- You installed LiteLLM from source via the GitHub repository, which was **not** compromised + + +### How to check if you are affected + + + + +```bash +pip show litellm +``` + + + +Go to the proxy base url, and check the version of the installed LiteLLM. + +![Proxy version check](../../img/security_update_march_2026/proxy_version.png) + + + +Scans all repositories in a GitHub organization for workflow jobs that installed the compromised versions. + +**Requirements:** Python 3 and `requests` (`pip install requests`). + +**Setup:** + +```bash +export GITHUB_TOKEN="your-github-pat" +``` + +**Run:** + +```bash +python find_litellm_github.py +``` + +Set the `ORG` variable in the script to your GitHub organization name. + +Both scripts default to scanning jobs from **today**. Adjust the `WINDOW_START` and `WINDOW_END` constants to cover **March 24, 2026** (the incident date) if running on a different day. + +
+View full script (find_litellm_github.py) + +```python +#!/usr/bin/env python3 +""" +Scan all GitHub Actions jobs in a GitHub org that ran between +0800-1244 UTC today and identify any that installed litellm 1.82.7 or 1.82.8. + +Adjust WINDOW_START / WINDOW_END to cover March 24, 2026 if running later. +""" + +import io +import os +import re +import sys +import zipfile +from concurrent.futures import ThreadPoolExecutor, as_completed +from datetime import datetime, timezone + +import requests + +GITHUB_URL = "https://api.github.com" +ORG = "your-org" # <-- set to your GitHub organization +TOKEN = os.environ.get("GITHUB_TOKEN", "") + +TODAY = datetime.now(timezone.utc).date() +WINDOW_START = datetime(TODAY.year, TODAY.month, TODAY.day, 8, 0, 0, tzinfo=timezone.utc) +WINDOW_END = datetime(TODAY.year, TODAY.month, TODAY.day, 12, 44, 0, tzinfo=timezone.utc) + +TARGET_VERSIONS = {"1.82.7", "1.82.8"} +VERSION_PATTERN = re.compile(r"litellm[=\-](\d+\.\d+\.\d+)", re.IGNORECASE) + +SESSION = requests.Session() +SESSION.headers.update({ + "Authorization": f"Bearer {TOKEN}", + "Accept": "application/vnd.github+json", + "X-GitHub-Api-Version": "2022-11-28", +}) + + +def get_paginated(url, params=None): + params = dict(params or {}) + params.setdefault("per_page", 100) + page = 1 + while True: + params["page"] = page + resp = SESSION.get(url, params=params, timeout=30) + if resp.status_code == 404: + return + resp.raise_for_status() + data = resp.json() + if isinstance(data, dict): + items = next((v for v in data.values() if isinstance(v, list)), []) + else: + items = data + if not items: + break + yield from items + if len(items) < params["per_page"]: + break + page += 1 + + +def parse_ts(ts_str): + if not ts_str: + return None + return datetime.fromisoformat(ts_str.replace("Z", "+00:00")) + + +def get_repos(): + repos = [] + for r in get_paginated(f"{GITHUB_URL}/orgs/{ORG}/repos", {"type": "all"}): + repos.append({"id": r["id"], "name": r["name"], "full_name": r["full_name"]}) + return repos + + +def get_runs_in_window(repo_full_name): + created_filter = ( + f"{WINDOW_START.strftime('%Y-%m-%dT%H:%M:%SZ')}" + f"..{WINDOW_END.strftime('%Y-%m-%dT%H:%M:%SZ')}" + ) + url = f"{GITHUB_URL}/repos/{repo_full_name}/actions/runs" + runs = [] + for run in get_paginated(url, {"created": created_filter, "per_page": 100}): + ts = parse_ts(run.get("run_started_at") or run.get("created_at")) + if ts and WINDOW_START <= ts <= WINDOW_END: + runs.append(run) + return runs + + +def get_jobs_for_run(repo_full_name, run_id): + url = f"{GITHUB_URL}/repos/{repo_full_name}/actions/runs/{run_id}/jobs" + jobs = [] + for job in get_paginated(url, {"filter": "all"}): + ts = parse_ts(job.get("started_at")) + if ts and WINDOW_START <= ts <= WINDOW_END: + jobs.append(job) + return jobs + + +def fetch_job_log(repo_full_name, job_id): + url = f"{GITHUB_URL}/repos/{repo_full_name}/actions/jobs/{job_id}/logs" + resp = SESSION.get(url, timeout=60, allow_redirects=True) + if resp.status_code in (403, 404, 410): + return "" + resp.raise_for_status() + + content_type = resp.headers.get("Content-Type", "") + if "zip" in content_type or resp.content[:2] == b"PK": + try: + with zipfile.ZipFile(io.BytesIO(resp.content)) as zf: + parts = [] + for name in sorted(zf.namelist()): + with zf.open(name) as f: + parts.append(f.read().decode("utf-8", errors="replace")) + return "\n".join(parts) + except zipfile.BadZipFile: + pass + return resp.text + + +def check_job(repo_full_name, job): + job_id = job["id"] + job_name = job["name"] + run_id = job["run_id"] + started = job.get("started_at", "") + + log_text = fetch_job_log(repo_full_name, job_id) + if not log_text: + return None + + found_versions = set() + context_lines = [] + for line in log_text.splitlines(): + m = VERSION_PATTERN.search(line) + if m: + ver = m.group(1) + if ver in TARGET_VERSIONS: + found_versions.add(ver) + context_lines.append(line.strip()) + + if not found_versions: + return None + + return { + "repo": repo_full_name, + "run_id": run_id, + "job_id": job_id, + "job_name": job_name, + "started_at": started, + "versions": sorted(found_versions), + "context": context_lines[:10], + "job_url": job.get("html_url", f"https://github.com/{repo_full_name}/actions/runs/{run_id}"), + } + + +def main(): + if not TOKEN: + print("ERROR: Set GITHUB_TOKEN environment variable.", file=sys.stderr) + sys.exit(1) + + print(f"Time window : {WINDOW_START.isoformat()} -> {WINDOW_END.isoformat()}") + print(f"Hunting for : litellm {', '.join(sorted(TARGET_VERSIONS))}") + print() + + print(f"Fetching repositories for org '{ORG}'...") + repos = get_repos() + print(f" Found {len(repos)} repositories") + print() + + jobs_to_check = [] + + print("Scanning workflow runs for time window...") + for repo in repos: + full_name = repo["full_name"] + try: + runs = get_runs_in_window(full_name) + except requests.HTTPError as e: + print(f" WARN: {full_name} - {e}", file=sys.stderr) + continue + if not runs: + continue + print(f" {full_name}: {len(runs)} run(s) in window") + for run in runs: + try: + jobs = get_jobs_for_run(full_name, run["id"]) + except requests.HTTPError as e: + print(f" WARN: run {run['id']} - {e}", file=sys.stderr) + continue + for job in jobs: + jobs_to_check.append((full_name, job)) + + total = len(jobs_to_check) + print(f"\nFetching logs for {total} job(s)...") + print() + + hits = [] + with ThreadPoolExecutor(max_workers=8) as pool: + futures = { + pool.submit(check_job, full_name, job): (full_name, job["id"]) + for full_name, job in jobs_to_check + } + done = 0 + for future in as_completed(futures): + done += 1 + full_name, jid = futures[future] + try: + result = future.result() + except Exception as e: + print(f" ERROR {full_name} job {jid}: {e}", file=sys.stderr) + continue + if result: + hits.append(result) + print( + f" [{done}/{total}] {full_name} job {jid}" + + (f" *** HIT: litellm {result['versions']} ***" if result else ""), + flush=True, + ) + + print() + print("=" * 72) + print(f"RESULTS: {len(hits)} job(s) installed litellm {' or '.join(sorted(TARGET_VERSIONS))}") + print("=" * 72) + + if not hits: + print("No matches found.") + return + + for h in sorted(hits, key=lambda x: x["started_at"]): + print() + print(f" Repo : {h['repo']}") + print(f" Job : {h['job_name']} (#{h['job_id']})") + print(f" Run ID : {h['run_id']}") + print(f" Started : {h['started_at']}") + print(f" Versions : litellm {', '.join(h['versions'])}") + print(f" URL : {h['job_url']}") + print(f" Log lines :") + for line in h["context"]: + print(f" {line}") + + +if __name__ == "__main__": + main() +``` + +
+ + + + +Scans all projects in a GitLab group (including subgroups) for CI/CD jobs that installed the compromised versions. + +**Requirements:** Python 3 and `requests` (`pip install requests`). + +**Setup:** + +```bash +export GITLAB_TOKEN="your-gitlab-pat" +``` + +**Run:** + +```bash +python find_litellm_jobs.py +``` + +Set the `GROUP_NAME` variable in the script to your GitLab group name. + +Both scripts default to scanning jobs from **today**. Adjust the `WINDOW_START` and `WINDOW_END` constants to cover **March 24, 2026** (the incident date) if running on a different day. + +
+View full script (find_litellm_jobs.py) + +```python +#!/usr/bin/env python3 +""" +Scan all GitLab CI/CD jobs in a GitLab group that ran between +0800-1244 UTC today and identify any that installed litellm 1.82.7 or 1.82.8. + +Adjust WINDOW_START / WINDOW_END to cover March 24, 2026 if running later. +""" + +import os +import re +import sys +from concurrent.futures import ThreadPoolExecutor, as_completed +from datetime import datetime, timezone + +import requests + +GITLAB_URL = "https://gitlab.com" +GROUP_NAME = "YourGroup" # <-- set to your GitLab group name +TOKEN = os.environ.get("GITLAB_TOKEN", "") + +TODAY = datetime.now(timezone.utc).date() +WINDOW_START = datetime(TODAY.year, TODAY.month, TODAY.day, 8, 0, 0, tzinfo=timezone.utc) +WINDOW_END = datetime(TODAY.year, TODAY.month, TODAY.day, 12, 44, 0, tzinfo=timezone.utc) + +TARGET_VERSIONS = {"1.82.7", "1.82.8"} +VERSION_PATTERN = re.compile(r"litellm[=\-](\d+\.\d+\.\d+)", re.IGNORECASE) + +HEADERS = {"PRIVATE-TOKEN": TOKEN} +SESSION = requests.Session() +SESSION.headers.update(HEADERS) + + +def get_paginated(url, params=None): + params = dict(params or {}) + params.setdefault("per_page", 100) + page = 1 + while True: + params["page"] = page + resp = SESSION.get(url, params=params, timeout=30) + resp.raise_for_status() + data = resp.json() + if not data: + break + yield from data + if len(data) < params["per_page"]: + break + page += 1 + + +def get_group_id(group_name): + resp = SESSION.get(f"{GITLAB_URL}/api/v4/groups/{group_name}", timeout=30) + resp.raise_for_status() + return resp.json()["id"] + + +def get_all_projects(group_id): + projects = [] + for p in get_paginated( + f"{GITLAB_URL}/api/v4/groups/{group_id}/projects", + {"include_subgroups": "true", "archived": "false"}, + ): + projects.append({"id": p["id"], "name": p["path_with_namespace"]}) + return projects + + +def parse_ts(ts_str): + if not ts_str: + return None + ts_str = ts_str.replace("Z", "+00:00") + return datetime.fromisoformat(ts_str) + + +def jobs_in_window(project_id): + matching = [] + url = f"{GITLAB_URL}/api/v4/projects/{project_id}/jobs" + params = {"per_page": 100, "scope[]": ["success", "failed", "canceled", "running"]} + + page = 1 + while True: + params["page"] = page + resp = SESSION.get(url, params=params, timeout=30) + if resp.status_code == 403: + return matching + resp.raise_for_status() + jobs = resp.json() + if not jobs: + break + + stop_early = False + for job in jobs: + ts = parse_ts(job.get("started_at") or job.get("created_at")) + if ts is None: + continue + if ts > WINDOW_END: + continue + if ts < WINDOW_START: + stop_early = True + continue + matching.append(job) + + if stop_early or len(jobs) < 100: + break + page += 1 + + return matching + + +def fetch_trace(project_id, job_id): + url = f"{GITLAB_URL}/api/v4/projects/{project_id}/jobs/{job_id}/trace" + resp = SESSION.get(url, timeout=60) + if resp.status_code in (403, 404): + return "" + resp.raise_for_status() + return resp.text + + +def check_job(project_name, project_id, job): + job_id = job["id"] + job_name = job["name"] + ref = job.get("ref", "") + started = job.get("started_at", job.get("created_at", "")) + + trace = fetch_trace(project_id, job_id) + if not trace: + return None + + found_versions = set() + for match in VERSION_PATTERN.finditer(trace): + ver = match.group(1) + if ver in TARGET_VERSIONS: + found_versions.add(ver) + + if not found_versions: + return None + + context_lines = [] + for line in trace.splitlines(): + if VERSION_PATTERN.search(line): + ver_match = VERSION_PATTERN.search(line) + if ver_match and ver_match.group(1) in TARGET_VERSIONS: + context_lines.append(line.strip()) + + return { + "project": project_name, + "project_id": project_id, + "job_id": job_id, + "job_name": job_name, + "ref": ref, + "started_at": started, + "versions": sorted(found_versions), + "context": context_lines[:10], + "job_url": f"{GITLAB_URL}/{project_name}/-/jobs/{job_id}", + } + + +def main(): + if not TOKEN: + print("ERROR: Set GITLAB_TOKEN environment variable.", file=sys.stderr) + sys.exit(1) + + print(f"Time window : {WINDOW_START.isoformat()} -> {WINDOW_END.isoformat()}") + print(f"Hunting for : litellm {', '.join(sorted(TARGET_VERSIONS))}") + print() + + print(f"Resolving group '{GROUP_NAME}'...") + group_id = get_group_id(GROUP_NAME) + + print("Fetching projects...") + projects = get_all_projects(group_id) + print(f" Found {len(projects)} projects") + print() + + all_jobs_to_check = [] + + print("Scanning job listings for time window...") + for proj in projects: + try: + jobs = jobs_in_window(proj["id"]) + except requests.HTTPError as e: + print(f" WARN: {proj['name']} - {e}", file=sys.stderr) + continue + if jobs: + print(f" {proj['name']}: {len(jobs)} job(s) in window") + for j in jobs: + all_jobs_to_check.append((proj["name"], proj["id"], j)) + + total = len(all_jobs_to_check) + print(f"\nFetching traces for {total} job(s)...") + print() + + hits = [] + with ThreadPoolExecutor(max_workers=10) as pool: + futures = { + pool.submit(check_job, pname, pid, job): (pname, job["id"]) + for pname, pid, job in all_jobs_to_check + } + done = 0 + for future in as_completed(futures): + done += 1 + pname, jid = futures[future] + try: + result = future.result() + except Exception as e: + print(f" ERROR checking {pname} job {jid}: {e}", file=sys.stderr) + continue + if result: + hits.append(result) + print(f" [{done}/{total}] checked {pname} job {jid}" + + (f" *** HIT: litellm {result['versions']} ***" if result else ""), + flush=True) + + print() + print("=" * 72) + print(f"RESULTS: {len(hits)} job(s) installed litellm {' or '.join(sorted(TARGET_VERSIONS))}") + print("=" * 72) + + if not hits: + print("No matches found.") + return + + for h in sorted(hits, key=lambda x: x["started_at"]): + print() + print(f" Project : {h['project']}") + print(f" Job : {h['job_name']} (#{h['job_id']})") + print(f" Branch/tag: {h['ref']}") + print(f" Started : {h['started_at']}") + print(f" Versions : litellm {', '.join(h['versions'])}") + print(f" URL : {h['job_url']}") + print(f" Log lines :") + for line in h["context"]: + print(f" {line}") + + +if __name__ == "__main__": + main() +``` + +
+ + + + +*CI/CD scripts contributed by the community ([original gist](https://gist.github.com/fryz/93ec8d4898ffe5b5ac5706a208823ef3)). Review before running.* + + +## Indicators of compromise (IoCs) + +Review affected systems for the following indicators: + +- `litellm_init.pth` present in your `site-packages` +- Outbound traffic or requests to `models.litellm[.]cloud` + This domain is **not** affiliated with LiteLLM +- Outbound traffic or requests to `checkmarx[.]zone` + This domain is **not** affiliated with LiteLLM + + +## Immediate actions for affected users + +If you installed or ran **v1.82.7** or **v1.82.8**, take the following actions immediately. + +### 1. Rotate all secrets + +Treat any credentials present on the affected systems as compromised, including: + +- API keys +- Cloud access keys +- Database passwords +- SSH keys +- Kubernetes tokens +- Any secrets stored in environment variables or configuration files + +### 2. Inspect your filesystem + +Check your `site-packages` directory for a file named `litellm_init.pth`: + +```bash +find /usr/lib/python3.13/site-packages/ -name "litellm_init.pth" +``` + +If present: + +- remove it immediately +- investigate the host for further compromise +- preserve relevant artifacts if your security team is performing forensics + +### 3. Audit version history + +Review your: + +- Local environments +- CI/CD pipelines +- Docker builds +- Deployment logs + +Confirm whether **v1.82.7** or **v1.82.8** was installed anywhere. + +Pin LiteLLM to a known safe version such as **v1.82.6 or earlier**, or to a later verified release once announced. + + +## Response and remediation + +The LiteLLM AI Gateway team has already taken the following steps: + +- Removed compromised packages from PyPI +- Rotated maintainer credentials and established new authorized maintainers +- Engaged Google's Mandiant security team to assist with forensic analysis of the build and publishing chain + + +## Verified safe versions + +We have audited every LiteLLM release published between v1.78.0 and v1.82.6 across both PyPI and Docker. Each artifact was verified by: + +1. Downloading the published artifact and computing its SHA-256 digest +2. Scanning for the known [indicators of compromise](#indicators-of-compromise-iocs) (IOCs) +3. Comparing the artifact contents against the corresponding Git commit in the BerriAI/litellm repository + +**All versions listed below are confirmed clean.** + + + + + + + + + + + + + + + +## Questions and support + +If you believe your systems may be affected, contact us immediately: + +- **Security:** `security@berri.ai` +- **Support:** `support@berri.ai` +- **Slack:** Reach out to the LiteLLM team directly + +For real-time updates, follow [LiteLLM (YC W23) on X](https://x.com/LiteLLM). + diff --git a/docs/my-website/blog/vanta_compliance_recertification/index.md b/docs/my-website/blog/vanta_compliance_recertification/index.md new file mode 100644 index 00000000000..d05c113967f --- /dev/null +++ b/docs/my-website/blog/vanta_compliance_recertification/index.md @@ -0,0 +1,18 @@ +--- +slug: vanta-compliance-recertification +title: "LiteLLM + Vanta: SOC 2 Type 2 and ISO 27001 Recertification" +date: 2026-03-30T10:00:00 +authors: + - krrish +description: "LiteLLM is partnering with Vanta on SOC 2 Type 2 and ISO 27001 recertification and engaging independent auditors for verification." +tags: [security, compliance] +hide_table_of_contents: true +--- + +![LiteLLM x Vanta SOC-2 Recertification](/img/blog/vanta_soc2_recertification.png) + +We are partnering with [Vanta](https://www.vanta.com/) to recertify LiteLLM's compliance for SOC 2 Type 2 and ISO 27001. + +As part of this process, we are also identifying independent auditors to validate and verify our compliance posture. + +This is part of our commitment to being the most secure and transparent AI Gateway possible. diff --git a/docs/my-website/docs/anthropic_count_tokens.md b/docs/my-website/docs/anthropic_count_tokens.md index 5985516d69c..a62e46f156a 100644 --- a/docs/my-website/docs/anthropic_count_tokens.md +++ b/docs/my-website/docs/anthropic_count_tokens.md @@ -96,7 +96,7 @@ model_list: - model_name: claude-bedrock litellm_params: - model: bedrock/anthropic.claude-3-5-sonnet-20241022-v2:0 + model: bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0 aws_region_name: us-west-2 ``` diff --git a/docs/my-website/docs/completion/computer_use.md b/docs/my-website/docs/completion/computer_use.md index ed09a73b219..400f108f97e 100644 --- a/docs/my-website/docs/completion/computer_use.md +++ b/docs/my-website/docs/completion/computer_use.md @@ -80,7 +80,7 @@ model_list: api_key: os.environ/ANTHROPIC_API_KEY - model_name: claude-bedrock # Bedrock Anthropic model litellm_params: - model: bedrock/anthropic.claude-3-5-sonnet-20241022-v2:0 + model: bedrock/anthropic.claude-haiku-4-5-20251001:0 aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY aws_region_name: us-west-2 @@ -153,7 +153,7 @@ import litellm assert litellm.supports_computer_use(model="anthropic/claude-3-5-sonnet-latest") == True assert litellm.supports_computer_use(model="anthropic/claude-3-7-sonnet-20250219") == True -assert litellm.supports_computer_use(model="bedrock/anthropic.claude-3-5-sonnet-20241022-v2:0") == True +assert litellm.supports_computer_use(model="bedrock/anthropic.claude-haiku-4-5-20251001:0") == True assert litellm.supports_computer_use(model="vertex_ai/claude-3-5-sonnet") == True assert litellm.supports_computer_use(model="openai/gpt-4") == False ``` @@ -171,7 +171,7 @@ model_list: api_key: os.environ/ANTHROPIC_API_KEY - model_name: claude-bedrock # Bedrock Anthropic model litellm_params: - model: bedrock/anthropic.claude-3-5-sonnet-20241022-v2:0 + model: bedrock/anthropic.claude-haiku-4-5-20251001:0 aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY aws_region_name: us-west-2 diff --git a/docs/my-website/docs/completion/document_understanding.md b/docs/my-website/docs/completion/document_understanding.md index 172e0792801..f510a33f79a 100644 --- a/docs/my-website/docs/completion/document_understanding.md +++ b/docs/my-website/docs/completion/document_understanding.md @@ -32,7 +32,7 @@ os.environ["AWS_REGION_NAME"] = "" file_url = "https://www.w3.org/WAI/ER/tests/xhtml/testfiles/resources/pdf/dummy.pdf" # model -model = "bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0" +model = "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0" file_content = [ {"type": "text", "text": "What's this file about?"}, @@ -63,7 +63,7 @@ assert response is not None model_list: - model_name: bedrock-model litellm_params: - model: bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0 + model: bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0 aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY aws_region_name: os.environ/AWS_REGION_NAME @@ -122,7 +122,7 @@ encoded_file = base64.b64encode(file_data).decode("utf-8") base64_url = f"data:application/pdf;base64,{encoded_file}" # model -model = "bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0" +model = "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0" file_content = [ {"type": "text", "text": "What's this file about?"}, @@ -153,7 +153,7 @@ assert response is not None model_list: - model_name: bedrock-model litellm_params: - model: bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0 + model: bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0 aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY aws_region_name: os.environ/AWS_REGION_NAME @@ -210,7 +210,7 @@ os.environ["AWS_REGION_NAME"] = "" file_url = "https://www.w3.org/WAI/ER/tests/xhtml/testfiles/resources/pdf/dummy.pdf" # model -model = "bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0" +model = "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0" file_content = [ {"type": "text", "text": "What's this file about?"}, @@ -242,7 +242,7 @@ assert response is not None model_list: - model_name: bedrock-model litellm_params: - model: bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0 + model: bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0 aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY aws_region_name: os.environ/AWS_REGION_NAME @@ -350,10 +350,10 @@ curl -X POST 'http://0.0.0.0:4000/chat/completions' \ -Use `litellm.supports_pdf_input(model="bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0")` -> returns `True` if model can accept pdf input +Use `litellm.supports_pdf_input(model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0")` -> returns `True` if model can accept pdf input ```python -assert litellm.supports_pdf_input(model="bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0") == True +assert litellm.supports_pdf_input(model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0") == True ``` @@ -365,7 +365,7 @@ assert litellm.supports_pdf_input(model="bedrock/anthropic.claude-3-5-sonnet-202 model_list: - model_name: bedrock-model # model group name litellm_params: - model: bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0 + model: bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0 aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY aws_region_name: os.environ/AWS_REGION_NAME diff --git a/docs/my-website/docs/completion/provider_specific_params.md b/docs/my-website/docs/completion/provider_specific_params.md index 250b410c9c4..791153d2bc8 100644 --- a/docs/my-website/docs/completion/provider_specific_params.md +++ b/docs/my-website/docs/completion/provider_specific_params.md @@ -450,7 +450,7 @@ curl -X POST 'http://0.0.0.0:4000/chat/completions' \ import litellm response = litellm.completion( - model="bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0", + model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", messages=[{"role": "user", "content": "Hello!"}], requestMetadata={"cost_center": "engineering"} ) diff --git a/docs/my-website/docs/data_security.md b/docs/my-website/docs/data_security.md index 2c4b1247e2b..d93d17aa0de 100644 --- a/docs/my-website/docs/data_security.md +++ b/docs/my-website/docs/data_security.md @@ -128,8 +128,6 @@ We'll review all reports promptly. Note that we don't currently offer a bug boun Legal Entity Name: Berrie AI Incorporated -Company Phone Number: 7708783106 - Point of contact email address for security incidents: krrish@berri.ai Point of contact email address for general security-related questions: krrish@berri.ai diff --git a/docs/my-website/docs/debugging/local_debugging.md b/docs/my-website/docs/debugging/local_debugging.md index 8a56d6c34a0..53daa4e366b 100644 --- a/docs/my-website/docs/debugging/local_debugging.md +++ b/docs/my-website/docs/debugging/local_debugging.md @@ -67,6 +67,6 @@ response = completion("command-nightly", messages, logger_fn=my_custom_logging_f ## Still Seeing Issues? -Text us @ +17708783106 or Join the [Discord](https://discord.com/invite/wuPM9dRgDw). +Join the [Discord](https://discord.com/invite/wuPM9dRgDw). We promise to help you in `lite`ning speed ❤️ diff --git a/docs/my-website/docs/enterprise.md b/docs/my-website/docs/enterprise.md index 6dccf7ff4e7..a3fc9e38b6e 100644 --- a/docs/my-website/docs/enterprise.md +++ b/docs/my-website/docs/enterprise.md @@ -4,7 +4,7 @@ import Image from '@theme/IdealImage'; :::info - ✨ SSO is free for up to 5 users. After that, an enterprise license is required. [Get Started with Enterprise here](https://www.litellm.ai/enterprise) -- Who is Enterprise for? Companies giving access to 100+ users **OR** 10+ AI use-cases. If you're not sure, [get in touch with us](https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions) to discuss your needs. +- Who is Enterprise for? Companies giving access to 100+ users **OR** 10+ AI use-cases. If you're not sure, [get in touch with us](https://enterprise.litellm.ai/demo) to discuss your needs. ::: For companies that need SSO, user management and professional support for LiteLLM Proxy @@ -36,7 +36,7 @@ Manage Yourself - you can deploy our Docker Image or build a custom image from o ### What’s the cost of the Self-Managed Enterprise edition? -Self-Managed Enterprise deployments require our team to understand your exact needs. [Get in touch with us to learn more](https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions) +Self-Managed Enterprise deployments require our team to understand your exact needs. [Get in touch with us to learn more](https://enterprise.litellm.ai/demo) ### How does deployment with Enterprise License work? @@ -106,7 +106,7 @@ Professional Support can assist with LLM/Provider integrations, deployment, upgr Pricing is based on usage. We can figure out a price that works for your team, on the call. -[**Contact Us to learn more**](https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions) +[**Contact Us to learn more**](https://enterprise.litellm.ai/demo) diff --git a/docs/my-website/docs/fine_tuning.md b/docs/my-website/docs/fine_tuning.md index d0bd98a76f9..52e96f28688 100644 --- a/docs/my-website/docs/fine_tuning.md +++ b/docs/my-website/docs/fine_tuning.md @@ -6,7 +6,7 @@ import TabItem from '@theme/TabItem'; :::info -This is an Enterprise only endpoint [Get Started with Enterprise here](https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions) +This is an Enterprise only endpoint [Get Started with Enterprise here](https://enterprise.litellm.ai/demo) ::: diff --git a/docs/my-website/docs/index.md b/docs/my-website/docs/index.md index ca63c9e39ff..6410e052b05 100644 --- a/docs/my-website/docs/index.md +++ b/docs/my-website/docs/index.md @@ -103,7 +103,7 @@ os.environ["AWS_SECRET_ACCESS_KEY"] = "your-secret" os.environ["AWS_REGION_NAME"] = "us-east-1" response = completion( - model="bedrock/anthropic.claude-3-5-sonnet-20241022-v2:0", + model="bedrock/anthropic.claude-haiku-4-5-20251001:0", messages=[{"role": "user", "content": "Hello, how are you?"}] ) print(response.choices[0].message.content) diff --git a/docs/my-website/docs/learn/sdk_quickstart.md b/docs/my-website/docs/learn/sdk_quickstart.md index bdf7b63eb5d..0fb8c3f02a5 100644 --- a/docs/my-website/docs/learn/sdk_quickstart.md +++ b/docs/my-website/docs/learn/sdk_quickstart.md @@ -11,7 +11,7 @@ Use this path if you are integrating LiteLLM directly into application code. ## 1. Install LiteLLM ```bash -pip install litellm +pip install litellm==1.82.6 ``` ## 2. Set Provider Credentials diff --git a/docs/my-website/docs/migration.md b/docs/my-website/docs/migration.md index e1af07d4684..fda1155905d 100644 --- a/docs/my-website/docs/migration.md +++ b/docs/my-website/docs/migration.md @@ -31,5 +31,4 @@ When we have breaking changes (i.e. going from 1.x.x to 2.x.x), we will document **How can we communicate changes better?** Tell us - [Discord](https://discord.com/invite/wuPM9dRgDw) -- Email (krrish@berri.ai/ishaan@berri.ai) -- Text us (+17708783106) +- Email (support@berri.ai) diff --git a/docs/my-website/docs/observability/arize_integration.md b/docs/my-website/docs/observability/arize_integration.md index b3ccf98ea3b..4486fb2b718 100644 --- a/docs/my-website/docs/observability/arize_integration.md +++ b/docs/my-website/docs/observability/arize_integration.md @@ -194,5 +194,4 @@ print(response) - [Schedule Demo 👋](https://calendly.com/d/4mp-gd3-k5k/berriai-1-1-onboarding-litellm-hosted-version) - [Community Discord 💭](https://discord.gg/wuPM9dRgDw) -- Our numbers 📞 +1 (770) 8783-106 / +1 (412) 618-6238 - Our emails ✉️ ishaan@berri.ai / krrish@berri.ai diff --git a/docs/my-website/docs/observability/gcs_bucket_integration.md b/docs/my-website/docs/observability/gcs_bucket_integration.md index 69b956950e5..5f8d42508ae 100644 --- a/docs/my-website/docs/observability/gcs_bucket_integration.md +++ b/docs/my-website/docs/observability/gcs_bucket_integration.md @@ -6,7 +6,7 @@ Log LLM Logs to [Google Cloud Storage Buckets](https://cloud.google.com/storage? :::info -✨ This is an Enterprise only feature [Get Started with Enterprise here](https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions) +✨ This is an Enterprise only feature [Get Started with Enterprise here](https://enterprise.litellm.ai/demo) ::: @@ -79,5 +79,4 @@ curl --location 'http://0.0.0.0:4000/chat/completions' \ - [Schedule Demo 👋](https://calendly.com/d/4mp-gd3-k5k/berriai-1-1-onboarding-litellm-hosted-version) - [Community Discord 💭](https://discord.gg/wuPM9dRgDw) -- Our numbers 📞 +1 (770) 8783-106 / ‭+1 (412) 618-6238‬ - Our emails ✉️ ishaan@berri.ai / krrish@berri.ai diff --git a/docs/my-website/docs/observability/langfuse_integration.md b/docs/my-website/docs/observability/langfuse_integration.md index d3c5a44d481..32849ebdb92 100644 --- a/docs/my-website/docs/observability/langfuse_integration.md +++ b/docs/my-website/docs/observability/langfuse_integration.md @@ -342,5 +342,4 @@ Be aware that if you are continuing an existing trace, and you set `update_trace - [Schedule Demo 👋](https://calendly.com/d/4mp-gd3-k5k/berriai-1-1-onboarding-litellm-hosted-version) - [Community Discord 💭](https://discord.gg/wuPM9dRgDw) -- Our numbers 📞 +1 (770) 8783-106 / ‭+1 (412) 618-6238‬ - Our emails ✉️ ishaan@berri.ai / krrish@berri.ai diff --git a/docs/my-website/docs/observability/langsmith_integration.md b/docs/my-website/docs/observability/langsmith_integration.md index cada4122b20..bf867319de8 100644 --- a/docs/my-website/docs/observability/langsmith_integration.md +++ b/docs/my-website/docs/observability/langsmith_integration.md @@ -225,5 +225,4 @@ environment_variables: - [Schedule Demo 👋](https://calendly.com/d/4mp-gd3-k5k/berriai-1-1-onboarding-litellm-hosted-version) - [Community Discord 💭](https://discord.gg/wuPM9dRgDw) -- Our numbers 📞 +1 (770) 8783-106 / ‭+1 (412) 618-6238‬ - Our emails ✉️ ishaan@berri.ai / krrish@berri.ai diff --git a/docs/my-website/docs/observability/logfire_integration.md b/docs/my-website/docs/observability/logfire_integration.md index a1bd43a4bc4..00652c0f1a1 100644 --- a/docs/my-website/docs/observability/logfire_integration.md +++ b/docs/my-website/docs/observability/logfire_integration.md @@ -63,5 +63,4 @@ response = litellm.completion( - [Schedule Demo 👋](https://calendly.com/d/4mp-gd3-k5k/berriai-1-1-onboarding-litellm-hosted-version) - [Community Discord 💭](https://discord.gg/wuPM9dRgDw) -- Our numbers 📞 +1 (770) 8783-106 / ‭+1 (412) 618-6238‬ - Our emails ✉️ ishaan@berri.ai / krrish@berri.ai diff --git a/docs/my-website/docs/observability/lunary_integration.md b/docs/my-website/docs/observability/lunary_integration.md index 8d28321c807..4f1dca9d4af 100644 --- a/docs/my-website/docs/observability/lunary_integration.md +++ b/docs/my-website/docs/observability/lunary_integration.md @@ -176,5 +176,4 @@ You can find more details about the different ways of making requests to the Lit - [Schedule Demo 👋](https://calendly.com/d/4mp-gd3-k5k/berriai-1-1-onboarding-litellm-hosted-version) - [Community Discord 💭](https://discord.gg/wuPM9dRgDw) -- Our numbers 📞 +1 (770) 8783-106 / ‭+1 (412) 618-6238‬ - Our emails ✉️ ishaan@berri.ai / krrish@berri.ai diff --git a/docs/my-website/docs/observability/opik_integration.md b/docs/my-website/docs/observability/opik_integration.md index d28c46f0b4b..5b5cbe0f185 100644 --- a/docs/my-website/docs/observability/opik_integration.md +++ b/docs/my-website/docs/observability/opik_integration.md @@ -261,5 +261,4 @@ All requests made with this key will automatically be tracked in the "TestProjec - [Schedule Demo 👋](https://calendly.com/d/4mp-gd3-k5k/berriai-1-1-onboarding-litellm-hosted-version) - [Community Discord 💭](https://discord.gg/wuPM9dRgDw) -- Our numbers 📞 +1 (770) 8783-106 / ‭+1 (412) 618-6238‬ - Our emails ✉️ ishaan@berri.ai / krrish@berri.ai diff --git a/docs/my-website/docs/observability/phoenix_integration.md b/docs/my-website/docs/observability/phoenix_integration.md index 191f1f8044a..67ba815557e 100644 --- a/docs/my-website/docs/observability/phoenix_integration.md +++ b/docs/my-website/docs/observability/phoenix_integration.md @@ -127,5 +127,4 @@ Depending on which Phoenix Cloud version or deployment you are using, you should - [Schedule Demo 👋](https://calendly.com/d/4mp-gd3-k5k/berriai-1-1-onboarding-litellm-hosted-version) - [Community Discord 💭](https://discord.gg/wuPM9dRgDw) -- Our numbers 📞 +1 (770) 8783-106 / ‭+1 (412) 618-6238‬ - Our emails ✉️ ishaan@berri.ai / krrish@berri.ai diff --git a/docs/my-website/docs/observability/promptlayer_integration.md b/docs/my-website/docs/observability/promptlayer_integration.md index 7f62a316972..9462e755f74 100644 --- a/docs/my-website/docs/observability/promptlayer_integration.md +++ b/docs/my-website/docs/observability/promptlayer_integration.md @@ -84,5 +84,4 @@ Credits to [Nick Bradford](https://github.com/nsbradford), from [Vim-GPT](https: - [Schedule Demo 👋](https://calendly.com/d/4mp-gd3-k5k/berriai-1-1-onboarding-litellm-hosted-version) - [Community Discord 💭](https://discord.gg/wuPM9dRgDw) -- Our numbers 📞 +1 (770) 8783-106 / ‭+1 (412) 618-6238‬ - Our emails ✉️ ishaan@berri.ai / krrish@berri.ai \ No newline at end of file diff --git a/docs/my-website/docs/observability/slack_integration.md b/docs/my-website/docs/observability/slack_integration.md index 0ca7f616683..468d8b5945c 100644 --- a/docs/my-website/docs/observability/slack_integration.md +++ b/docs/my-website/docs/observability/slack_integration.md @@ -101,5 +101,4 @@ response = litellm.completion( - [Schedule Demo 👋](https://calendly.com/d/4mp-gd3-k5k/berriai-1-1-onboarding-litellm-hosted-version) - [Community Discord 💭](https://discord.gg/wuPM9dRgDw) -- Our numbers 📞 +1 (770) 8783-106 / ‭+1 (412) 618-6238‬ - Our emails ✉️ ishaan@berri.ai / krrish@berri.ai diff --git a/docs/my-website/docs/observability/sumologic_integration.md b/docs/my-website/docs/observability/sumologic_integration.md index c30ee94dad4..87e20ca57ed 100644 --- a/docs/my-website/docs/observability/sumologic_integration.md +++ b/docs/my-website/docs/observability/sumologic_integration.md @@ -328,5 +328,4 @@ If you get authentication errors, regenerate the HTTP Source URL in Sumo Logic: - [Schedule Demo 👋](https://calendly.com/d/4mp-gd3-k5k/berriai-1-1-onboarding-litellm-hosted-version) - [Community Discord 💭](https://discord.gg/wuPM9dRgDw) -- Our numbers 📞 +1 (770) 8783-106 / ‭+1 (412) 618-6238‬ - Our emails ✉️ ishaan@berri.ai / krrish@berri.ai diff --git a/docs/my-website/docs/observability/supabase_integration.md b/docs/my-website/docs/observability/supabase_integration.md index fd3f1c3d5a0..c29871d752f 100644 --- a/docs/my-website/docs/observability/supabase_integration.md +++ b/docs/my-website/docs/observability/supabase_integration.md @@ -105,5 +105,4 @@ litellm.modify_integration("supabase",{"table_name": "litellm_logs"}) - [Schedule Demo 👋](https://calendly.com/d/4mp-gd3-k5k/berriai-1-1-onboarding-litellm-hosted-version) - [Community Discord 💭](https://discord.gg/wuPM9dRgDw) -- Our numbers 📞 +1 (770) 8783-106 / ‭+1 (412) 618-6238‬ - Our emails ✉️ ishaan@berri.ai / krrish@berri.ai diff --git a/docs/my-website/docs/observability/wandb_integration.md b/docs/my-website/docs/observability/wandb_integration.md index 37057f43db5..3c1a3363957 100644 --- a/docs/my-website/docs/observability/wandb_integration.md +++ b/docs/my-website/docs/observability/wandb_integration.md @@ -57,5 +57,4 @@ response = litellm.completion( - [Schedule Demo 👋](https://calendly.com/d/4mp-gd3-k5k/berriai-1-1-onboarding-litellm-hosted-version) - [Community Discord 💭](https://discord.gg/wuPM9dRgDw) -- Our numbers 📞 +1 (770) 8783-106 / ‭+1 (412) 618-6238‬ - Our emails ✉️ ishaan@berri.ai / krrish@berri.ai \ No newline at end of file diff --git a/docs/my-website/docs/oidc.md b/docs/my-website/docs/oidc.md index b541329aa38..23eb431b7e0 100644 --- a/docs/my-website/docs/oidc.md +++ b/docs/my-website/docs/oidc.md @@ -268,7 +268,7 @@ Please contact us for paid enterprise support if you need help setting up Azure model list: - model_name: aws/claude-3-5-sonnet litellm_params: - model: bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0 + model: bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0 aws_region_name: "eu-central-1" aws_role_name: "arn:aws:iam::12345678:role/bedrock-role" aws_web_identity_token: "oidc/azure/api://123-456-789-9d04" diff --git a/docs/my-website/docs/providers/azure/azure_responses.md b/docs/my-website/docs/providers/azure/azure_responses.md index 34ec0e194f7..de085001ba1 100644 --- a/docs/my-website/docs/providers/azure/azure_responses.md +++ b/docs/my-website/docs/providers/azure/azure_responses.md @@ -246,7 +246,7 @@ You can also call the Azure Responses API via the `/chat/completions` endpoint. from litellm import completion import os -os.environ["AZURE_API_BASE"] = "https://my-endpoint-sweden-berri992.openai.azure.com/" +os.environ["AZURE_API_BASE"] = "https://my-azure-endpoint.openai.azure.com/" os.environ["AZURE_API_VERSION"] = "2023-03-15-preview" os.environ["AZURE_API_KEY"] = "my-api-key" @@ -268,7 +268,7 @@ model_list: litellm_params: model: azure/responses/my-custom-o1-pro api_key: os.environ/AZURE_API_KEY - api_base: https://my-endpoint-sweden-berri992.openai.azure.com/ + api_base: https://my-azure-endpoint.openai.azure.com/ api_version: 2023-03-15-preview ``` diff --git a/docs/my-website/docs/providers/bedrock.md b/docs/my-website/docs/providers/bedrock.md index bb07216a295..e5942cc1119 100644 --- a/docs/my-website/docs/providers/bedrock.md +++ b/docs/my-website/docs/providers/bedrock.md @@ -95,7 +95,7 @@ Here's how to call Bedrock with the LiteLLM Proxy Server model_list: - model_name: bedrock-claude-3-5-sonnet litellm_params: - model: bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0 + model: bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0 aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY aws_region_name: os.environ/AWS_REGION_NAME @@ -337,7 +337,7 @@ os.environ["AWS_SECRET_ACCESS_KEY"] = "" os.environ["AWS_REGION_NAME"] = "" response = completion( - model="bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0", + model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", messages=[{"role": "user", "content": "Hello, how are you?"}], requestMetadata={ "cost_center": "engineering", @@ -354,7 +354,7 @@ response = completion( model_list: - model_name: bedrock-claude-v1 litellm_params: - model: bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0 + model: bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0 requestMetadata: cost_center: "engineering" ``` @@ -1543,7 +1543,7 @@ file_data = response.content encoded_file = base64.b64encode(file_data).decode("utf-8") # model -model = "bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0" +model = "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0" image_content = [ {"type": "text", "text": "What's this file about?"}, @@ -1574,7 +1574,7 @@ assert response is not None model_list: - model_name: bedrock-model litellm_params: - model: bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0 + model: bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0 aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY aws_region_name: os.environ/AWS_REGION_NAME @@ -1631,7 +1631,7 @@ encoded_file = base64.b64encode(file_data).decode("utf-8") base64_url = f"data:application/pdf;base64,{encoded_file}" # model -model = "bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0" +model = "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0" image_content = [ {"type": "text", "text": "What's this file about?"}, @@ -1660,7 +1660,7 @@ assert response is not None model_list: - model_name: bedrock-model litellm_params: - model: bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0 + model: bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0 aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY aws_region_name: os.environ/AWS_REGION_NAME @@ -1941,7 +1941,7 @@ Here's an example of using a bedrock model with LiteLLM. For a complete list, re | GPT-OSS 120B | `completion(model='bedrock/converse/openai.gpt-oss-120b-1:0', messages=messages)` | `os.environ['AWS_ACCESS_KEY_ID']`, `os.environ['AWS_SECRET_ACCESS_KEY']`, `os.environ['AWS_REGION_NAME']` | | Deepseek R1 | `completion(model='bedrock/us.deepseek.r1-v1:0', messages=messages)` | `os.environ['AWS_ACCESS_KEY_ID']`, `os.environ['AWS_SECRET_ACCESS_KEY']` | | Anthropic Claude Sonnet 4.5 | `completion(model='bedrock/us.anthropic.claude-sonnet-4-5-20250929-v1:0', messages=messages)` | `os.environ['AWS_ACCESS_KEY_ID']`, `os.environ['AWS_SECRET_ACCESS_KEY']` | -| Anthropic Claude-V3.5 Sonnet | `completion(model='bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0', messages=messages)` | `os.environ['AWS_ACCESS_KEY_ID']`, `os.environ['AWS_SECRET_ACCESS_KEY']` | +| Anthropic Claude-V3.5 Sonnet | `completion(model='bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0', messages=messages)` | `os.environ['AWS_ACCESS_KEY_ID']`, `os.environ['AWS_SECRET_ACCESS_KEY']` | | Anthropic Claude-V3 sonnet | `completion(model='bedrock/anthropic.claude-3-sonnet-20240229-v1:0', messages=messages)` | `os.environ['AWS_ACCESS_KEY_ID']`, `os.environ['AWS_SECRET_ACCESS_KEY']` | | Anthropic Claude-V3 Haiku | `completion(model='bedrock/anthropic.claude-3-haiku-20240307-v1:0', messages=messages)` | `os.environ['AWS_ACCESS_KEY_ID']`, `os.environ['AWS_SECRET_ACCESS_KEY']` | | Anthropic Claude-V3 Opus | `completion(model='bedrock/anthropic.claude-3-opus-20240229-v1:0', messages=messages)` | `os.environ['AWS_ACCESS_KEY_ID']`, `os.environ['AWS_SECRET_ACCESS_KEY']` | @@ -2051,7 +2051,7 @@ os.environ["AWS_SECRET_ACCESS_KEY"] = "" os.environ["AWS_REGION_NAME"] = "" response = completion( - model="bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0", + model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", messages=[{"role": "user", "content": "Hello, how are you?"}], model_id="arn:aws:bedrock:eu-central-1:000000000000:application-inference-profile/a0a0a0a0a0a0", ) @@ -2068,7 +2068,7 @@ print(response) model_list: - model_name: anthropic-claude-3-5-sonnet litellm_params: - model: bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0 + model: bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0 # You have to set the ARN application inference profile in the model_id parameter model_id: arn:aws:bedrock:eu-central-1:000000000000:application-inference-profile/a0a0a0a0a0a0 ``` diff --git a/docs/my-website/docs/providers/gemini.md b/docs/my-website/docs/providers/gemini.md index c8c9114ea87..87ab5ad40f4 100644 --- a/docs/my-website/docs/providers/gemini.md +++ b/docs/my-website/docs/providers/gemini.md @@ -11,6 +11,7 @@ import TabItem from '@theme/TabItem'; | Provider Doc | [Google AI Studio ↗](https://aistudio.google.com/) | | API Endpoint for Provider | https://generativelanguage.googleapis.com | | Supported OpenAI Endpoints | `/chat/completions`, [`/embeddings`](../embedding/supported_embedding#gemini-ai-embedding-models), `/completions`, [`/videos`](./gemini/videos.md), [`/images/edits`](../image_edits.md) | +| Lyria (music) | [Cost map & notes](./gemini/music.md) | | Pass-through Endpoint | [Supported](../pass_through/google_ai_studio.md) |
diff --git a/docs/my-website/docs/providers/gemini/music.md b/docs/my-website/docs/providers/gemini/music.md new file mode 100644 index 00000000000..f3968f2db39 --- /dev/null +++ b/docs/my-website/docs/providers/gemini/music.md @@ -0,0 +1,28 @@ +# Gemini — Lyria (music generation) + +Google Lyria 3 preview models are listed in LiteLLM’s [model cost map](https://github.com/BerriAI/litellm/blob/main/model_prices_and_context_window.json) under the `gemini/` provider for metadata and spend tracking. + +| Property | Details | +|----------|---------| +| Provider route | `gemini/` | +| Models | `gemini/lyria-3-clip-preview`, `gemini/lyria-3-pro-preview` | +| Provider docs | [Gemini API pricing / models ↗](https://ai.google.dev/gemini-api/docs/pricing) | + +## Models + +| Model | Notes | +|-------|--------| +| `gemini/lyria-3-clip-preview` | ~30s clip; paid tier listed as per generated song in Google’s pricing | +| `gemini/lyria-3-pro-preview` | Full song; paid tier listed as per generated song in Google’s pricing | + +Input context limit in the cost map: **131,072** tokens. For modalities, limits, and features, see [Google’s Gemini API docs ↗](https://ai.google.dev/gemini-api/docs/models). + +## LiteLLM behavior + +- **Cost map**: Per-song paid pricing is stored as `output_cost_per_image` on those entries (flat per generation unit). Token-based completion cost may not reflect music billing until a dedicated path exists. +- **API calls**: Use the Gemini API as documented by Google. LiteLLM does not ship a separate `music_generation` helper like Veo’s `video_generation`. + +## Auth + +Same as other Gemini API models: `GEMINI_API_KEY` or `GOOGLE_API_KEY`. + diff --git a/docs/my-website/docs/providers/openai.md b/docs/my-website/docs/providers/openai.md index 2907cdf9f47..1f4a1687e8b 100644 --- a/docs/my-website/docs/providers/openai.md +++ b/docs/my-website/docs/providers/openai.md @@ -581,6 +581,90 @@ curl -X POST 'http://0.0.0.0:4000/chat/completions' \ See [OpenAI Reasoning documentation](https://platform.openai.com/docs/guides/reasoning) for more details on organization verification requirements. +### Multi-turn Conversations with `reasoning_items` + +For multi-turn conversations you need `reasoning_items`: structured blocks that include the `encrypted_content` token OpenAI uses to restore reasoning state on the next request. Pass `include=["reasoning.encrypted_content"]` on every call where you want that token returned. + + + + +```python showLineNumbers title="Non-streaming: round-trip reasoning_items" +import litellm + +messages = [{"role": "user", "content": "Solve this step by step: 2 + 2"}] + +# Turn 1 — get reasoning_items (encrypted_content); +response = litellm.completion( + model="openai/responses/gpt-5-mini", + messages=messages, + reasoning_effort="low", + include=["reasoning.encrypted_content"], +) + +assistant_msg = response.choices[0].message + +# Turn 2 — pass reasoning_items back; LiteLLM converts to the correct Responses API format +messages.append({ + "role": "assistant", + "content": assistant_msg.content, + "reasoning_items": assistant_msg.reasoning_items, +}) +messages.append({"role": "user", "content": "Now summarize your reasoning."}) + +response2 = litellm.completion( + model="openai/responses/gpt-5-mini", + messages=messages, + reasoning_effort="low", + include=["reasoning.encrypted_content"], +) +``` + + + + +`reasoning_items` (with `encrypted_content`) arrive on the final chunk when the full response completes: + +```python showLineNumbers title="Streaming: collect and round-trip reasoning_items" +import litellm + +messages = [{"role": "user", "content": "Solve this step by step: 2 + 2"}] + +collected_content = [] +collected_reasoning_items = [] + +stream = litellm.completion( + model="openai/responses/gpt-5-mini", + messages=messages, + stream=True, + reasoning_effort="low", + include=["reasoning.encrypted_content"], +) + +for chunk in stream: + delta = chunk.choices[0].delta + if delta.content: + collected_content.append(delta.content) + if getattr(delta, "reasoning_items", None): + collected_reasoning_items.extend(delta.reasoning_items) + +messages.append({ + "role": "assistant", + "content": "".join(collected_content), + "reasoning_items": collected_reasoning_items or None, +}) +messages.append({"role": "user", "content": "Continue the conversation."}) + +response2 = litellm.completion( + model="openai/responses/gpt-5-mini", + messages=messages, + reasoning_effort="low", + include=["reasoning.encrypted_content"], +) +``` + + + + ### Verbosity Control for GPT-5 Models The `verbosity` parameter controls the length and detail of responses from GPT-5 family models. It accepts three values: `"low"`, `"medium"`, or `"high"`. diff --git a/docs/my-website/docs/proxy/config_settings.md b/docs/my-website/docs/proxy/config_settings.md index 91445863f76..cc9090c2de6 100644 --- a/docs/my-website/docs/proxy/config_settings.md +++ b/docs/my-website/docs/proxy/config_settings.md @@ -206,7 +206,7 @@ router_settings: | Name | Type | Description | |------|------|-------------| -| completion_model | string | The default model to use for completions when `model` is not specified in the request | +| completion_model | string | The model to use for all completions, overriding any `model` specified in the request | | disable_spend_logs | boolean | If true, turns off writing each transaction to the database | | disable_spend_updates | boolean | If true, turns off all spend updates to the DB. Including key/user/team spend updates. | | disable_master_key_return | boolean | If true, turns off returning master key on UI. (checked on '/user/info' endpoint) | @@ -279,6 +279,33 @@ router_settings: | forward_client_headers_to_llm_api | boolean | If true, forwards the client headers (any `x-` headers and `anthropic-beta` headers) to the backend LLM call | | maximum_spend_logs_retention_period | str | Used to set the max retention time for spend logs in the db, after which they will be auto-purged | | maximum_spend_logs_retention_interval | str | Used to set the interval in which the spend log cleanup task should run in. | +| alert_type_config | dict | Configuration mapping alert types to their handler settings | +| always_include_stream_usage | boolean | If true, includes usage metrics in every streaming response chunk | +| auto_redirect_ui_login_to_sso | boolean | If true, automatically redirects UI login page to SSO provider | +| control_plane_url | string | URL of the control plane for cross-instance state sharing | +| custom_auth_run_common_checks | boolean | If true, runs standard auth validation checks alongside custom auth handlers | +| custom_ui_sso_sign_in_handler | string | Custom handler for SSO sign-in logic in the UI | +| database_connection_pool_timeout | integer | Database connection pool timeout in seconds | +| disable_error_logs | boolean | If true, suppresses error tracking and storage in the database | +| enable_health_check_routing | boolean | If true, enables health check-driven request routing to avoid unhealthy deployments | +| enable_mcp_registry | boolean | If true, enables access to the centralized MCP server registry | +| enforce_rbac | boolean | If true, enables role-based access control (RBAC) for all proxy operations | +| forward_llm_provider_auth_headers | boolean | If true, forwards provider-specific auth headers to LLM API calls | +| health_check_concurrency | integer | Maximum number of concurrent health check operations | +| health_check_staleness_threshold | integer | Maximum age in seconds for health check results before marking deployments as stale | +| maximum_spend_logs_cleanup_cron | string | Cron expression for scheduling automatic spend log cleanup tasks | +| mcp_client_side_auth_header_name | string | HTTP header name for client-side MCP server credentials | +| mcp_internal_ip_ranges | list | CIDR ranges considered internal for non-public MCP server access control | +| mcp_required_fields | list | List of required field names for MCP server submissions | +| mcp_trusted_proxy_ranges | list | CIDR ranges of proxies trusted to forward X-Forwarded-For headers for MCP | +| require_end_user_mcp_access_defined | boolean | If true, requires end users to have explicit MCP access permissions defined | +| role_permissions | list | List of role-based permission configurations | +| search_tools | list | List of search tool configurations for enabling web search capabilities | +| token_rate_limit_type | string | Rate limit counting method: "total", "output", or "input" tokens | +| use_redis_transaction_buffer | boolean | If true, buffers database transactions in Redis before writing | +| use_shared_health_check | boolean | If true, uses Redis-backed shared health check state across multiple proxy instances | +| user_header_mappings | dict | Map custom request headers to user IDs using lookup rules | +| user_header_name | string | HTTP header name to extract user identity from requests | ### router_settings - Reference @@ -367,6 +394,8 @@ router_settings: | ignore_invalid_deployments | boolean | If true, ignores invalid deployments. Default for proxy is True - to prevent invalid models from blocking other models from being loaded. | | search_tools | List[SearchToolTypedDict] | List of search tool configurations for Search API integration. Each tool specifies a search_tool_name and litellm_params with search_provider, api_key, api_base, etc. [Further Docs](../search/index.md) | | guardrail_list | List[GuardrailTypedDict] | List of guardrail configurations for guardrail load balancing. Enables load balancing across multiple guardrail deployments with the same guardrail_name. [Further Docs](./guardrails/guardrail_load_balancing.md) | +| enable_health_check_routing | boolean | If true, enables health check-driven deployment filtering to avoid routing requests to unhealthy deployments | +| health_check_staleness_threshold | integer | Maximum age in seconds for cached health check results before marking deployments as stale | ### environment variables - Reference @@ -804,6 +833,7 @@ router_settings: | LITELLM_OTEL_INTEGRATION_ENABLE_EVENTS | Optionally enable semantic logs for OTEL | LITELLM_OTEL_INTEGRATION_ENABLE_METRICS | Optionally enable emantic metrics for OTEL | LITELLM_ENABLE_PYROSCOPE | If true, enables Pyroscope CPU profiling. Profiles are sent to PYROSCOPE_SERVER_ADDRESS. Off by default. See [Pyroscope profiling](/proxy/pyroscope_profiling). +| LITELLM_ENABLE_TEAM_STALE_ALIAS_BYPASS | When `true`, if a team's legacy `model_aliases` entry maps a public model name to an internal `model_name__` deployment, pre-call handling can skip that rewrite when team-scoped sibling deployments exist for the public name—so load balancing / `order` apply across siblings. Default is `false` for backwards compatibility. See [Team-scoped models and legacy aliases](./load_balancing#team-scoped-models-and-legacy-model_aliases). When stale aliases are detected and this flag is off, the proxy may log a one-time warning. | PYROSCOPE_APP_NAME | Application name reported to Pyroscope. Required when LITELLM_ENABLE_PYROSCOPE is true. No default. | PYROSCOPE_SERVER_ADDRESS | Pyroscope server URL to send profiles to. Required when LITELLM_ENABLE_PYROSCOPE is true. No default. | PYROSCOPE_SAMPLE_RATE | Optional. Sample rate for Pyroscope profiling (integer). No default; when unset, the pyroscope-io library default is used. diff --git a/docs/my-website/docs/proxy/cost_tracking.md b/docs/my-website/docs/proxy/cost_tracking.md index f28eec287d4..f9e22cfecd3 100644 --- a/docs/my-website/docs/proxy/cost_tracking.md +++ b/docs/my-website/docs/proxy/cost_tracking.md @@ -163,7 +163,7 @@ Use this when you want non-proxy admins to access `/spend` endpoints :::info -Schedule a [meeting with us to get your Enterprise License](https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions) +Schedule a [meeting with us to get your Enterprise License](https://enterprise.litellm.ai/demo) ::: diff --git a/docs/my-website/docs/proxy/email.md b/docs/my-website/docs/proxy/email.md index 86a79cbcfc8..ba737c6782c 100644 --- a/docs/my-website/docs/proxy/email.md +++ b/docs/my-website/docs/proxy/email.md @@ -203,7 +203,7 @@ After regenerating the key, the user will receive an email notification with: :::info -Customizing Email Branding is an Enterprise Feature [Get in touch with us for a Free Trial](https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions) +Customizing Email Branding is an Enterprise Feature [Get in touch with us for a Free Trial](https://enterprise.litellm.ai/demo) ::: diff --git a/docs/my-website/docs/proxy/enterprise.md b/docs/my-website/docs/proxy/enterprise.md index 4b525837a20..09b103ca4a0 100644 --- a/docs/my-website/docs/proxy/enterprise.md +++ b/docs/my-website/docs/proxy/enterprise.md @@ -5,7 +5,7 @@ import TabItem from '@theme/TabItem'; # ✨ Enterprise Features :::tip -To get a license, get in touch with us [here](https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions) +To get a license, get in touch with us [here](https://enterprise.litellm.ai/demo) ::: diff --git a/docs/my-website/docs/proxy/guardrails/aporia_api.md b/docs/my-website/docs/proxy/guardrails/aporia_api.md index ceafc19a1cc..e6ff0d5fed3 100644 --- a/docs/my-website/docs/proxy/guardrails/aporia_api.md +++ b/docs/my-website/docs/proxy/guardrails/aporia_api.md @@ -139,7 +139,7 @@ curl -i http://localhost:4000/v1/chat/completions \ :::info -✨ This is an Enterprise only feature [Contact us to get a free trial](https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions) +✨ This is an Enterprise only feature [Contact us to get a free trial](https://enterprise.litellm.ai/demo) ::: diff --git a/docs/my-website/docs/proxy/guardrails/custom_guardrail.md b/docs/my-website/docs/proxy/guardrails/custom_guardrail.md index 638cae9c835..37579ad870d 100644 --- a/docs/my-website/docs/proxy/guardrails/custom_guardrail.md +++ b/docs/my-website/docs/proxy/guardrails/custom_guardrail.md @@ -417,7 +417,7 @@ curl -i -X POST http://localhost:4000/v1/chat/completions \ :::info -✨ This is an Enterprise only feature [Contact us to get a free trial](https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions) +✨ This is an Enterprise only feature [Contact us to get a free trial](https://enterprise.litellm.ai/demo) ::: diff --git a/docs/my-website/docs/proxy/guardrails/guardrails_ai.md b/docs/my-website/docs/proxy/guardrails/guardrails_ai.md index 55d586aee7b..19ae34014a4 100644 --- a/docs/my-website/docs/proxy/guardrails/guardrails_ai.md +++ b/docs/my-website/docs/proxy/guardrails/guardrails_ai.md @@ -59,7 +59,7 @@ curl -i http://localhost:4000/v1/chat/completions \ :::info -✨ This is an Enterprise only feature [Contact us to get a free trial](https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions) +✨ This is an Enterprise only feature [Contact us to get a free trial](https://enterprise.litellm.ai/demo) ::: diff --git a/docs/my-website/docs/proxy/health.md b/docs/my-website/docs/proxy/health.md index 2764a6f0d4f..530bea3d06b 100644 --- a/docs/my-website/docs/proxy/health.md +++ b/docs/my-website/docs/proxy/health.md @@ -314,6 +314,89 @@ general_settings: health_check_details: False ``` +## Health Check Driven Routing + +By default, background health checks are observability-only — they populate the `/health` endpoint but don't affect routing. Unhealthy deployments still receive traffic until request failures trigger cooldown. + +With `enable_health_check_routing: true`, the router **excludes deployments that failed their last background health check** before selecting a candidate. This gives you proactive failover instead of reactive cooldown. + +### How it works + +1. Background health checks run on their configured interval +2. After each cycle, every deployment is marked healthy or unhealthy +3. On each incoming request, the router filters out unhealthy deployments **before** cooldown filtering and load balancing +4. If all deployments are unhealthy, the filter is bypassed (safety net — never causes a total outage) +5. If health state is stale (older than `health_check_staleness_threshold`), it is ignored + +### Quick start + +```yaml +model_list: + - model_name: gpt-4 + litellm_params: + model: openai/gpt-4 + api_key: os.environ/OPENAI_API_KEY + - model_name: gpt-4 + litellm_params: + model: openai/gpt-4 + api_key: os.environ/OPENAI_API_KEY_SECONDARY + +general_settings: + background_health_checks: true + health_check_interval: 60 + enable_health_check_routing: true +``` + +### Configuration + +| Setting | Where | Default | Description | +|---------|-------|---------|-------------| +| `enable_health_check_routing` | `general_settings` | `false` | Enable/disable health-check-driven routing | +| `health_check_staleness_threshold` | `general_settings` | `health_check_interval * 2` | Seconds before health state is considered stale and ignored | +| `background_health_checks` | `general_settings` | `false` | Must be `true` for health check routing to work | +| `health_check_interval` | `general_settings` | `300` | Seconds between health check cycles | + +### Interaction with cooldown + +Health check filtering and cooldown are **additive**. A deployment can be excluded by either mechanism: + +- **Health check filter** — proactive, runs on the configured interval, excludes deployments that failed the last check +- **Cooldown** — reactive, triggered by request failures, excludes deployments for a short TTL + +This means request failures still provide fast detection between health check intervals. + +### Staleness + +If a health check result is older than `health_check_staleness_threshold`, it is ignored and the deployment is treated as eligible. This prevents stale data from permanently excluding a deployment if the health check loop stops or slows down. + +The default staleness threshold is `health_check_interval * 2`. For a 60s interval, health state expires after 120s. + +### Example: custom staleness + +```yaml +general_settings: + background_health_checks: true + health_check_interval: 30 + enable_health_check_routing: true + health_check_staleness_threshold: 90 # ignore health state older than 90s +``` + +### Debugging + +Run the proxy with `--detailed_debug` and look for: + +``` +health_check_routing_state_updated healthy=3 unhealthy=1 +``` + +This is logged after each health check cycle when routing state is written. + +If the safety net triggers (all deployments unhealthy), you'll see: + +``` +All deployments marked unhealthy by health checks, bypassing health filter +``` + ## Health Check Timeout The health check timeout is set in `litellm/constants.py` and defaults to 60 seconds. diff --git a/docs/my-website/docs/proxy/high_availability_control_plane.md b/docs/my-website/docs/proxy/high_availability_control_plane.md index 324fba3a180..4cc6d2952fb 100644 --- a/docs/my-website/docs/proxy/high_availability_control_plane.md +++ b/docs/my-website/docs/proxy/high_availability_control_plane.md @@ -42,9 +42,9 @@ The **High Availability Control Plane** takes a different approach: -The **control plane** is a LiteLLM instance that serves the admin UI and knows about all the workers. It does not proxy LLM requests, it is purely for administration. +The **control plane** is a LiteLLM instance that serves the admin UI and knows about all the workers. It is **not a router** — it does not proxy or route any LLM requests. It exists purely so admins can switch between workers and manage them from a single UI. -Each **worker** is a fully independent LiteLLM proxy that handles LLM requests for its region or team. Workers have their own users, keys, teams, and budgets. +Each **worker** is a fully independent LiteLLM proxy that handles LLM requests for its region or team. Workers have their own database, Redis, users, keys, teams, and budgets. No infrastructure is shared between workers. ## Setup diff --git a/docs/my-website/docs/proxy/ip_address.md b/docs/my-website/docs/proxy/ip_address.md index 8f042d9f183..4c469b81e0b 100644 --- a/docs/my-website/docs/proxy/ip_address.md +++ b/docs/my-website/docs/proxy/ip_address.md @@ -3,7 +3,7 @@ :::info -You need a LiteLLM License to unlock this feature. [Grab time](https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions), to get one today! +You need a LiteLLM License to unlock this feature. [Grab time](https://enterprise.litellm.ai/demo), to get one today! ::: diff --git a/docs/my-website/docs/proxy/jwt_key_mapping.md b/docs/my-website/docs/proxy/jwt_key_mapping.md new file mode 100644 index 00000000000..452bf821016 --- /dev/null +++ b/docs/my-website/docs/proxy/jwt_key_mapping.md @@ -0,0 +1,318 @@ +# JWT → Virtual Key Mapping + +:::info Enterprise + +JWT → Virtual Key Mapping is an Enterprise feature. + +[Get a free trial](https://enterprise.litellm.ai/demo) + +::: + +Map JWT tokens to LiteLLM virtual keys — so every JWT client gets the same granular controls as a virtual key: model restrictions, spend limits, rate limits, guardrails, and full spend tracking. + +**Why this matters:** Standard JWT auth maps a JWT to a *team*. That's a shared boundary — all clients under a team share the same limits. With JWT → Virtual Key Mapping, each individual JWT client (identified by a claim like `client_id`, `azp`, or `sub`) maps to its own virtual key. You get per-client accountability without issuing API keys to your users. + +**Common use case:** Your company uses SSO/OIDC. Developers use Claude Code with their identity tokens. You want to enforce per-developer model access and spend limits without giving each person a LiteLLM API key. + +--- + +## How It Works + +```mermaid +sequenceDiagram + participant Client as Client (Claude Code / API) + participant Proxy as LiteLLM Proxy + participant OIDC as OIDC Provider + participant DB as Mapping Table + + Client->>Proxy: POST /v1/chat/completions
Authorization: Bearer + + Proxy->>OIDC: Verify JWT signature + OIDC-->>Proxy: Valid ✓ + + Proxy->>Proxy: Extract claim
(e.g. client_id = "alice@corp.com") + + Proxy->>DB: Look up (claim_name, claim_value) + alt Mapping found + DB-->>Proxy: virtual_key_id = sk-abc123 + Proxy->>Proxy: Apply virtual key permissions
(models, budget, rate limits) + Proxy-->>Client: 200 OK + else No mapping — fallback_team_mapping + Proxy->>Proxy: Fall through to team JWT auth + Proxy-->>Client: 200 OK + else No mapping — reject + Proxy-->>Client: 403 Forbidden + else No mapping — auto_register + Proxy->>DB: Create new virtual key + mapping + Proxy-->>Client: 200 OK + end +``` + +--- + +## Setup + +### Prerequisites + +Complete [OIDC JWT Auth setup](./token_auth.md) first — you need `JWT_PUBLIC_KEY_URL` configured and `enable_jwt_auth: True` in your proxy config. + +### Step 1. Configure the JWT claim to map on + +Add `jwt_client_id_field` to your `litellm_jwtauth` config. This is the JWT claim LiteLLM uses as the lookup key: + +```yaml +general_settings: + master_key: sk-1234 + enable_jwt_auth: True + litellm_jwtauth: + team_id_jwt_field: "team_id" # existing team mapping (optional) + user_id_jwt_field: "sub" + jwt_client_id_field: "client_id" # 👈 claim used for key mapping + unregistered_jwt_client_behavior: "fallback_team_mapping" # see below +``` + +**`unregistered_jwt_client_behavior`** controls what happens when a JWT has no registered mapping: + +| Value | Behavior | +|-------|----------| +| `fallback_team_mapping` | Fall through to team-based JWT auth (default — backward compatible) | +| `reject` | Return 403 if no mapping found | +| `auto_register` | Auto-create a virtual key + mapping on first encounter | + +### Step 2. Register a JWT client → virtual key mapping + +**Option A: Single call (creates key + mapping atomically)** + +```bash +curl -X POST 'http://0.0.0.0:4000/jwt_client/new' \ + -H 'Authorization: Bearer ' \ + -H 'Content-Type: application/json' \ + -d '{ + "jwt_claim_name": "client_id", + "jwt_claim_value": "dev-alice", + "models": ["claude-sonnet-4-5", "claude-haiku-4-5"], + "max_budget": 50.0, + "budget_duration": "30d", + "rpm_limit": 100, + "tpm_limit": 50000, + "team_id": "engineering" + }' +``` + +Response includes the virtual key token (only shown on creation): + +```json +{ + "key": "sk-abc123...", + "key_id": "key_123", + "mapping_id": "mapping_456", + "jwt_claim_name": "client_id", + "jwt_claim_value": "dev-alice" +} +``` + +**Option B: Map an existing virtual key** + +```bash +curl -X POST 'http://0.0.0.0:4000/jwt/key/mapping/new' \ + -H 'Authorization: Bearer ' \ + -H 'Content-Type: application/json' \ + -d '{ + "jwt_claim_name": "client_id", + "jwt_claim_value": "dev-alice", + "virtual_key_id": "key_123" + }' +``` + +### Step 3. Test it + +```bash +# Get a JWT from your OIDC provider (must have client_id: dev-alice) +JWT_TOKEN="eyJhbG..." + +curl -X POST 'http://0.0.0.0:4000/v1/chat/completions' \ + -H "Authorization: Bearer $JWT_TOKEN" \ + -H 'Content-Type: application/json' \ + -d '{ + "model": "claude-sonnet-4-5", + "messages": [{"role": "user", "content": "Hello"}] + }' +``` + +The request is now tracked against `dev-alice`'s virtual key — spend, rate limits, and model access enforced per-client. + +--- + +## Walkthrough: Admin grants granular access, team uses Claude Code + +This is the full flow for an engineering team using Claude Code with company SSO. + +### Admin setup + +**1. Create a team for engineering** + +```bash +curl -X POST 'http://0.0.0.0:4000/team/new' \ + -H 'Authorization: Bearer ' \ + -H 'Content-Type: application/json' \ + -d '{ + "team_alias": "engineering", + "models": ["claude-sonnet-4-5", "claude-haiku-4-5"] + }' +``` + +**2. Register each developer with their own key and spend limit** + +```bash +# Alice — senior eng, higher budget +curl -X POST 'http://0.0.0.0:4000/jwt_client/new' \ + -H 'Authorization: Bearer ' \ + -H 'Content-Type: application/json' \ + -d '{ + "jwt_claim_name": "client_id", + "jwt_claim_value": "alice@corp.com", + "team_id": "engineering", + "models": ["claude-sonnet-4-5", "claude-haiku-4-5"], + "max_budget": 200.0, + "budget_duration": "30d", + "rpm_limit": 200 + }' + +# Bob — contractor, tighter limits +curl -X POST 'http://0.0.0.0:4000/jwt_client/new' \ + -H 'Authorization: Bearer ' \ + -H 'Content-Type: application/json' \ + -d '{ + "jwt_claim_name": "client_id", + "jwt_claim_value": "bob@contractor.com", + "team_id": "engineering", + "models": ["claude-haiku-4-5"], + "max_budget": 20.0, + "budget_duration": "30d", + "rpm_limit": 30 + }' +``` + +**3. Configure Claude Code to use the proxy** + +Set the proxy as the API base in your team's Claude Code config: + +```bash +# Point Claude Code at the LiteLLM proxy instead of Anthropic directly. +# ANTHROPIC_API_KEY here is the bearer token sent to the proxy — set it to +# the user's SSO/OIDC JWT token (obtained from your IdP at login). +export ANTHROPIC_API_KEY="" +export ANTHROPIC_BASE_URL="http://your-litellm-proxy:4000" +``` + +Or in `~/.claude/settings.json`: + +```json +{ + "env": { + "ANTHROPIC_BASE_URL": "http://your-litellm-proxy:4000" + } +} +``` + +**4. Developers authenticate with SSO as usual** + +When Alice runs Claude Code, her JWT (issued by your IdP with `client_id: alice@corp.com`) goes to the proxy. LiteLLM looks up the mapping, finds her virtual key, and enforces her specific limits — her $200/month budget, 200 RPM cap, and access to Sonnet and Haiku only. + +Bob's token maps to his own key — $20/month, Haiku only, 30 RPM. + +No API keys distributed. No shared limits. Full per-developer spend visibility in the LiteLLM dashboard. + +--- + +## Managing mappings + +**View a mapping + its key settings** + +```bash +curl 'http://0.0.0.0:4000/jwt/key/mapping/info?jwt_claim_name=client_id&jwt_claim_value=alice@corp.com' \ + -H 'Authorization: Bearer ' +``` + +Response includes the linked key's `models`, `max_budget`, `spend`, `rpm_limit`, `expires`, etc. + +**Update a mapping** + +```bash +curl -X POST 'http://0.0.0.0:4000/jwt_client/update' \ + -H 'Authorization: Bearer ' \ + -H 'Content-Type: application/json' \ + -d '{ + "jwt_claim_name": "client_id", + "jwt_claim_value": "alice@corp.com", + "max_budget": 300.0 + }' +``` + +**Delete a mapping** + +```bash +curl -X DELETE 'http://0.0.0.0:4000/jwt/key/mapping/delete' \ + -H 'Authorization: Bearer ' \ + -H 'Content-Type: application/json' \ + -d '{ + "jwt_claim_name": "client_id", + "jwt_claim_value": "alice@corp.com" + }' +``` + +--- + +## Security + +JWT-bound keys are locked down: + +- Non-admin users cannot call `/key/update`, `/key/delete`, or `/key/regenerate` on a JWT-bound key. These return 403. +- JWT-bound keys are automatically restricted to `llm_api_routes` — they can make LLM calls but cannot manage other keys or admin resources. +- Only proxy admins can create, update, or delete mappings. + +--- + +## Multi-IdP support + +If you have users across multiple identity providers that share the same claim values (e.g. two services both have `sub: user-123` from different issuers), set `issuer` when creating the mapping: + +```bash +curl -X POST 'http://0.0.0.0:4000/jwt_client/new' \ + -H 'Authorization: Bearer ' \ + -H 'Content-Type: application/json' \ + -d '{ + "jwt_claim_name": "sub", + "jwt_claim_value": "user-123", + "issuer": "https://idp-a.corp.com", + "models": ["claude-sonnet-4-5"], + "max_budget": 50.0 + }' +``` + +Mappings are unique per `(claim_name, claim_value, issuer)` — so `user-123` from IdP A and `user-123` from IdP B resolve to different virtual keys. + +--- + +## What JWT clients can and can't do vs virtual keys + +| Capability | Virtual Key | JWT → Key Mapping | +|---|---|---| +| Per-client model access | ✅ | ✅ | +| Per-client spend budget | ✅ | ✅ | +| Per-client RPM/TPM limits | ✅ | ✅ | +| Team membership | ✅ | ✅ | +| Spend tracking in dashboard | ✅ | ✅ | +| Guardrails | ✅ | ✅ | +| Key rotation | ✅ | ✅ (admin only) | +| Key expiry | ✅ | ✅ | +| No API key distribution needed | ❌ | ✅ | +| Works with existing SSO/OIDC | ❌ | ✅ | + +--- + +## Related + +- [OIDC JWT Auth](./token_auth.md) — base JWT auth setup required before using this feature +- [Virtual Keys](./virtual_keys.md) — full virtual key documentation +- [Access Control](./access_control.md) — model and team access control diff --git a/docs/my-website/docs/proxy/load_balancing.md b/docs/my-website/docs/proxy/load_balancing.md index 74b3e8a5117..93f3d944340 100644 --- a/docs/my-website/docs/proxy/load_balancing.md +++ b/docs/my-website/docs/proxy/load_balancing.md @@ -324,17 +324,58 @@ model_list: litellm_params: model: azure/gpt-4-fallback api_key: os.environ/AZURE_API_KEY_2 - order: 2 # 👈 Used when order=1 is unavailable - -router_settings: - enable_pre_call_checks: true # 👈 Required for 'order' to work + order: 2 # 👈 Used when order=1 fails ``` -:::important -The `order` parameter requires `enable_pre_call_checks: true` in `router_settings`. -::: +### How order-based fallback works -If `order=1` deployment is unavailable (e.g., rate-limited), the router falls back to `order=2` deployments. +When a request to an `order=1` deployment fails (connection error, 404, 429, etc.), the router automatically tries `order=2` deployments, then `order=3`, and so on. Each order level gets its own set of retries before escalating to the next. + +If all order levels are exhausted, the router falls through to any configured [model-level fallbacks](#fallbacks). + +```yaml +model_list: + - model_name: gpt-4 + litellm_params: + model: azure/gpt-4-primary + api_key: os.environ/AZURE_API_KEY + order: 1 + + - model_name: gpt-4 + litellm_params: + model: azure/gpt-4-secondary + api_key: os.environ/AZURE_API_KEY_2 + order: 2 + + - model_name: gpt-4-fallback + litellm_params: + model: openai/gpt-4 + api_key: os.environ/OPENAI_API_KEY + +router_settings: + fallbacks: + - gpt-4: + - gpt-4-fallback # tried after all order levels fail +``` + +The fallback chain for the above config: `order=1` → `order=2` → `gpt-4-fallback`. + +For 429 (rate limit) errors specifically, the failed deployment is immediately placed on cooldown. If all `order=1` deployments are on cooldown, the router picks `order=2` deployments directly during retries without waiting for the fallback path. + +### Team-scoped models and legacy `model_aliases` {#team-scoped-models-and-legacy-model_aliases} + +Team-scoped deployments are identified by `model_info.team_id` and `model_info.team_public_model_name`. Requests should use the **public** model name; the router resolves all sibling deployments (same public name, different `api_base` / `order`, etc.) for routing, failover, and deployment `order`. + +For router internals: when a `team_id` is in scope, optimized lookups key off `(team_id, team_public_model_name)`. If code passes an internal deployment id (e.g. `model_name__`) instead of the public name, routing still works via the usual deployment-name paths, but the team-specific fast path applies only to the public name. + +**Legacy teams:** Older proxy versions could persist `model_aliases` on the team row mapping a public name to a single internal deployment id (`model_name__`). On each request, pre-call logic may still rewrite `model` to that internal name **before** routing, which collapses to one deployment and can make newer sibling deployments unreachable. + +**Migration options:** + +1. **Recommended for upgrades:** Set environment variable `LITELLM_ENABLE_TEAM_STALE_ALIAS_BYPASS=true` so that when sibling team deployments exist for the public name, the stale alias rewrite is skipped and team-scoped routing (including `order` and failover) applies. See the [Environment variables](./config_settings) table in the proxy settings doc. +2. **Data cleanup:** Remove obsolete `model_aliases` entries for team public names from the team record in the database so only `team_public_model_name` + team model list drive access. + +If a stale alias is detected and the bypass is **not** enabled, the proxy may emit a **one-time** warning in logs explaining that sibling deployments may be unreachable until the flag is set or aliases are cleaned up. ### When You'll See Load Balancing in Action diff --git a/docs/my-website/docs/proxy/logging.md b/docs/my-website/docs/proxy/logging.md index 74a79776fbd..2f81498799a 100644 --- a/docs/my-website/docs/proxy/logging.md +++ b/docs/my-website/docs/proxy/logging.md @@ -1109,7 +1109,7 @@ Log LLM Logs to [Google Cloud Storage Buckets](https://cloud.google.com/storage? :::info -✨ This is an Enterprise only feature [Get Started with Enterprise here](https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions) +✨ This is an Enterprise only feature [Get Started with Enterprise here](https://enterprise.litellm.ai/demo) ::: @@ -1194,7 +1194,7 @@ Log LLM Logs/SpendLogs to [Google Cloud Storage PubSub Topic](https://cloud.goog :::info -✨ This is an Enterprise only feature [Get Started with Enterprise here](https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions) +✨ This is an Enterprise only feature [Get Started with Enterprise here](https://enterprise.litellm.ai/demo) ::: @@ -1497,7 +1497,7 @@ Log LLM Logs to [Azure Data Lake Storage](https://learn.microsoft.com/en-us/azur :::info -✨ This is an Enterprise only feature [Get Started with Enterprise here](https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions) +✨ This is an Enterprise only feature [Get Started with Enterprise here](https://enterprise.litellm.ai/demo) ::: diff --git a/docs/my-website/docs/proxy/multiple_admins.md b/docs/my-website/docs/proxy/multiple_admins.md index 8d39674df19..83d0c5863df 100644 --- a/docs/my-website/docs/proxy/multiple_admins.md +++ b/docs/my-website/docs/proxy/multiple_admins.md @@ -20,7 +20,7 @@ LiteLLM tracks changes to the following entities and actions: :::tip -Requires Enterprise License, Get in touch with us [here](https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions) +Requires Enterprise License, Get in touch with us [here](https://enterprise.litellm.ai/demo) ::: @@ -56,6 +56,40 @@ On the LiteLLM UI, navigate to Logs -> Audit Logs. You should see the audit log /> +## Export Audit Logs to External Storage + +You can export audit logs to an external storage backend (e.g. S3) in addition to storing them in the database. Logs are batched and uploaded asynchronously, so they do not block your proxy requests. + +### S3 Example + +Add `audit_log_callbacks` and `s3_callback_params` to your `litellm_settings`: + +```yaml +litellm_settings: + store_audit_logs: true + audit_log_callbacks: ["s3_v2"] + s3_callback_params: + s3_bucket_name: my-audit-logs-bucket # AWS Bucket Name + s3_region_name: us-west-2 # AWS Region + s3_aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID + s3_aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY + s3_path: litellm-audit # [OPTIONAL] prefix path in the bucket +``` + +Audit logs are written as JSON files to: + +``` +s3:///audit_logs//_.json +# or, when s3_path is set: +s3:////audit_logs//_.json +``` + +:::info + +Both `store_audit_logs: true` and `audit_log_callbacks` must be set. If `store_audit_logs` is not enabled, the callbacks will not fire. + +::: + ## Advanced ### Attribute Management changes to Users diff --git a/docs/my-website/docs/proxy/oauth2.md b/docs/my-website/docs/proxy/oauth2.md index 41c4110e447..204a01538cc 100644 --- a/docs/my-website/docs/proxy/oauth2.md +++ b/docs/my-website/docs/proxy/oauth2.md @@ -4,7 +4,7 @@ Use this if you want to use an Oauth2.0 token to make `/chat`, `/embeddings` req :::info -This is an Enterprise Feature - [get in touch with us if you want a free trial to test if this feature meets your needs]((https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions)) +This is an Enterprise Feature - [get in touch with us if you want a free trial to test if this feature meets your needs]((https://enterprise.litellm.ai/demo)) ::: diff --git a/docs/my-website/docs/proxy/pass_through.md b/docs/my-website/docs/proxy/pass_through.md index f47d7064140..700bfb0831d 100644 --- a/docs/my-website/docs/proxy/pass_through.md +++ b/docs/my-website/docs/proxy/pass_through.md @@ -422,6 +422,5 @@ general_settings: [Community Discord 💭](https://discord.gg/wuPM9dRgDw) -Our numbers 📞 +1 (770) 8783-106 / ‭+1 (412) 618-6238‬ Our emails ✉️ ishaan@berri.ai / krrish@berri.ai diff --git a/docs/my-website/docs/proxy/prod.md b/docs/my-website/docs/proxy/prod.md index 26cb484cbe9..d40a0343106 100644 --- a/docs/my-website/docs/proxy/prod.md +++ b/docs/my-website/docs/proxy/prod.md @@ -47,7 +47,7 @@ export LITELLM_LOG="ERROR" :::info -Need Help or want dedicated support ? Talk to a founder [here]: (https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions) +Need Help or want dedicated support ? Talk to a founder [here]: (https://enterprise.litellm.ai/demo) ::: diff --git a/docs/my-website/docs/proxy/public_routes.md b/docs/my-website/docs/proxy/public_routes.md index d5f3941751f..e53548349dc 100644 --- a/docs/my-website/docs/proxy/public_routes.md +++ b/docs/my-website/docs/proxy/public_routes.md @@ -5,7 +5,7 @@ import TabItem from '@theme/TabItem'; :::info -Requires a LiteLLM Enterprise License. [Get a free trial](https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions). +Requires a LiteLLM Enterprise License. [Get a free trial](https://enterprise.litellm.ai/demo). ::: diff --git a/docs/my-website/docs/proxy/tag_routing.md b/docs/my-website/docs/proxy/tag_routing.md index a1ae52e5e45..57d16a59b54 100644 --- a/docs/my-website/docs/proxy/tag_routing.md +++ b/docs/my-website/docs/proxy/tag_routing.md @@ -315,7 +315,7 @@ LiteLLM Proxy supports team-based tag routing, allowing you to associate specifi :::info -This is an enterprise feature, [Contact us here to get a free trial](https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions) +This is an enterprise feature, [Contact us here to get a free trial](https://enterprise.litellm.ai/demo) ::: diff --git a/docs/my-website/docs/proxy/team_logging.md b/docs/my-website/docs/proxy/team_logging.md index 2ad7e2a4a8e..3f57d0d6d8b 100644 --- a/docs/my-website/docs/proxy/team_logging.md +++ b/docs/my-website/docs/proxy/team_logging.md @@ -26,7 +26,7 @@ Team 3 -> Disabled Logging (for GDPR compliance) :::info -✨ This is an Enterprise only feature [Get Started with Enterprise here](https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions) +✨ This is an Enterprise only feature [Get Started with Enterprise here](https://enterprise.litellm.ai/demo) ::: @@ -248,7 +248,7 @@ Use the `/key/generate` or `/key/update` endpoints to add logging callbacks to a :::info -✨ This is an Enterprise only feature [Get Started with Enterprise here](https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions) +✨ This is an Enterprise only feature [Get Started with Enterprise here](https://enterprise.litellm.ai/demo) ::: diff --git a/docs/my-website/docs/proxy/team_model_add.md b/docs/my-website/docs/proxy/team_model_add.md index 7db59a3300e..4aa286f3e5f 100644 --- a/docs/my-website/docs/proxy/team_model_add.md +++ b/docs/my-website/docs/proxy/team_model_add.md @@ -5,7 +5,7 @@ This is an Enterprise feature. [Enterprise Pricing](https://www.litellm.ai/#pricing) -[Contact us here to get a free trial](https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions) +[Contact us here to get a free trial](https://enterprise.litellm.ai/demo) ::: @@ -26,7 +26,7 @@ curl -L -X POST 'http://0.0.0.0:4000/model/new' \ "model": "openai/gpt-4o", "custom_llm_provider": "openai", "api_key": "******ccb07", - "api_base": "https://my-endpoint-sweden-berri992.openai.azure.com", + "api_base": "https://my-azure-endpoint.openai.azure.com", "api_version": "2023-12-01-preview" }, "model_info": { diff --git a/docs/my-website/docs/proxy/token_auth.md b/docs/my-website/docs/proxy/token_auth.md index 7364ae0fb56..c287ab03644 100644 --- a/docs/my-website/docs/proxy/token_auth.md +++ b/docs/my-website/docs/proxy/token_auth.md @@ -11,11 +11,17 @@ Use JWT's to auth admins / users / projects into the proxy. [Enterprise Pricing](https://www.litellm.ai/#pricing) -[Contact us here to get a free trial](https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions) +[Contact us here to get a free trial](https://enterprise.litellm.ai/demo) ::: +:::tip JWT → Virtual Key Mapping + +Want per-user model restrictions, spend limits, and rate limits without distributing API keys? See **[JWT → Virtual Key Mapping](./jwt_key_mapping.md)** — enterprise-grade granular access control for JWT-authenticated users (e.g. Claude Code + SSO). + +::: + ## Usage ### Step 1. Setup Proxy diff --git a/docs/my-website/docs/proxy_server.md b/docs/my-website/docs/proxy_server.md index e23d64e443b..7b6f15a604b 100644 --- a/docs/my-website/docs/proxy_server.md +++ b/docs/my-website/docs/proxy_server.md @@ -813,5 +813,4 @@ Thread Stats Avg Stdev Max +/- Stdev - [Schedule Demo 👋](https://calendly.com/d/4mp-gd3-k5k/berriai-1-1-onboarding-litellm-hosted-version) - [Community Discord 💭](https://discord.gg/wuPM9dRgDw) -- Our numbers 📞 +1 (770) 8783-106 / ‭+1 (412) 618-6238‬ - Our emails ✉️ ishaan@berri.ai / krrish@berri.ai diff --git a/docs/my-website/docs/realtime.md b/docs/my-website/docs/realtime.md index 15a838bb7d7..08f1e47fa73 100644 --- a/docs/my-website/docs/realtime.md +++ b/docs/my-website/docs/realtime.md @@ -82,7 +82,7 @@ Run this script using node - `node test.js` const WebSocket = require("ws"); const url = "ws://0.0.0.0:4000/v1/realtime?model=openai-gpt-4o-realtime-audio"; -// const url = "wss://my-endpoint-sweden-berri992.openai.azure.com/openai/realtime?api-version=2024-10-01-preview&deployment=gpt-4o-realtime-preview"; +// const url = "wss://my-azure-endpoint.openai.azure.com/openai/realtime?api-version=2024-10-01-preview&deployment=gpt-4o-realtime-preview"; const ws = new WebSocket(url, { headers: { "api-key": `sk-1234`, diff --git a/docs/my-website/docs/reasoning_content.md b/docs/my-website/docs/reasoning_content.md index 3f7b76dab65..6e6a30cdb49 100644 --- a/docs/my-website/docs/reasoning_content.md +++ b/docs/my-website/docs/reasoning_content.md @@ -13,6 +13,7 @@ Supported Providers: - Deepseek (`deepseek/`) - Anthropic API (`anthropic/`) - Bedrock (Anthropic + Deepseek + GPT-OSS) (`bedrock/`) +- OpenAI Responses API (`openai/responses/`) - Vertex AI (Anthropic) (`vertexai/`) - OpenRouter (`openrouter/`) - XAI (`xai/`) diff --git a/docs/my-website/docs/routing.md b/docs/my-website/docs/routing.md index 67e7f681147..5aa655ae212 100644 --- a/docs/my-website/docs/routing.md +++ b/docs/my-website/docs/routing.md @@ -842,6 +842,8 @@ Traffic mirroring allows you to "mimic" production traffic to a secondary (silen Set `order` in `litellm_params` to prioritize deployments. Lower values = higher priority. When multiple deployments share the same `order`, the routing strategy picks among them. +When a request to an `order=1` deployment fails (connection error, 404, 429, etc.), the router automatically tries `order=2` deployments, then `order=3`, and so on. Each order level gets its own set of retries before escalating to the next. If all order levels are exhausted, the router falls through to any configured [fallbacks](#fallbacks). + @@ -862,18 +864,14 @@ model_list = [ "litellm_params": { "model": "azure/gpt-4-fallback", "api_key": os.getenv("AZURE_API_KEY_2"), - "order": 2, # 👈 Used when order=1 is unavailable + "order": 2, # 👈 Tried when order=1 fails }, }, ] -router = Router(model_list=model_list, enable_pre_call_checks=True) # 👈 Required for 'order' to work +router = Router(model_list=model_list) ``` -:::important -The `order` parameter requires `enable_pre_call_checks=True` to be set on the Router. -::: - @@ -889,10 +887,7 @@ model_list: litellm_params: model: azure/gpt-4-fallback api_key: os.environ/AZURE_API_KEY_2 - order: 2 # 👈 Used when order=1 is unavailable - -router_settings: - enable_pre_call_checks: true # 👈 Required for 'order' to work + order: 2 # 👈 Tried when order=1 fails ``` diff --git a/docs/my-website/docs/secret.md b/docs/my-website/docs/secret.md index c5c80311475..57f576fd56a 100644 --- a/docs/my-website/docs/secret.md +++ b/docs/my-website/docs/secret.md @@ -6,7 +6,7 @@ [Enterprise Pricing](https://www.litellm.ai/#pricing) -[Contact us here to get a free trial](https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions) +[Contact us here to get a free trial](https://enterprise.litellm.ai/demo) ::: diff --git a/docs/my-website/docs/secret_managers/aws_kms.md b/docs/my-website/docs/secret_managers/aws_kms.md index 7f69d91fe87..806223a2539 100644 --- a/docs/my-website/docs/secret_managers/aws_kms.md +++ b/docs/my-website/docs/secret_managers/aws_kms.md @@ -6,7 +6,7 @@ [Enterprise Pricing](https://www.litellm.ai/#pricing) -[Contact us here to get a free trial](https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions) +[Contact us here to get a free trial](https://enterprise.litellm.ai/demo) ::: diff --git a/docs/my-website/docs/secret_managers/aws_secret_manager.md b/docs/my-website/docs/secret_managers/aws_secret_manager.md index c49797a15dd..a7e24ea69ae 100644 --- a/docs/my-website/docs/secret_managers/aws_secret_manager.md +++ b/docs/my-website/docs/secret_managers/aws_secret_manager.md @@ -9,7 +9,7 @@ import TabItem from '@theme/TabItem'; [Enterprise Pricing](https://www.litellm.ai/#pricing) -[Contact us here to get a free trial](https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions) +[Contact us here to get a free trial](https://enterprise.litellm.ai/demo) ::: diff --git a/docs/my-website/docs/secret_managers/azure_key_vault.md b/docs/my-website/docs/secret_managers/azure_key_vault.md index 81aeaa32159..4ea53d2ea9e 100644 --- a/docs/my-website/docs/secret_managers/azure_key_vault.md +++ b/docs/my-website/docs/secret_managers/azure_key_vault.md @@ -6,7 +6,7 @@ [Enterprise Pricing](https://www.litellm.ai/#pricing) -[Contact us here to get a free trial](https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions) +[Contact us here to get a free trial](https://enterprise.litellm.ai/demo) ::: diff --git a/docs/my-website/docs/secret_managers/cyberark.md b/docs/my-website/docs/secret_managers/cyberark.md index 0a17c0afc30..cd7c0ea5d25 100644 --- a/docs/my-website/docs/secret_managers/cyberark.md +++ b/docs/my-website/docs/secret_managers/cyberark.md @@ -8,7 +8,7 @@ import Image from '@theme/IdealImage'; [Enterprise Pricing](https://www.litellm.ai/#pricing) -[Contact us here to get a free trial](https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions) +[Contact us here to get a free trial](https://enterprise.litellm.ai/demo) ::: diff --git a/docs/my-website/docs/secret_managers/google_kms.md b/docs/my-website/docs/secret_managers/google_kms.md index 31fd6195bdb..152ecbaae80 100644 --- a/docs/my-website/docs/secret_managers/google_kms.md +++ b/docs/my-website/docs/secret_managers/google_kms.md @@ -6,7 +6,7 @@ [Enterprise Pricing](https://www.litellm.ai/#pricing) -[Contact us here to get a free trial](https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions) +[Contact us here to get a free trial](https://enterprise.litellm.ai/demo) ::: diff --git a/docs/my-website/docs/secret_managers/google_secret_manager.md b/docs/my-website/docs/secret_managers/google_secret_manager.md index 81878b7e398..f3e7367e8a4 100644 --- a/docs/my-website/docs/secret_managers/google_secret_manager.md +++ b/docs/my-website/docs/secret_managers/google_secret_manager.md @@ -6,7 +6,7 @@ [Enterprise Pricing](https://www.litellm.ai/#pricing) -[Contact us here to get a free trial](https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions) +[Contact us here to get a free trial](https://enterprise.litellm.ai/demo) ::: diff --git a/docs/my-website/docs/secret_managers/hashicorp_vault.md b/docs/my-website/docs/secret_managers/hashicorp_vault.md index 52d9b556200..11e25e88a7d 100644 --- a/docs/my-website/docs/secret_managers/hashicorp_vault.md +++ b/docs/my-website/docs/secret_managers/hashicorp_vault.md @@ -8,7 +8,7 @@ import Image from '@theme/IdealImage'; [Enterprise Pricing](https://www.litellm.ai/#pricing) -[Contact us here to get a free trial](https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions) +[Contact us here to get a free trial](https://enterprise.litellm.ai/demo) ::: diff --git a/docs/my-website/docs/secret_managers/overview.md b/docs/my-website/docs/secret_managers/overview.md index bf7386ab89c..f02362f4932 100644 --- a/docs/my-website/docs/secret_managers/overview.md +++ b/docs/my-website/docs/secret_managers/overview.md @@ -8,7 +8,7 @@ import Image from '@theme/IdealImage'; [Enterprise Pricing](https://www.litellm.ai/#pricing) -[Contact us here to get a free trial](https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions) +[Contact us here to get a free trial](https://enterprise.litellm.ai/demo) ::: diff --git a/docs/my-website/docs/troubleshoot.md b/docs/my-website/docs/troubleshoot.md index 1539e1959f7..1afa35df8e4 100644 --- a/docs/my-website/docs/troubleshoot.md +++ b/docs/my-website/docs/troubleshoot.md @@ -50,7 +50,6 @@ Full error logs, stack traces, and any images from service metrics (CPU, memory, [Community Discord 💭](https://discord.gg/wuPM9dRgDw) [Community Slack 💭](https://www.litellm.ai/support) -Our numbers 📞 +1 (770) 8783-106 / +1 (412) 618-6238 Our emails ✉️ ishaan@berri.ai / krrish@berri.ai diff --git a/docs/my-website/docs/tutorials/claude_responses_api.md b/docs/my-website/docs/tutorials/claude_responses_api.md index 03ac9935fd2..2a6a1236ab1 100644 --- a/docs/my-website/docs/tutorials/claude_responses_api.md +++ b/docs/my-website/docs/tutorials/claude_responses_api.md @@ -214,7 +214,7 @@ model_list: # AWS Bedrock - model_name: claude-bedrock litellm_params: - model: bedrock/anthropic.claude-3-5-sonnet-20241022-v2:0 + model: bedrock/anthropic.claude-haiku-4-5-20251001:0 aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY aws_region_name: us-east-1 diff --git a/docs/my-website/docs/tutorials/compare_llms.md b/docs/my-website/docs/tutorials/compare_llms.md index 02877b46607..0252263a16e 100644 --- a/docs/my-website/docs/tutorials/compare_llms.md +++ b/docs/my-website/docs/tutorials/compare_llms.md @@ -82,7 +82,7 @@ Benchmark Results for 'When will BerriAI IPO?': +-----------------+----------------------------------------------------------------------------------+---------------------------+------------+ ``` ## Support -**🤝 Schedule a 1-on-1 Session:** Book a [1-on-1 session](https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions) with Krrish and Ishaan, the founders, to discuss any issues, provide feedback, or explore how we can improve LiteLLM for you. +**🤝 Schedule a 1-on-1 Session:** Book a [1-on-1 session](https://enterprise.litellm.ai/demo) with Krrish and Ishaan, the founders, to discuss any issues, provide feedback, or explore how we can improve LiteLLM for you.
Stripeimage Google ADK Greptile OpenHands